Implement field length introspection and make available in schema (#3552

)

Resolves #3551

.github/workflows/ci.yml

@@ -246,6 +246,10 @@ jobs:
         shell: bash
         run: cargo +${{ matrix.rust }} test --manifest-path diesel_migrations/migrations_macros/Cargo.toml --features "diesel/${{ matrix.backend }} ${{ matrix.backend }}"
 
+      - name: Test table-macro-syntax
+        shell: bash
+        run: cargo +${{ matrix.rust }} test --manifest-path diesel_table_macro_syntax/Cargo.toml
+
       - name: Test diesel_migrations
         shell: bash
         run: cargo +${{ matrix.rust }} test --manifest-path diesel_migrations/Cargo.toml --features "${{ matrix.backend }} diesel/${{ matrix.backend }}"

diesel/src/lib.rs

@@ -649,6 +649,7 @@ pub mod prelude {
     pub use crate::query_dsl::{
         BelongingToDsl, CombineDsl, JoinOnDsl, QueryDsl, RunQueryDsl, SaveChangesDsl,
     };
+    pub use crate::query_source::SizeRestrictedColumn as _;
     #[doc(inline)]
     pub use crate::query_source::{Column, JoinTo, QuerySource, Table};
     #[doc(inline)]

diesel/src/query_source/mod.rs

@@ -189,3 +189,12 @@ mod impls_which_are_only_here_to_improve_error_messages {
         type Selection = this_table_appears_in_your_query_more_than_once_and_must_be_aliased;
     }
 }
+
+/// Max length for columns of type Char/Varchar...
+///
+/// If a given column has a such constraint, this trait will be implemented and specify that
+/// length.
+pub trait SizeRestrictedColumn: Column {
+    /// Max length of that column
+    const MAX_LENGTH: usize;
+}

diesel_cli/src/infer_schema_internals/data_structures.rs

@@ -1,11 +1,7 @@
-#[cfg(feature = "uses_information_schema")]
-use diesel::backend::Backend;
-use diesel::deserialize::{self, FromStaticSqlRow, Queryable};
-#[cfg(feature = "sqlite")]
-use diesel::sqlite::Sqlite;
+use diesel_table_macro_syntax::ColumnDef;
+
+use std::error::Error;
 
-#[cfg(feature = "uses_information_schema")]
-use super::information_schema::DefaultSchema;
 use super::table_data::TableName;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -14,6 +10,7 @@ pub struct ColumnInformation {
     pub type_name: String,
     pub type_schema: Option<String>,
     pub nullable: bool,
+    pub max_length: Option<u64>,
     pub comment: Option<String>,
 }
 
@@ -25,10 +22,26 @@ pub struct ColumnType {
     pub is_array: bool,
     pub is_nullable: bool,
     pub is_unsigned: bool,
+    pub max_length: Option<u64>,
 }
 
-impl From<&syn::TypePath> for ColumnType {
-    fn from(t: &syn::TypePath) -> Self {
+impl ColumnType {
+    pub(crate) fn for_column_def(c: &ColumnDef) -> Result<Self, Box<dyn Error + Send + Sync>> {
+        Ok(Self::for_type_path(
+            &c.tpe,
+            c.max_length
+                .as_ref()
+                .map(|l| {
+                    l.base10_parse::<u64>()
+                        .map_err(|e| -> Box<dyn Error + Send + Sync> {
+                            format!("Column length literal can't be parsed as u64: {e}").into()
+                        })
+                })
+                .transpose()?,
+        ))
+    }
+
+    fn for_type_path(t: &syn::TypePath, max_length: Option<u64>) -> Self {
         let last = t
             .path
             .segments
@@ -42,6 +55,7 @@ impl From<&syn::TypePath> for ColumnType {
             is_array: last.ident == "Array",
             is_nullable: last.ident == "Nullable",
             is_unsigned: last.ident == "Unsigned",
+            max_length,
         };
 
         let sql_name = if !ret.is_nullable && !ret.is_array && !ret.is_unsigned {
@@ -53,7 +67,7 @@ impl From<&syn::TypePath> for ColumnType {
         } else if let syn::PathArguments::AngleBracketed(ref args) = last.arguments {
             let arg = args.args.first().expect("There is at least one argument");
             if let syn::GenericArgument::Type(syn::Type::Path(p)) = arg {
-                let s = Self::from(p);
+                let s = Self::for_type_path(p, max_length);
                 ret.is_nullable |= s.is_nullable;
                 ret.is_array |= s.is_array;
                 ret.is_unsigned |= s.is_unsigned;
@@ -110,6 +124,7 @@ impl ColumnInformation {
         type_name: U,
         type_schema: Option<String>,
         nullable: bool,
+        max_length: Option<u64>,
         comment: Option<String>,
     ) -> Self
     where
@@ -121,42 +136,12 @@ impl ColumnInformation {
             type_name: type_name.into(),
             type_schema,
             nullable,
+            max_length,
             comment,
         }
     }
 }
 
-#[cfg(feature = "uses_information_schema")]
-impl<ST, DB> Queryable<ST, DB> for ColumnInformation
-where
-    DB: Backend + DefaultSchema,
-    (String, String, Option<String>, String, Option<String>): FromStaticSqlRow<ST, DB>,
-{
-    type Row = (String, String, Option<String>, String, Option<String>);
-
-    fn build(row: Self::Row) -> deserialize::Result<Self> {
-        Ok(ColumnInformation::new(
-            row.0,
-            row.1,
-            row.2,
-            row.3 == "YES",
-            row.4,
-        ))
-    }
-}
-
-#[cfg(feature = "sqlite")]
-impl<ST> Queryable<ST, Sqlite> for ColumnInformation
-where
-    (i32, String, String, bool, Option<String>, bool, i32): FromStaticSqlRow<ST, Sqlite>,
-{
-    type Row = (i32, String, String, bool, Option<String>, bool, i32);
-
-    fn build(row: Self::Row) -> deserialize::Result<Self> {
-        Ok(ColumnInformation::new(row.1, row.2, None, !row.3, None))
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct ForeignKeyConstraint {
     pub child_table: TableName,

diesel_cli/src/infer_schema_internals/mysql.rs

@@ -1,3 +1,4 @@
+use diesel::deserialize::{self, FromStaticSqlRow, Queryable};
 use diesel::mysql::{Mysql, MysqlConnection};
 use diesel::*;
 use heck::ToUpperCamelCase;
@@ -33,14 +34,60 @@ pub fn get_table_data(
             column_type,
             type_schema,
             __is_nullable,
+            character_maximum_length,
             // MySQL comments are not nullable and are empty strings if not set
             null_if_text(column_comment, ""),
         ))
         .filter(table_name.eq(&table.sql_name))
         .filter(table_schema.eq(schema_name));
-    match column_sorting {
-        ColumnSorting::OrdinalPosition => query.order(ordinal_position).load(conn),
-        ColumnSorting::Name => query.order(column_name).load(conn),
+    let mut table_columns: Vec<ColumnInformation> = match column_sorting {
+        ColumnSorting::OrdinalPosition => query.order(ordinal_position).load(conn)?,
+        ColumnSorting::Name => query.order(column_name).load(conn)?,
+    };
+    for c in &mut table_columns {
+        if c.max_length.is_some() && !c.type_name.contains('(') {
+            // Mysql returns something in character_maximum_length regardless
+            // of whether it's specified at field creation time
+            // In addition there is typically a shared limitation at row level,
+            // so it's typically not even the real max.
+            // This basically means no max.
+            // https://dev.mysql.com/doc/refman/8.0/en/column-count-limit.html
+            // https://chartio.com/resources/tutorials/understanding-strorage-sizes-for-mysql-text-data-types/
+            c.max_length = None;
+        }
+    }
+    Ok(table_columns)
+}
+
+impl<ST> Queryable<ST, Mysql> for ColumnInformation
+where
+    (
+        String,
+        String,
+        Option<String>,
+        String,
+        Option<u64>,
+        Option<String>,
+    ): FromStaticSqlRow<ST, Mysql>,
+{
+    type Row = (
+        String,
+        String,
+        Option<String>,
+        String,
+        Option<u64>,
+        Option<String>,
+    );
+
+    fn build(row: Self::Row) -> deserialize::Result<Self> {
+        Ok(ColumnInformation::new(
+            row.0,
+            row.1,
+            row.2,
+            row.3 == "YES",
+            row.4,
+            row.5,
+        ))
     }
 }
 
@@ -84,7 +131,8 @@ mod information_schema {
             column_name -> VarChar,
             #[sql_name = "is_nullable"]
             __is_nullable -> VarChar,
-            ordinal_position -> BigInt,
+            character_maximum_length -> Nullable<Unsigned<BigInt>>,
+            ordinal_position -> Unsigned<BigInt>,
             udt_name -> VarChar,
             udt_schema -> VarChar,
             column_type -> VarChar,
@@ -171,6 +219,7 @@ pub fn determine_column_type(
         is_array: false,
         is_nullable: attr.nullable,
         is_unsigned: unsigned,
+        max_length: attr.max_length,
     })
 }
 
@@ -324,9 +373,11 @@ mod test {
             "varchar(255)",
             None,
             false,
+            Some(255),
             Some("column comment".to_string()),
         );
-        let id_without_comment = ColumnInformation::new("id", "varchar(255)", None, false, None);
+        let id_without_comment =
+            ColumnInformation::new("id", "varchar(255)", None, false, Some(255), None);
         assert_eq!(
             Ok(vec![id_with_comment]),
             get_table_data(&mut connection, &table_1, &ColumnSorting::OrdinalPosition)

diesel_cli/src/infer_schema_internals/pg.rs

@@ -2,7 +2,14 @@ use super::data_structures::*;
 use super::information_schema::DefaultSchema;
 use super::TableName;
 use crate::print_schema::ColumnSorting;
-use diesel::{dsl::AsExprOf, expression::AsExpression, pg::Pg, prelude::*, sql_types};
+use diesel::{
+    deserialize::{self, FromStaticSqlRow, Queryable},
+    dsl::AsExprOf,
+    expression::AsExpression,
+    pg::Pg,
+    prelude::*,
+    sql_types,
+};
 use heck::ToUpperCamelCase;
 use std::borrow::Cow;
 use std::error::Error;
@@ -48,6 +55,7 @@ pub fn determine_column_type(
         is_array,
         is_nullable: attr.nullable,
         is_unsigned: false,
+        max_length: attr.max_length,
     })
 }
 
@@ -79,6 +87,7 @@ pub fn get_table_data(
             udt_name,
             udt_schema.nullable(),
             __is_nullable,
+            character_maximum_length,
             col_description(regclass(table), ordinal_position),
         ))
         .filter(table_name.eq(&table.sql_name))
@@ -89,6 +98,44 @@ pub fn get_table_data(
     }
 }
 
+impl<ST> Queryable<ST, Pg> for ColumnInformation
+where
+    (
+        String,
+        String,
+        Option<String>,
+        String,
+        Option<i32>,
+        Option<String>,
+    ): FromStaticSqlRow<ST, Pg>,
+{
+    type Row = (
+        String,
+        String,
+        Option<String>,
+        String,
+        Option<i32>,
+        Option<String>,
+    );
+
+    fn build(row: Self::Row) -> deserialize::Result<Self> {
+        Ok(ColumnInformation::new(
+            row.0,
+            row.1,
+            row.2,
+            row.3 == "YES",
+            row.4
+                .map(|n| {
+                    std::convert::TryInto::try_into(n).map_err(|e| {
+                        format!("Max column length can't be converted to u64: {e} (got: {n})")
+                    })
+                })
+                .transpose()?,
+            row.5,
+        ))
+    }
+}
+
 sql_function!(fn obj_description(oid: sql_types::Oid, catalog: sql_types::Text) -> Nullable<Text>);
 
 pub fn get_table_comment(
@@ -108,6 +155,7 @@ mod information_schema {
             column_name -> VarChar,
             #[sql_name = "is_nullable"]
             __is_nullable -> VarChar,
+            character_maximum_length -> Nullable<Integer>,
             ordinal_position -> BigInt,
             udt_name -> VarChar,
             udt_schema -> VarChar,
@@ -221,7 +269,7 @@ mod test {
             .execute(&mut connection)
             .unwrap();
         diesel::sql_query(
-                "CREATE TABLE test_schema.table_1 (id SERIAL PRIMARY KEY, text_col VARCHAR, not_null TEXT NOT NULL)",
+                "CREATE TABLE test_schema.table_1 (id SERIAL PRIMARY KEY, text_col VARCHAR(128), not_null TEXT NOT NULL)",
             ).execute(&mut connection)
             .unwrap();
         diesel::sql_query("COMMENT ON COLUMN test_schema.table_1.id IS 'column comment'")
@@ -239,12 +287,21 @@ mod test {
             "int4",
             pg_catalog.clone(),
             false,
+            None,
             Some("column comment".to_string()),
         );
-        let text_col =
-            ColumnInformation::new("text_col", "varchar", pg_catalog.clone(), true, None);
-        let not_null = ColumnInformation::new("not_null", "text", pg_catalog.clone(), false, None);
-        let array_col = ColumnInformation::new("array_col", "_varchar", pg_catalog, false, None);
+        let text_col = ColumnInformation::new(
+            "text_col",
+            "varchar",
+            pg_catalog.clone(),
+            true,
+            Some(128),
+            None,
+        );
+        let not_null =
+            ColumnInformation::new("not_null", "text", pg_catalog.clone(), false, None, None);
+        let array_col =
+            ColumnInformation::new("array_col", "_varchar", pg_catalog, false, None, None);
         assert_eq!(
             Ok(vec![id, text_col, not_null]),
             get_table_data(&mut connection, &table_1, &ColumnSorting::OrdinalPosition)