From adcb8b4f9193da1128c8a30363f7d57c79dcc934 Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 11 Sep 2024 23:12:19 +0200 Subject: [PATCH 1/3] Add copy from database test - plus throw an error with unsupported CAST_FROM_VARCHAR composite column types --- src/postgres_scanner.cpp | 22 +++++++- src/postgres_utils.cpp | 2 +- .../storage/attach_copy_from_database.test | 52 +++++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 test/sql/storage/attach_copy_from_database.test diff --git a/src/postgres_scanner.cpp b/src/postgres_scanner.cpp index 2e5f2b4..dfee871 100644 --- a/src/postgres_scanner.cpp +++ b/src/postgres_scanner.cpp @@ -177,6 +177,18 @@ static unique_ptr PostgresBind(ClientContext &context, TableFuncti return std::move(bind_data); } +static bool ContainsCastToVarchar(const PostgresType &type) { + if (type.info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { + return true; + } + for (auto &child : type.children) { + if (ContainsCastToVarchar(child)) { + return true; + } + } + return false; +} + static void PostgresInitInternal(ClientContext &context, const PostgresBindData *bind_data_p, PostgresLocalState &lstate, idx_t task_min, idx_t task_max) { D_ASSERT(bind_data_p); @@ -200,14 +212,20 @@ static void PostgresInitInternal(ClientContext &context, const PostgresBindData col_names += KeywordHelper::WriteQuoted(bind_data->names[column_id], '"'); if (bind_data->postgres_types[column_id].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR"; - } - if (bind_data->types[column_id].id() == LogicalTypeId::LIST) { + } else if (bind_data->types[column_id].id() == LogicalTypeId::LIST) { if (bind_data->postgres_types[column_id].info != PostgresTypeAnnotation::STANDARD) { continue; } if (bind_data->postgres_types[column_id].children[0].info == PostgresTypeAnnotation::CAST_TO_VARCHAR) { col_names += "::VARCHAR[]"; } + } else { + if (ContainsCastToVarchar(bind_data->postgres_types[column_id])) { + throw NotImplementedException("Error reading table \"%s\" - cast to varchar not implemented for " + "composite column \"%s\" (type %s)", + bind_data->table_name, bind_data->names[column_id], + bind_data->types[column_id].ToString()); + } } } } diff --git a/src/postgres_utils.cpp b/src/postgres_utils.cpp index 757b7c4..a178af0 100644 --- a/src/postgres_utils.cpp +++ b/src/postgres_utils.cpp @@ -253,7 +253,7 @@ LogicalType PostgresUtils::ToPostgresType(const LogicalType &input) { case LogicalTypeId::UINTEGER: return LogicalType::BIGINT; case LogicalTypeId::UBIGINT: - return LogicalType::DECIMAL(20, 0); + return LogicalType::DECIMAL(20, 0); case LogicalTypeId::HUGEINT: return LogicalType::DOUBLE; default: diff --git a/test/sql/storage/attach_copy_from_database.test b/test/sql/storage/attach_copy_from_database.test new file mode 100644 index 0000000..459165d --- /dev/null +++ b/test/sql/storage/attach_copy_from_database.test @@ -0,0 +1,52 @@ +# name: test/sql/storage/attach_copy_from_database.test +# description: Test copy from database +# group: [storage] + +require postgres_scanner + +require-env POSTGRES_TEST_DATABASE_AVAILABLE + +statement ok +PRAGMA enable_verification + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES) + +statement ok +DROP SCHEMA IF EXISTS s1.copy_schema CASCADE + +statement ok +CREATE SCHEMA s1.copy_schema + +statement ok +USE s1.copy_schema + +foreach table_name pg_numtypes pg_bytetypes pg_datetypes + +statement ok +CREATE TABLE ${table_name} AS FROM public.${table_name} + +endloop + +statement ok +create table big_tbl as from range(100000) t(id) + +statement ok +create index i_index on big_tbl(id) + +statement ok +create view my_view as select min(id) from copy_schema.big_tbl + +statement ok +ATTACH '__TEST_DIR__/copy_database.db' AS new_db; + +statement ok +COPY FROM DATABASE s1 TO new_db + +foreach table_name pg_numtypes pg_bytetypes pg_datetypes big_tbl my_view + +query I +FROM new_db.${table_name} EXCEPT FROM ${table_name} +---- + +endloop From 91956a417af13acb24ff21a545ac81369e4cc6ed Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Wed, 11 Sep 2024 23:13:50 +0200 Subject: [PATCH 2/3] upload-artifact v3 --- .github/workflows/Linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index e9873e8..fb276b9 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -113,7 +113,7 @@ jobs: source ./create-postgres-tables.sh make test_release - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: ${{matrix.arch}}-extensions path: | From 08d523a966ccef842085db4f1f905c2fdbb81b1c Mon Sep 17 00:00:00 2001 From: Mark Raasveldt Date: Thu, 12 Sep 2024 09:33:17 +0200 Subject: [PATCH 3/3] Fix test --- Makefile | 4 ++-- duckdb | 2 +- test/copydb.sql | 0 test/sql/storage/attach_copy_from_database.test | 15 ++++++++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 test/copydb.sql diff --git a/Makefile b/Makefile index a5dd2a3..d662db7 100644 --- a/Makefile +++ b/Makefile @@ -72,9 +72,9 @@ release: test: test_release test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" + ./build/release/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*" test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" + ./build/debug/$(TEST_PATH) --test-dir "$(PROJ_DIR)" "test/*" format: cp duckdb/.clang-format . diff --git a/duckdb b/duckdb index 9067c64..fa5c2fe 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 9067c648ef182084b3159b72213097505d5b5cab +Subproject commit fa5c2fe15f3da5f32397b009196c0895fce60820 diff --git a/test/copydb.sql b/test/copydb.sql new file mode 100644 index 0000000..e69de29 diff --git a/test/sql/storage/attach_copy_from_database.test b/test/sql/storage/attach_copy_from_database.test index 459165d..8e68ec3 100644 --- a/test/sql/storage/attach_copy_from_database.test +++ b/test/sql/storage/attach_copy_from_database.test @@ -28,6 +28,18 @@ CREATE TABLE ${table_name} AS FROM public.${table_name} endloop +statement ok +USE memory + +statement ok +DETACH s1 + +statement ok +ATTACH 'dbname=postgresscanner' AS s1 (TYPE POSTGRES, SCHEMA 'copy_schema') + +statement ok +USE s1.copy_schema + statement ok create table big_tbl as from range(100000) t(id) @@ -46,7 +58,8 @@ COPY FROM DATABASE s1 TO new_db foreach table_name pg_numtypes pg_bytetypes pg_datetypes big_tbl my_view query I -FROM new_db.${table_name} EXCEPT FROM ${table_name} +SELECT COUNT(*) FROM (FROM new_db.copy_schema.${table_name} EXCEPT FROM ${table_name}) ---- +0 endloop