From b5d2707f3c9198e65d35f4c9ea466acdcf4176fa Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 25 Jul 2024 11:17:46 +0800 Subject: [PATCH 1/4] [SPARK-48844][FOLLOWUP][TESTS] Cleanup duplicated data resource files in hive-thriftserver test --- .../analyzer-results/sql-on-files.sql.out | 92 ++++++++++++++---- .../sql-tests/inputs/sql-on-files.sql | 16 ++- .../sql-tests/results/sql-on-files.sql.out | 88 ++++++++++++++--- .../before_1582_date_v2_4.snappy.orc | Bin 201 -> 0 bytes .../src/test/resources/test-data/cars.csv | 7 -- .../test-data/dec-in-fixed-len.parquet | Bin 460 -> 0 bytes .../resources/test-data/with-map-fields.json | 5 - 7 files changed, 160 insertions(+), 48 deletions(-) delete mode 100644 sql/hive-thriftserver/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc delete mode 100644 sql/hive-thriftserver/src/test/resources/test-data/cars.csv delete mode 100644 sql/hive-thriftserver/src/test/resources/test-data/dec-in-fixed-len.parquet delete mode 100644 sql/hive-thriftserver/src/test/resources/test-data/with-map-fields.json diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out index 78e2a876da861..38bb5402e8e3c 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out @@ -1,4 +1,12 @@ -- Automatically generated by SQLQueryTestSuite +-- !query +CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1 +-- !query analysis +CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_PARQUET`, ErrorIfExists, [1] + +- Project [1 AS 1#x] + +- OneRowRelation + + -- !query SELECT * FROM parquet.`` -- !query analysis @@ -33,12 +41,25 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1 +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET` +-- !query analysis +Project [1#x] ++- Relation [1#x] parquet + + +-- !query +DROP TABLE TEST_PARQUET -- !query analysis -GlobalLimit 1 -+- LocalLimit 1 - +- Project [fixed_len_dec#x] - +- Relation [fixed_len_dec#x] parquet +DropTable false, false ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_PARQUET + + +-- !query +CREATE TABLE TEST_ORC USING ORC AS SELECT 1 +-- !query analysis +CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_ORC`, ErrorIfExists, [1] + +- Project [1 AS 1#x] + +- OneRowRelation -- !query @@ -75,12 +96,25 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1 +SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC` +-- !query analysis +Project [1#x] ++- Relation [1#x] orc + + +-- !query +DROP TABLE TEST_ORC -- !query analysis -GlobalLimit 1 -+- LocalLimit 1 - +- Project [dt#x] - +- Relation [dt#x] orc +DropTable false, false ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_ORC + + +-- !query +CREATE TABLE TEST_CSV USING CSV AS SELECT 1 +-- !query analysis +CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_CSV`, ErrorIfExists, [1] + +- Project [1 AS 1#x] + +- OneRowRelation -- !query @@ -117,12 +151,25 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1 +SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV` +-- !query analysis +Project [_c0#x] ++- Relation [_c0#x] csv + + +-- !query +DROP TABLE TEST_CSV -- !query analysis -GlobalLimit 1 -+- LocalLimit 1 - +- Project [_c0#x, _c1#x, _c2#x, _c3#x, _c4#x] - +- Relation [_c0#x,_c1#x,_c2#x,_c3#x,_c4#x] csv +DropTable false, false ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_CSV + + +-- !query +CREATE TABLE TEST_JSON USING JSON AS SELECT 1 +-- !query analysis +CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_JSON`, ErrorIfExists, [1] + +- Project [1 AS 1#x] + +- OneRowRelation -- !query @@ -159,9 +206,14 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1 +SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON` +-- !query analysis +Project [1#xL] ++- Relation [1#xL] json + + +-- !query +DROP TABLE TEST_JSON -- !query analysis -GlobalLimit 1 -+- LocalLimit 1 - +- Project [id#xL, intervals#x] - +- Relation [id#xL,intervals#x] json +DropTable false, false ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_JSON diff --git a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql index aee8aaa4d195b..9fcc7686f580b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql @@ -1,19 +1,27 @@ -- Parquet +CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1; SELECT * FROM parquet.``; SELECT * FROM parquet.`/file/not/found`; -SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1; +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET`; +DROP TABLE TEST_PARQUET; -- ORC +CREATE TABLE TEST_ORC USING ORC AS SELECT 1; SELECT * FROM orc.``; SELECT * FROM orc.`/file/not/found`; -SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1; +SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC`; +DROP TABLE TEST_ORC; -- CSV +CREATE TABLE TEST_CSV USING CSV AS SELECT 1; SELECT * FROM csv.``; SELECT * FROM csv.`/file/not/found`; -SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1; +SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV`; +DROP TABLE TEST_CSV; -- JSON +CREATE TABLE TEST_JSON USING JSON AS SELECT 1; SELECT * FROM json.``; SELECT * FROM json.`/file/not/found`; -SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1; +SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON`; +DROP TABLE TEST_JSON; diff --git a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out index 5c1e5697d029d..b65bc075e8199 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out @@ -1,4 +1,12 @@ -- Automatically generated by SQLQueryTestSuite +-- !query +CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1 +-- !query schema +struct<> +-- !query output + + + -- !query SELECT * FROM parquet.`` -- !query schema @@ -37,11 +45,27 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1 +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET` +-- !query schema +struct<1:int> +-- !query output +1 + + +-- !query +DROP TABLE TEST_PARQUET +-- !query schema +struct<> +-- !query output + + + +-- !query +CREATE TABLE TEST_ORC USING ORC AS SELECT 1 -- !query schema -struct +struct<> -- !query output -0.00 + -- !query @@ -82,11 +106,27 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1 +SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC` +-- !query schema +struct<1:int> +-- !query output +1 + + +-- !query +DROP TABLE TEST_ORC +-- !query schema +struct<> +-- !query output + + + +-- !query +CREATE TABLE TEST_CSV USING CSV AS SELECT 1 -- !query schema -struct +struct<> -- !query output -1200-01-01 + -- !query @@ -127,11 +167,27 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1 +SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV` +-- !query schema +struct<_c0:string> +-- !query output +1 + + +-- !query +DROP TABLE TEST_CSV +-- !query schema +struct<> +-- !query output + + + +-- !query +CREATE TABLE TEST_JSON USING JSON AS SELECT 1 -- !query schema -struct<_c0:string,_c1:string,_c2:string,_c3:string,_c4:string> +struct<> -- !query output -year make model comment blank + -- !query @@ -172,8 +228,16 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1 +SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON` -- !query schema -struct,b:struct>> +struct<1:bigint> -- !query output -1 {"a":{"endTime":211,"startTime":111},"b":{"endTime":221,"startTime":121}} +1 + + +-- !query +DROP TABLE TEST_JSON +-- !query schema +struct<> +-- !query output + diff --git a/sql/hive-thriftserver/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc b/sql/hive-thriftserver/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc deleted file mode 100644 index ebe01743b2e2043d9374630c5f24586b6e5df3cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 201 zcmeYdau#G@;9?VE;b074&<8R_xtJLk7=-vZ7_B%sX0K8b0O9}!ZU%-t29BxHjv&Px zYyu1t++dnfLJCMT3NT7=2r+Rm05Ov(P>BQ=*gP%)R8t~2Y8eT&8yPVu5U4^0 z3hK5WJW4SNaflr_N}BXrgbo(V<<*j?`)dokQEarvSr7`N-{ZFH k+I`P Date: Thu, 25 Jul 2024 13:51:38 +0800 Subject: [PATCH 2/4] [SPARK-48844][FOLLOWUP][TESTS] Cleanup duplicated data resource files in hive-thriftserver test --- .../sql-tests/analyzer-results/sql-on-files.sql.out | 8 ++++---- .../src/test/resources/sql-tests/inputs/sql-on-files.sql | 8 ++++---- .../test/resources/sql-tests/results/sql-on-files.sql.out | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out index 38bb5402e8e3c..d6c2d02a1e78a 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out @@ -41,7 +41,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET` +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet` -- !query analysis Project [1#x] +- Relation [1#x] parquet @@ -96,7 +96,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC` +SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc` -- !query analysis Project [1#x] +- Relation [1#x] orc @@ -151,7 +151,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV` +SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv` -- !query analysis Project [_c0#x] +- Relation [_c0#x] csv @@ -206,7 +206,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON` +SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json` -- !query analysis Project [1#xL] +- Relation [1#xL] json diff --git a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql index 9fcc7686f580b..34f3f178d5d75 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql @@ -2,26 +2,26 @@ CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1; SELECT * FROM parquet.``; SELECT * FROM parquet.`/file/not/found`; -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET`; +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet`; DROP TABLE TEST_PARQUET; -- ORC CREATE TABLE TEST_ORC USING ORC AS SELECT 1; SELECT * FROM orc.``; SELECT * FROM orc.`/file/not/found`; -SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC`; +SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc`; DROP TABLE TEST_ORC; -- CSV CREATE TABLE TEST_CSV USING CSV AS SELECT 1; SELECT * FROM csv.``; SELECT * FROM csv.`/file/not/found`; -SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV`; +SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv`; DROP TABLE TEST_CSV; -- JSON CREATE TABLE TEST_JSON USING JSON AS SELECT 1; SELECT * FROM json.``; SELECT * FROM json.`/file/not/found`; -SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON`; +SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json`; DROP TABLE TEST_JSON; diff --git a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out index b65bc075e8199..445de9302ce1e 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out @@ -45,7 +45,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/TEST_PARQUET` +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet` -- !query schema struct<1:int> -- !query output @@ -106,7 +106,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`${spark.sql.warehouse.dir}/TEST_ORC` +SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc` -- !query schema struct<1:int> -- !query output @@ -167,7 +167,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`${spark.sql.warehouse.dir}/TEST_CSV` +SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv` -- !query schema struct<_c0:string> -- !query output @@ -228,7 +228,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`${spark.sql.warehouse.dir}/TEST_JSON` +SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json` -- !query schema struct<1:bigint> -- !query output From 480f4313d5b9a2da8c4967c7d6d29901b28e7de8 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 25 Jul 2024 14:12:17 +0800 Subject: [PATCH 3/4] [SPARK-48844][FOLLOWUP][TESTS] Cleanup duplicated data resource files in hive-thriftserver test --- .../analyzer-results/sql-on-files.sql.out | 54 ++++++++++++------- .../sql-tests/inputs/sql-on-files.sql | 27 +++++----- .../sql-tests/results/sql-on-files.sql.out | 40 +++++++++----- 3 files changed, 77 insertions(+), 44 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out index d6c2d02a1e78a..bb718ed7eae9f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out @@ -1,8 +1,15 @@ -- Automatically generated by SQLQueryTestSuite -- !query -CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1 +CREATE DATABASE IF NOT EXISTS sql_on_files -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_PARQUET`, ErrorIfExists, [1] +CreateNamespace true ++- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files] + + +-- !query +CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1 +-- !query analysis +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_PARQUET`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -41,23 +48,23 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet` +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet` -- !query analysis Project [1#x] +- Relation [1#x] parquet -- !query -DROP TABLE TEST_PARQUET +DROP TABLE sql_on_files.TEST_PARQUET -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_PARQUET ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_PARQUET -- !query -CREATE TABLE TEST_ORC USING ORC AS SELECT 1 +CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_ORC`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_ORC`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -96,23 +103,23 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc` +SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc` -- !query analysis Project [1#x] +- Relation [1#x] orc -- !query -DROP TABLE TEST_ORC +DROP TABLE sql_on_files.TEST_ORC -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_ORC ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_ORC -- !query -CREATE TABLE TEST_CSV USING CSV AS SELECT 1 +CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_CSV`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_CSV`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -151,23 +158,23 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv` +SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv` -- !query analysis Project [_c0#x] +- Relation [_c0#x] csv -- !query -DROP TABLE TEST_CSV +DROP TABLE sql_on_files.TEST_CSV -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_CSV ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_CSV -- !query -CREATE TABLE TEST_JSON USING JSON AS SELECT 1 +CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`TEST_JSON`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_JSON`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -206,14 +213,21 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json` +SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json` -- !query analysis Project [1#xL] +- Relation [1#xL] json -- !query -DROP TABLE TEST_JSON +DROP TABLE sql_on_files.TEST_JSON -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.TEST_JSON ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_JSON + + +-- !query +DROP DATABASE sql_on_files +-- !query analysis +DropNamespace false, false ++- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files] diff --git a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql index 34f3f178d5d75..9e59c2eadaabf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql @@ -1,27 +1,30 @@ +CREATE DATABASE IF NOT EXISTS sql_on_files; -- Parquet -CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1; +CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1; SELECT * FROM parquet.``; SELECT * FROM parquet.`/file/not/found`; -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet`; -DROP TABLE TEST_PARQUET; +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`; +DROP TABLE sql_on_files.TEST_PARQUET; -- ORC -CREATE TABLE TEST_ORC USING ORC AS SELECT 1; +CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1; SELECT * FROM orc.``; SELECT * FROM orc.`/file/not/found`; -SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc`; -DROP TABLE TEST_ORC; +SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`; +DROP TABLE sql_on_files.TEST_ORC; -- CSV -CREATE TABLE TEST_CSV USING CSV AS SELECT 1; +CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1; SELECT * FROM csv.``; SELECT * FROM csv.`/file/not/found`; -SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv`; -DROP TABLE TEST_CSV; +SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`; +DROP TABLE sql_on_files.TEST_CSV; -- JSON -CREATE TABLE TEST_JSON USING JSON AS SELECT 1; +CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1; SELECT * FROM json.``; SELECT * FROM json.`/file/not/found`; -SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json`; -DROP TABLE TEST_JSON; +SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`; +DROP TABLE sql_on_files.TEST_JSON; + +DROP DATABASE sql_on_files; diff --git a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out index 445de9302ce1e..3dc5286b73e07 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out @@ -1,6 +1,14 @@ -- Automatically generated by SQLQueryTestSuite -- !query -CREATE TABLE TEST_PARQUET USING PARQUET AS SELECT 1 +CREATE DATABASE IF NOT EXISTS sql_on_files +-- !query schema +struct<> +-- !query output + + + +-- !query +CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1 -- !query schema struct<> -- !query output @@ -45,7 +53,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM parquet.`${spark.sql.warehouse.dir}/test_parquet` +SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet` -- !query schema struct<1:int> -- !query output @@ -53,7 +61,7 @@ struct<1:int> -- !query -DROP TABLE TEST_PARQUET +DROP TABLE sql_on_files.TEST_PARQUET -- !query schema struct<> -- !query output @@ -61,7 +69,7 @@ struct<> -- !query -CREATE TABLE TEST_ORC USING ORC AS SELECT 1 +CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1 -- !query schema struct<> -- !query output @@ -106,7 +114,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM orc.`${spark.sql.warehouse.dir}/test_orc` +SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc` -- !query schema struct<1:int> -- !query output @@ -114,7 +122,7 @@ struct<1:int> -- !query -DROP TABLE TEST_ORC +DROP TABLE sql_on_files.TEST_ORC -- !query schema struct<> -- !query output @@ -122,7 +130,7 @@ struct<> -- !query -CREATE TABLE TEST_CSV USING CSV AS SELECT 1 +CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1 -- !query schema struct<> -- !query output @@ -167,7 +175,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM csv.`${spark.sql.warehouse.dir}/test_csv` +SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv` -- !query schema struct<_c0:string> -- !query output @@ -175,7 +183,7 @@ struct<_c0:string> -- !query -DROP TABLE TEST_CSV +DROP TABLE sql_on_files.TEST_CSV -- !query schema struct<> -- !query output @@ -183,7 +191,7 @@ struct<> -- !query -CREATE TABLE TEST_JSON USING JSON AS SELECT 1 +CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1 -- !query schema struct<> -- !query output @@ -228,7 +236,7 @@ org.apache.spark.sql.AnalysisException -- !query -SELECT * FROM json.`${spark.sql.warehouse.dir}/test_json` +SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json` -- !query schema struct<1:bigint> -- !query output @@ -236,7 +244,15 @@ struct<1:bigint> -- !query -DROP TABLE TEST_JSON +DROP TABLE sql_on_files.TEST_JSON +-- !query schema +struct<> +-- !query output + + + +-- !query +DROP DATABASE sql_on_files -- !query schema struct<> -- !query output From fae3319fc8e4785d90f3a87b8e68b2ce3e0afbff Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 25 Jul 2024 14:59:59 +0800 Subject: [PATCH 4/4] address comments --- .../analyzer-results/sql-on-files.sql.out | 32 +++++++++---------- .../sql-tests/inputs/sql-on-files.sql | 16 +++++----- .../sql-tests/results/sql-on-files.sql.out | 16 +++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out index bb718ed7eae9f..b098a9758fe4e 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out @@ -7,9 +7,9 @@ CreateNamespace true -- !query -CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1 +CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_PARQUET`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_parquet`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -55,16 +55,16 @@ Project [1#x] -- !query -DROP TABLE sql_on_files.TEST_PARQUET +DROP TABLE sql_on_files.test_parquet -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_PARQUET ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_parquet -- !query -CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1 +CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_ORC`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_orc`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -110,16 +110,16 @@ Project [1#x] -- !query -DROP TABLE sql_on_files.TEST_ORC +DROP TABLE sql_on_files.test_orc -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_ORC ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_orc -- !query -CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1 +CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_CSV`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_csv`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -165,16 +165,16 @@ Project [_c0#x] -- !query -DROP TABLE sql_on_files.TEST_CSV +DROP TABLE sql_on_files.test_csv -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_CSV ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_csv -- !query -CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1 +CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1 -- !query analysis -CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`TEST_JSON`, ErrorIfExists, [1] +CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_json`, ErrorIfExists, [1] +- Project [1 AS 1#x] +- OneRowRelation @@ -220,10 +220,10 @@ Project [1#xL] -- !query -DROP TABLE sql_on_files.TEST_JSON +DROP TABLE sql_on_files.test_json -- !query analysis DropTable false, false -+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.TEST_JSON ++- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_json -- !query diff --git a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql index 9e59c2eadaabf..8a00e4400e6b0 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql @@ -1,30 +1,30 @@ CREATE DATABASE IF NOT EXISTS sql_on_files; -- Parquet -CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1; +CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1; SELECT * FROM parquet.``; SELECT * FROM parquet.`/file/not/found`; SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`; -DROP TABLE sql_on_files.TEST_PARQUET; +DROP TABLE sql_on_files.test_parquet; -- ORC -CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1; +CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1; SELECT * FROM orc.``; SELECT * FROM orc.`/file/not/found`; SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`; -DROP TABLE sql_on_files.TEST_ORC; +DROP TABLE sql_on_files.test_orc; -- CSV -CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1; +CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1; SELECT * FROM csv.``; SELECT * FROM csv.`/file/not/found`; SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`; -DROP TABLE sql_on_files.TEST_CSV; +DROP TABLE sql_on_files.test_csv; -- JSON -CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1; +CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1; SELECT * FROM json.``; SELECT * FROM json.`/file/not/found`; SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`; -DROP TABLE sql_on_files.TEST_JSON; +DROP TABLE sql_on_files.test_json; DROP DATABASE sql_on_files; diff --git a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out index 3dc5286b73e07..fc8f44bc22fee 100644 --- a/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out @@ -8,7 +8,7 @@ struct<> -- !query -CREATE TABLE sql_on_files.TEST_PARQUET USING PARQUET AS SELECT 1 +CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1 -- !query schema struct<> -- !query output @@ -61,7 +61,7 @@ struct<1:int> -- !query -DROP TABLE sql_on_files.TEST_PARQUET +DROP TABLE sql_on_files.test_parquet -- !query schema struct<> -- !query output @@ -69,7 +69,7 @@ struct<> -- !query -CREATE TABLE sql_on_files.TEST_ORC USING ORC AS SELECT 1 +CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1 -- !query schema struct<> -- !query output @@ -122,7 +122,7 @@ struct<1:int> -- !query -DROP TABLE sql_on_files.TEST_ORC +DROP TABLE sql_on_files.test_orc -- !query schema struct<> -- !query output @@ -130,7 +130,7 @@ struct<> -- !query -CREATE TABLE sql_on_files.TEST_CSV USING CSV AS SELECT 1 +CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1 -- !query schema struct<> -- !query output @@ -183,7 +183,7 @@ struct<_c0:string> -- !query -DROP TABLE sql_on_files.TEST_CSV +DROP TABLE sql_on_files.test_csv -- !query schema struct<> -- !query output @@ -191,7 +191,7 @@ struct<> -- !query -CREATE TABLE sql_on_files.TEST_JSON USING JSON AS SELECT 1 +CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1 -- !query schema struct<> -- !query output @@ -244,7 +244,7 @@ struct<1:bigint> -- !query -DROP TABLE sql_on_files.TEST_JSON +DROP TABLE sql_on_files.test_json -- !query schema struct<> -- !query output