Skip to content

Commit

Permalink
[SPARK-48844][FOLLOWUP][TESTS] Cleanup duplicated data resource files…
Browse files Browse the repository at this point in the history 
… in hive-thriftserver test

### What changes were proposed in this pull request?

A follow up of SPARK-48844 to cleanup duplicated data resource files in hive-thriftserver test

### Why are the changes needed?

code refactoring

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

new tests

### Was this patch authored or co-authored using generative AI tooling?

no

Closes apache#47480 from yaooqinn/SPARK-48844-F.

Authored-by: Kent Yao <yao@apache.org>
Signed-off-by: Kent Yao <yao@apache.org>
  • Loading branch information
@yaooqinn
yaooqinn authored and himadripal committed Oct 19, 2024
1 parent e5487d2 commit 0ed04a3
Show file tree
Hide file tree
Showing 7 changed files with 193 additions and 48 deletions.
106 changes: 86 additions & 20 deletions sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,19 @@
-- Automatically generated by SQLQueryTestSuite
-- !query
CREATE DATABASE IF NOT EXISTS sql_on_files
-- !query analysis
CreateNamespace true
+- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files]


-- !query
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1
-- !query analysis
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_parquet`, ErrorIfExists, [1]
+- Project [1 AS 1#x]
+- OneRowRelation


-- !query
SELECT * FROM parquet.``
-- !query analysis
Expand Down Expand Up @@ -33,12 +48,25 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`
-- !query analysis
Project [1#x]
+- Relation [1#x] parquet


-- !query
DROP TABLE sql_on_files.test_parquet
-- !query analysis
DropTable false, false
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_parquet


-- !query
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1
-- !query analysis
GlobalLimit 1
+- LocalLimit 1
+- Project [fixed_len_dec#x]
+- Relation [fixed_len_dec#x] parquet
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_orc`, ErrorIfExists, [1]
+- Project [1 AS 1#x]
+- OneRowRelation


-- !query
Expand Down Expand Up @@ -75,12 +103,25 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`
-- !query analysis
Project [1#x]
+- Relation [1#x] orc


-- !query
DROP TABLE sql_on_files.test_orc
-- !query analysis
GlobalLimit 1
+- LocalLimit 1
+- Project [dt#x]
+- Relation [dt#x] orc
DropTable false, false
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_orc


-- !query
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1
-- !query analysis
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_csv`, ErrorIfExists, [1]
+- Project [1 AS 1#x]
+- OneRowRelation


-- !query
Expand Down Expand Up @@ -117,12 +158,25 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`
-- !query analysis
Project [_c0#x]
+- Relation [_c0#x] csv


-- !query
DROP TABLE sql_on_files.test_csv
-- !query analysis
DropTable false, false
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_csv


-- !query
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1
-- !query analysis
GlobalLimit 1
+- LocalLimit 1
+- Project [_c0#x, _c1#x, _c2#x, _c3#x, _c4#x]
+- Relation [_c0#x,_c1#x,_c2#x,_c3#x,_c4#x] csv
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_json`, ErrorIfExists, [1]
+- Project [1 AS 1#x]
+- OneRowRelation


-- !query
Expand Down Expand Up @@ -159,9 +213,21 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`
-- !query analysis
Project [1#xL]
+- Relation [1#xL] json


-- !query
DROP TABLE sql_on_files.test_json
-- !query analysis
DropTable false, false
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_json


-- !query
DROP DATABASE sql_on_files
-- !query analysis
GlobalLimit 1
+- LocalLimit 1
+- Project [id#xL, intervals#x]
+- Relation [id#xL,intervals#x] json
DropNamespace false, false
+- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files]
19 changes: 15 additions & 4 deletions sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,30 @@
CREATE DATABASE IF NOT EXISTS sql_on_files;
-- Parquet
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1;
SELECT * FROM parquet.``;
SELECT * FROM parquet.`/file/not/found`;
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1;
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`;
DROP TABLE sql_on_files.test_parquet;

-- ORC
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1;
SELECT * FROM orc.``;
SELECT * FROM orc.`/file/not/found`;
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1;
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`;
DROP TABLE sql_on_files.test_orc;

-- CSV
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1;
SELECT * FROM csv.``;
SELECT * FROM csv.`/file/not/found`;
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1;
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`;
DROP TABLE sql_on_files.test_csv;

-- JSON
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1;
SELECT * FROM json.``;
SELECT * FROM json.`/file/not/found`;
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1;
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`;
DROP TABLE sql_on_files.test_json;

DROP DATABASE sql_on_files;
104 changes: 92 additions & 12 deletions sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
-- Automatically generated by SQLQueryTestSuite
-- !query
CREATE DATABASE IF NOT EXISTS sql_on_files
-- !query schema
struct<>
-- !query output



-- !query
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1
-- !query schema
struct<>
-- !query output



-- !query
SELECT * FROM parquet.``
-- !query schema
Expand Down Expand Up @@ -37,11 +53,27 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`
-- !query schema
struct<fixed_len_dec:decimal(10,2)>
struct<1:int>
-- !query output
0.00
1


-- !query
DROP TABLE sql_on_files.test_parquet
-- !query schema
struct<>
-- !query output



-- !query
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1
-- !query schema
struct<>
-- !query output



-- !query
Expand Down Expand Up @@ -82,11 +114,27 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`
-- !query schema
struct<1:int>
-- !query output
1


-- !query
DROP TABLE sql_on_files.test_orc
-- !query schema
struct<>
-- !query output



-- !query
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1
-- !query schema
struct<dt:date>
struct<>
-- !query output
1200-01-01



-- !query
Expand Down Expand Up @@ -127,11 +175,27 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`
-- !query schema
struct<_c0:string>
-- !query output
1


-- !query
DROP TABLE sql_on_files.test_csv
-- !query schema
struct<_c0:string,_c1:string,_c2:string,_c3:string,_c4:string>
struct<>
-- !query output
year make model comment blank



-- !query
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1
-- !query schema
struct<>
-- !query output



-- !query
Expand Down Expand Up @@ -172,8 +236,24 @@ org.apache.spark.sql.AnalysisException


-- !query
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`
-- !query schema
struct<id:bigint,intervals:struct<a:struct<endTime:bigint,startTime:bigint>,b:struct<endTime:bigint,startTime:bigint>>>
struct<1:bigint>
-- !query output
1


-- !query
DROP TABLE sql_on_files.test_json
-- !query schema
struct<>
-- !query output
1 {"a":{"endTime":211,"startTime":111},"b":{"endTime":221,"startTime":121}}



-- !query
DROP DATABASE sql_on_files
-- !query schema
struct<>
-- !query output

Binary file removed sql/hive-thriftserver/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc
View file
Binary file not shown.
7 changes: 0 additions & 7 deletions sql/hive-thriftserver/src/test/resources/test-data/cars.csv
View file

This file was deleted.

Binary file removed sql/hive-thriftserver/src/test/resources/test-data/dec-in-fixed-len.parquet
View file
Binary file not shown.
5 changes: 0 additions & 5 deletions sql/hive-thriftserver/src/test/resources/test-data/with-map-fields.json
View file

This file was deleted.

0 comments on commit 0ed04a3

Please sign in to comment.