From 6ea2c1d1a3ffefb2bba8a9c1746473c2bf4ad190 Mon Sep 17 00:00:00 2001 From: mplmoknijb Date: Wed, 6 Nov 2024 15:14:40 +0800 Subject: [PATCH] [#5014]feat: add the basic module of catalog-jdbc-maxcompute --- build.gradle.kts | 3 +- .../catalog-jdbc-maxcompute/build.gradle.kts | 103 +++++++++++++++ .../catalog/maxcompute/MaxComputeCatalog.java | 69 ++++++++++ .../MaxComputeCatalogCapability.java | 23 ++++ .../MaxComputeCatalogOperations.java | 43 ++++++ ...MaxComputeColumnDefaultValueConverter.java | 100 ++++++++++++++ .../converter/MaxComputeTypeConverter.java | 124 ++++++++++++++++++ .../operation/MaxComputeSchemaOpeartions.java | 35 +++++ .../operation/MaxComputeTableOpeations.java | 70 ++++++++++ .../src/main/resources/jdbc-maxcompute.conf | 22 ++++ .../org.apache.gravitino.CatalogProvider | 19 +++ .../TestMaxComputeTypeConverter.java | 109 +++++++++++++++ .../maxcompute/operation/TestMaxCompute.java | 82 ++++++++++++ .../TestMaxComputeCatalogOperations.java | 34 +++++ .../TestMaxComputeTableOperations.java | 69 ++++++++++ .../resources/resources/log4j2.properties | 73 +++++++++++ gradle/libs.versions.toml | 2 + settings.gradle.kts | 3 +- web/integration-test/build.gradle.kts | 1 + 19 files changed, 982 insertions(+), 2 deletions(-) create mode 100644 catalogs/catalog-jdbc-maxcompute/build.gradle.kts create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalog.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogCapability.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogOperations.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeColumnDefaultValueConverter.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeTypeConverter.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeSchemaOpeartions.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeTableOpeations.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/resources/jdbc-maxcompute.conf create mode 100644 catalogs/catalog-jdbc-maxcompute/src/main/resources/services/org.apache.gravitino.CatalogProvider create mode 100644 catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/converter/TestMaxComputeTypeConverter.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxCompute.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeCatalogOperations.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeTableOperations.java create mode 100644 catalogs/catalog-jdbc-maxcompute/src/test/resources/resources/log4j2.properties diff --git a/build.gradle.kts b/build.gradle.kts index e6c49df4064..4421959bd15 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -29,7 +29,7 @@ import org.gradle.internal.hash.ChecksumService import org.gradle.internal.os.OperatingSystem import org.gradle.kotlin.dsl.support.serviceOf import java.io.IOException -import java.util.Locale +import java.util.* plugins { `maven-publish` @@ -809,6 +809,7 @@ tasks { ":catalogs:catalog-jdbc-mysql:copyLibAndConfig", ":catalogs:catalog-jdbc-oceanbase:copyLibAndConfig", ":catalogs:catalog-jdbc-postgresql:copyLibAndConfig", + ":catalogs:catalog-jdbc-maxcompute:copyLibAndConfig", ":catalogs:catalog-hadoop:copyLibAndConfig", ":catalogs:catalog-kafka:copyLibAndConfig" ) diff --git a/catalogs/catalog-jdbc-maxcompute/build.gradle.kts b/catalogs/catalog-jdbc-maxcompute/build.gradle.kts new file mode 100644 index 00000000000..752f3b30439 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/build.gradle.kts @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "catalog-jdbc-maxcompute" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + + implementation(project(":api")) { + exclude(group = "*") + } + implementation(project(":catalogs:catalog-jdbc-common")) { + exclude(group = "*") + } + implementation(project(":common")) { + exclude(group = "*") + } + implementation(project(":core")) { + exclude(group = "*") + } + + implementation(libs.bundles.log4j) + implementation(libs.commons.collections4) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.odps.jdbc) + implementation(libs.postgresql.driver) + implementation(project(mapOf("path" to ":catalogs:catalog-jdbc-common"))) + + testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.odps.jdbc) + testImplementation(libs.postgresql.driver) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + val copyCatalogLibs by registering(Copy::class) { + dependsOn("jar", "runtimeJars") + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } + into("$rootDir/distribution/package/catalogs/jdbc-maxcompute/libs") + } + + val copyCatalogConfig by registering(Copy::class) { + from("src/main/resources") + into("$rootDir/distribution/package/catalogs/jdbc-maxcompute/conf") + + include("jdbc-maxcompute.conf") + + exclude { details -> + details.file.isDirectory() + } + + fileMode = 0b111101101 + } + + val copyLibAndConfig by registering(Copy::class) { + dependsOn(copyCatalogLibs, copyCatalogConfig) + } +} + +tasks.test{ + enabled = false +} + +tasks.getByName("generateMetadataFileForMavenJavaPublication") { + dependsOn("runtimeJars") +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalog.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalog.java new file mode 100644 index 00000000000..dd70b155008 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalog.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute; + +import java.util.Map; +import org.apache.gravitino.catalog.jdbc.JdbcCatalog; +import org.apache.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import org.apache.gravitino.catalog.maxcompute.converter.MaxComputeColumnDefaultValueConverter; +import org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter; +import org.apache.gravitino.catalog.maxcompute.operation.MaxComputeSchemaOpeartions; +import org.apache.gravitino.catalog.maxcompute.operation.MaxComputeTableOpeations; +import org.apache.gravitino.connector.CatalogOperations; + +public class MaxComputeCatalog extends JdbcCatalog { + + @Override + public String shortName() { + return "jdbc-maxcompute"; + } + + @Override + protected CatalogOperations newOps(Map config) { + return new MaxComputeCatalogOperations( + createExceptionConverter(), + createJdbcTypeConverter(), + createJdbcDatabaseOperations(), + createJdbcTableOperations(), + createJdbcColumnDefaultValueConverter()); + } + + @Override + protected JdbcTypeConverter createJdbcTypeConverter() { + return new MaxComputeTypeConverter(); + } + + @Override + protected JdbcDatabaseOperations createJdbcDatabaseOperations() { + return new MaxComputeSchemaOpeartions(); + } + + @Override + protected JdbcTableOperations createJdbcTableOperations() { + return new MaxComputeTableOpeations(); + } + + @Override + protected JdbcColumnDefaultValueConverter createJdbcColumnDefaultValueConverter() { + return new MaxComputeColumnDefaultValueConverter(); + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogCapability.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogCapability.java new file mode 100644 index 00000000000..cf759fd2785 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogCapability.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute; + +import org.apache.gravitino.catalog.jdbc.JdbcCatalogCapability; + +public class MaxComputeCatalogCapability extends JdbcCatalogCapability {} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogOperations.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogOperations.java new file mode 100644 index 00000000000..23002066a78 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/MaxComputeCatalogOperations.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute; + +import org.apache.gravitino.catalog.jdbc.JdbcCatalogOperations; +import org.apache.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import org.apache.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations; + +public class MaxComputeCatalogOperations extends JdbcCatalogOperations { + + public MaxComputeCatalogOperations( + JdbcExceptionConverter exceptionConverter, + JdbcTypeConverter jdbcTypeConverter, + JdbcDatabaseOperations databaseOperation, + JdbcTableOperations tableOperation, + JdbcColumnDefaultValueConverter columnDefaultValueConverter) { + super( + exceptionConverter, + jdbcTypeConverter, + databaseOperation, + tableOperation, + columnDefaultValueConverter); + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeColumnDefaultValueConverter.java new file mode 100644 index 00000000000..5bad67e8e12 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeColumnDefaultValueConverter.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.converter; + +import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; +import static org.apache.gravitino.rel.Column.DEFAULT_VALUE_OF_CURRENT_TIMESTAMP; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import org.apache.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.rel.expressions.Expression; +import org.apache.gravitino.rel.expressions.UnparsedExpression; +import org.apache.gravitino.rel.expressions.literals.Literals; +import org.apache.gravitino.rel.types.Decimal; +import org.apache.gravitino.rel.types.Types; + +public class MaxComputeColumnDefaultValueConverter extends JdbcColumnDefaultValueConverter { + + @Override + public Expression toGravitino( + JdbcTypeConverter.JdbcTypeBean type, + String columnDefaultValue, + boolean isExpression, + boolean nullable) { + if (columnDefaultValue == null) { + return nullable ? Literals.NULL : DEFAULT_VALUE_NOT_SET; + } + + if (columnDefaultValue.equalsIgnoreCase(NULL)) { + return Literals.NULL; + } + + if (isExpression) { + if (columnDefaultValue.equals(CURRENT_TIMESTAMP)) { + return DEFAULT_VALUE_OF_CURRENT_TIMESTAMP; + } + // The parsing of MaxCompute expressions is complex, so we are not currently undertaking the + // parsing. + return UnparsedExpression.of(columnDefaultValue); + } + + switch (type.getTypeName().toLowerCase()) { + case MaxComputeTypeConverter.TINYINT: + return Literals.byteLiteral(Byte.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.SMALLINT: + return Literals.shortLiteral(Short.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.INT: + return Literals.integerLiteral(Integer.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.BIGINT: + return Literals.longLiteral(Long.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.FLOAT: + return Literals.floatLiteral(Float.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.DOUBLE: + return Literals.doubleLiteral(Double.valueOf(columnDefaultValue)); + case MaxComputeTypeConverter.DECIMAL: + return Literals.decimalLiteral( + Decimal.of( + columnDefaultValue, + Integer.parseInt(type.getColumnSize()), + Integer.parseInt(type.getScale()))); + case JdbcTypeConverter.DATE: + return Literals.dateLiteral(LocalDate.parse(columnDefaultValue, DATE_FORMATTER)); + case JdbcTypeConverter.TIMESTAMP: + case MaxComputeTypeConverter.DATETIME: + return CURRENT_TIMESTAMP.equals(columnDefaultValue) + ? DEFAULT_VALUE_OF_CURRENT_TIMESTAMP + : Literals.timestampLiteral( + LocalDateTime.parse(columnDefaultValue, DATE_TIME_FORMATTER)); + case JdbcTypeConverter.VARCHAR: + return Literals.of( + columnDefaultValue, Types.VarCharType.of(Integer.parseInt(type.getColumnSize()))); + case MaxComputeTypeConverter.CHAR: + return Literals.of( + columnDefaultValue, Types.FixedCharType.of(Integer.parseInt(type.getColumnSize()))); + case MaxComputeTypeConverter.STRING: + return Literals.stringLiteral(columnDefaultValue); + case MaxComputeTypeConverter.BOOLEAN: + return Literals.booleanLiteral(Boolean.valueOf(columnDefaultValue)); + default: + return UnparsedExpression.of(columnDefaultValue); + } + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeTypeConverter.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeTypeConverter.java new file mode 100644 index 00000000000..69b2eea43df --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/converter/MaxComputeTypeConverter.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.converter; + +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; + +public class MaxComputeTypeConverter extends JdbcTypeConverter { + + static final String TINYINT = "tinyint"; + static final String SMALLINT = "smallint"; + static final String INT = "int"; + static final String BIGINT = "bigint"; + static final String BINARY = "binary"; + static final String FLOAT = "float"; + static final String DOUBLE = "double"; + static final String DECIMAL = "decimal"; + static final String VARCHAR = "varchar"; + static final String CHAR = "char"; + static final String STRING = "string"; + static final String DATE = "date"; + static final String DATETIME = "datetime"; + static final String TIMESTAMP = "timestamp"; + static final String BOOLEAN = "boolean"; + + @Override + public Type toGravitino(JdbcTypeBean typeBean) { + switch (typeBean.getTypeName().toLowerCase()) { + case TINYINT: + return Types.ByteType.get(); + case SMALLINT: + return Types.ShortType.get(); + case INT: + return Types.IntegerType.get(); + case BIGINT: + return Types.LongType.get(); + case BINARY: + return Types.BinaryType.get(); + case FLOAT: + return Types.FloatType.get(); + case DOUBLE: + return Types.DoubleType.get(); + case DECIMAL: + return Types.DecimalType.of( + Integer.parseInt(typeBean.getColumnSize()), Integer.parseInt(typeBean.getScale())); + case VARCHAR: + return Types.VarCharType.of(Integer.parseInt(typeBean.getColumnSize())); + case CHAR: + return Types.FixedCharType.of(Integer.parseInt(typeBean.getColumnSize())); + case STRING: + return Types.StringType.get(); + case DATE: + return Types.DateType.get(); + case DATETIME: + return Types.TimestampType.withoutTimeZone(); + case TIMESTAMP: + return Types.TimestampType.withTimeZone(); + case BOOLEAN: + return Types.BooleanType.get(); + default: + return Types.ExternalType.of(typeBean.getTypeName()); + } + } + + @Override + public String fromGravitino(Type type) { + if (type instanceof Types.ByteType) { + return TINYINT; + } else if (type instanceof Types.ShortType) { + return SMALLINT; + } else if (type instanceof Types.IntegerType) { + return INT; + } else if (type instanceof Types.LongType) { + return BIGINT; + } else if (type instanceof Types.BinaryType) { + return type.simpleString(); + } else if (type instanceof Types.FloatType) { + return type.simpleString(); + } else if (type instanceof Types.DoubleType) { + return type.simpleString(); + } else if (type instanceof Types.DecimalType) { + return DECIMAL + + "(" + + ((Types.DecimalType) type).precision() + + "," + + ((Types.DecimalType) type).scale() + + ")"; + } else if (type instanceof Types.VarCharType) { + return VARCHAR + "(" + ((Types.VarCharType) type).length() + ")"; + } else if (type instanceof Types.FixedCharType) { + return CHAR + "(" + ((Types.FixedCharType) type).length() + ")"; + } else if (type instanceof Types.StringType) { + return STRING; + } else if (type instanceof Types.DateType) { + return DATE; + } else if (type instanceof Types.TimestampType) { + return ((Types.TimestampType) type).hasTimeZone() ? TIMESTAMP : DATETIME; + } else if (type instanceof Types.BooleanType) { + return BOOLEAN; + } else if (type instanceof Types.ExternalType) { + return ((Types.ExternalType) type).catalogString(); + } + throw new IllegalArgumentException( + String.format( + "Couldn't convert Gravitino type %s to MaxCompute type", type.simpleString())); + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeSchemaOpeartions.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeSchemaOpeartions.java new file mode 100644 index 00000000000..f110343be90 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeSchemaOpeartions.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.operation; + +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.apache.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; + +public class MaxComputeSchemaOpeartions extends JdbcDatabaseOperations { + @Override + protected boolean supportSchemaComment() { + return false; + } + + @Override + protected Set createSysDatabaseNameSet() { + return ImmutableSet.of("information_schema", "SYSTEM_CATALOG"); + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeTableOpeations.java b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeTableOpeations.java new file mode 100644 index 00000000000..aca0f3bc770 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/java/org/apache/gravitino/catalog/maxcompute/operation/MaxComputeTableOpeations.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.operation; + +import java.util.Map; +import org.apache.gravitino.catalog.jdbc.JdbcColumn; +import org.apache.gravitino.catalog.jdbc.JdbcTable; +import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import org.apache.gravitino.rel.TableChange; +import org.apache.gravitino.rel.expressions.distributions.Distribution; +import org.apache.gravitino.rel.expressions.transforms.Transform; +import org.apache.gravitino.rel.indexes.Index; + +public class MaxComputeTableOpeations extends JdbcTableOperations { + + @Override + protected String generateCreateTableSql( + String tableName, + JdbcColumn[] columns, + String comment, + Map properties, + Transform[] partitioning, + Distribution distribution, + Index[] indexes) { + return null; + } + + @Override + protected String generateRenameTableSql(String oldTableName, String newTableName) { + return null; + } + + @Override + protected String generateDropTableSql(String tableName) { + return null; + } + + @Override + protected String generatePurgeTableSql(String tableName) { + return null; + } + + @Override + protected String generateAlterTableSql( + String databaseName, String tableName, TableChange... changes) { + return null; + } + + @Override + protected JdbcTable getOrCreateTable( + String databaseName, String tableName, JdbcTable lazyLoadCreateTable) { + return null; + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/resources/jdbc-maxcompute.conf b/catalogs/catalog-jdbc-maxcompute/src/main/resources/jdbc-maxcompute.conf new file mode 100644 index 00000000000..6f25765c82b --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/resources/jdbc-maxcompute.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# jdbc-url = jdbc:odps:https://service.cn-hangzhou.maxcompute.aliyun.com/api?project=${your_project}&useProjectTimeZone=true +# jdbc-user = user +# jdbc-password = password +# jdbc-driver = com.aliyun.odps.jdbc.OdpsDriver \ No newline at end of file diff --git a/catalogs/catalog-jdbc-maxcompute/src/main/resources/services/org.apache.gravitino.CatalogProvider b/catalogs/catalog-jdbc-maxcompute/src/main/resources/services/org.apache.gravitino.CatalogProvider new file mode 100644 index 00000000000..1c50ef00e0d --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/main/resources/services/org.apache.gravitino.CatalogProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.catalog.maxcompute.MaxComputeCatalog \ No newline at end of file diff --git a/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/converter/TestMaxComputeTypeConverter.java b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/converter/TestMaxComputeTypeConverter.java new file mode 100644 index 00000000000..cd71e915bf3 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/converter/TestMaxComputeTypeConverter.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.converter; + +import static org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter.DATE; +import static org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter.TIMESTAMP; +import static org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter.VARCHAR; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.BIGINT; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.BINARY; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.BOOLEAN; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.CHAR; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.DATETIME; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.DECIMAL; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.DOUBLE; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.FLOAT; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.INT; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.STRING; +import static org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter.TINYINT; + +import org.apache.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import org.apache.gravitino.rel.types.Type; +import org.apache.gravitino.rel.types.Types; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestMaxComputeTypeConverter { + + private static final MaxComputeTypeConverter MAXCOMPUTE_TYPE_CONVERTER = + new MaxComputeTypeConverter(); + private static final String USER_DEFINED_TYPE = "user-defined"; + + @Test + public void testToGravitinoType() { + checkJdbcTypeToGravitinoType(Types.ByteType.get(), TINYINT, null, null); + checkJdbcTypeToGravitinoType(Types.IntegerType.get(), INT, null, null); + checkJdbcTypeToGravitinoType(Types.LongType.get(), BIGINT, null, null); + checkJdbcTypeToGravitinoType(Types.BinaryType.get(), BINARY, null, null); + checkJdbcTypeToGravitinoType(Types.FloatType.get(), FLOAT, null, null); + checkJdbcTypeToGravitinoType(Types.DoubleType.get(), DOUBLE, null, null); + checkJdbcTypeToGravitinoType(Types.DecimalType.of(10, 2), DECIMAL, "10", "2"); + checkJdbcTypeToGravitinoType(Types.DateType.get(), DATE, null, null); + checkJdbcTypeToGravitinoType(Types.TimestampType.withoutTimeZone(), DATETIME, null, null); + checkJdbcTypeToGravitinoType(Types.TimestampType.withTimeZone(), TIMESTAMP, null, null); + checkJdbcTypeToGravitinoType(Types.VarCharType.of(20), VARCHAR, "20", null); + checkJdbcTypeToGravitinoType(Types.FixedCharType.of(20), CHAR, "20", null); + checkJdbcTypeToGravitinoType(Types.StringType.get(), STRING, null, null); + checkJdbcTypeToGravitinoType(Types.BooleanType.get(), BOOLEAN, null, null); + checkJdbcTypeToGravitinoType( + Types.ExternalType.of(USER_DEFINED_TYPE), USER_DEFINED_TYPE, null, null); + } + + @Test + public void testFromGravitinoType() { + checkGravitinoTypeToJdbcType(TINYINT, Types.ByteType.get()); + checkGravitinoTypeToJdbcType(INT, Types.IntegerType.get()); + checkGravitinoTypeToJdbcType(BIGINT, Types.LongType.get()); + checkGravitinoTypeToJdbcType(BINARY, Types.BinaryType.get()); + checkGravitinoTypeToJdbcType(FLOAT, Types.FloatType.get()); + checkGravitinoTypeToJdbcType(DOUBLE, Types.DoubleType.get()); + checkGravitinoTypeToJdbcType(DECIMAL + "(10,2)", Types.DecimalType.of(10, 2)); + checkGravitinoTypeToJdbcType(DATE, Types.DateType.get()); + checkGravitinoTypeToJdbcType(TIMESTAMP, Types.TimestampType.withTimeZone()); + checkGravitinoTypeToJdbcType(DATETIME, Types.TimestampType.withoutTimeZone()); + checkGravitinoTypeToJdbcType(VARCHAR + "(20)", Types.VarCharType.of(20)); + checkGravitinoTypeToJdbcType(CHAR + "(20)", Types.FixedCharType.of(20)); + checkGravitinoTypeToJdbcType(STRING, Types.StringType.get()); + checkGravitinoTypeToJdbcType(BOOLEAN, Types.BooleanType.get()); + checkGravitinoTypeToJdbcType(USER_DEFINED_TYPE, Types.ExternalType.of(USER_DEFINED_TYPE)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> MAXCOMPUTE_TYPE_CONVERTER.fromGravitino(Types.UnparsedType.of(USER_DEFINED_TYPE))); + } + + protected void checkGravitinoTypeToJdbcType(String jdbcTypeName, Type gravitinoType) { + Assertions.assertEquals(jdbcTypeName, MAXCOMPUTE_TYPE_CONVERTER.fromGravitino(gravitinoType)); + } + + protected void checkJdbcTypeToGravitinoType( + Type gravitinoType, String jdbcTypeName, String columnSize, String scale) { + JdbcTypeConverter.JdbcTypeBean typeBean = createTypeBean(jdbcTypeName, columnSize, scale); + Assertions.assertEquals(gravitinoType, MAXCOMPUTE_TYPE_CONVERTER.toGravitino(typeBean)); + } + + protected static JdbcTypeConverter.JdbcTypeBean createTypeBean( + String typeName, String columnSize, String scale) { + return new JdbcTypeConverter.JdbcTypeBean(typeName) { + { + setColumnSize(columnSize); + setScale(scale); + } + }; + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxCompute.java b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxCompute.java new file mode 100644 index 00000000000..d6439e535cd --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxCompute.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.operation; + +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import javax.sql.DataSource; +import org.apache.gravitino.catalog.jdbc.config.JdbcConfig; +import org.apache.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import org.apache.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import org.apache.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import org.apache.gravitino.catalog.jdbc.utils.DataSourceUtils; +import org.apache.gravitino.catalog.maxcompute.converter.MaxComputeColumnDefaultValueConverter; +import org.apache.gravitino.catalog.maxcompute.converter.MaxComputeTypeConverter; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +public class TestMaxCompute { + + protected static JdbcDatabaseOperations DATABASE_OPERATIONS; + + protected static JdbcTableOperations TABLE_OPERATIONS; + + protected static JdbcExceptionConverter JDBC_EXCEPTION_CONVERTER; + + protected static DataSource DATA_SOURCE; + + private static final String DRIVER_CLASS_NAME = "com.aliyun.odps.jdbc.OdpsDriver"; + private static String ALIBABA_CLOUD_ACCESS_KEY_ID = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID"); + private static String ALIBABA_CLOUD_ACCESS_KEY_SECRET = + System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET"); + private static String ALIBABA_CLOUD_JDBC_URL = System.getenv("ALIBABA_CLOUD_JDBC_URL"); + + @BeforeAll + public static void startup() { + DATA_SOURCE = DataSourceUtils.createDataSource(getMaxComputeCatalogProperties()); + + DATABASE_OPERATIONS = new MaxComputeSchemaOpeartions(); + TABLE_OPERATIONS = new MaxComputeTableOpeations(); + DATABASE_OPERATIONS.initialize(DATA_SOURCE, JDBC_EXCEPTION_CONVERTER, Collections.emptyMap()); + TABLE_OPERATIONS.initialize( + DATA_SOURCE, + JDBC_EXCEPTION_CONVERTER, + new MaxComputeTypeConverter(), + new MaxComputeColumnDefaultValueConverter(), + Collections.emptyMap()); + } + + // Overwrite the stop method to close the data source and stop the container + @AfterAll + public static void stop() { + DataSourceUtils.closeDataSource(DATA_SOURCE); + } + + private static Map getMaxComputeCatalogProperties() { + Map catalogProperties = Maps.newHashMap(); + + catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), ALIBABA_CLOUD_JDBC_URL); + catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), DRIVER_CLASS_NAME); + catalogProperties.put(JdbcConfig.USERNAME.getKey(), ALIBABA_CLOUD_ACCESS_KEY_ID); + catalogProperties.put(JdbcConfig.PASSWORD.getKey(), ALIBABA_CLOUD_ACCESS_KEY_SECRET); + + return catalogProperties; + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeCatalogOperations.java b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeCatalogOperations.java new file mode 100644 index 00000000000..a6efd1da70b --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeCatalogOperations.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.operation; + +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestMaxComputeCatalogOperations extends TestMaxCompute { + + @Test + public void testListDatabases() { + List initDatabases = DATABASE_OPERATIONS.listDatabases(); + + Assertions.assertFalse(CollectionUtils.isEmpty(initDatabases)); + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeTableOperations.java b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeTableOperations.java new file mode 100644 index 00000000000..45a9ff09857 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/test/java/org/apache/gravitino/catalog/maxcompute/operation/TestMaxComputeTableOperations.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.maxcompute.operation; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestMaxComputeTableOperations extends TestMaxCompute { + + private static final Logger LOG = LoggerFactory.getLogger(TestMaxComputeTableOperations.class); + + @Test + public void testListTableColumns() throws SQLException { + List databases = DATABASE_OPERATIONS.listDatabases(); + Assertions.assertFalse(CollectionUtils.isEmpty(databases)); + + String catalog = databases.get(1); + String schemaPattern = null; + String tableNamePattern = "%"; + String[] types = {"TABLE"}; + + Connection connection = DATA_SOURCE.getConnection(); + DatabaseMetaData metaData = connection.getMetaData(); + + ResultSet tablesResultSet = metaData.getTables(catalog, schemaPattern, tableNamePattern, types); + Assertions.assertTrue(tablesResultSet.next()); + + while (tablesResultSet.next()) { + String tableName = tablesResultSet.getString("TABLE_NAME"); + + ResultSet columnsResultSet = metaData.getColumns(catalog, schemaPattern, tableName, "%"); + + while (columnsResultSet.next()) { + String columnName = columnsResultSet.getString("COLUMN_NAME"); + String columnType = columnsResultSet.getString("TYPE_NAME"); + int columnSize = columnsResultSet.getInt("COLUMN_SIZE"); + int nullable = columnsResultSet.getInt("NULLABLE"); + String isNullable = nullable == 1 ? "YES" : "NO"; + LOG.info( + "Column Name: %s,Column Type: %s,Column Size: %d,Nullable: %b", + columnName, columnType, columnSize, isNullable); + } + } + } +} diff --git a/catalogs/catalog-jdbc-maxcompute/src/test/resources/resources/log4j2.properties b/catalogs/catalog-jdbc-maxcompute/src/test/resources/resources/log4j2.properties new file mode 100644 index 00000000000..467ccf52422 --- /dev/null +++ b/catalogs/catalog-jdbc-maxcompute/src/test/resources/resources/log4j2.properties @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Set to debug or trace if log4j initialization is failing +status = info + +# Name of the configuration +name = ConsoleLogConfig + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n + +# Log files location +property.logPath = ${sys:gravitino.log.path:-build/catalog-maxcompute-integration-test.log} + +# File appender configuration +appender.file.type = File +appender.file.name = fileLogger +appender.file.fileName = ${logPath} +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console and file appenders +rootLogger.appenderRef.stdout.ref = consoleLogger +rootLogger.appenderRef.file.ref = fileLogger + +# File appender configuration for testcontainers +appender.testcontainersFile.type = File +appender.testcontainersFile.name = testcontainersLogger +appender.testcontainersFile.fileName = build/testcontainers.log +appender.testcontainersFile.layout.type = PatternLayout +appender.testcontainersFile.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c - %m%n + +# Logger for testcontainers +logger.testcontainers.name = org.testcontainers +logger.testcontainers.level = debug +logger.testcontainers.additivity = false +logger.testcontainers.appenderRef.file.ref = testcontainersLogger + +logger.tc.name = tc +logger.tc.level = debug +logger.tc.additivity = false +logger.tc.appenderRef.file.ref = testcontainersLogger + +logger.docker.name = com.github.dockerjava +logger.docker.level = warn +logger.docker.additivity = false +logger.docker.appenderRef.file.ref = testcontainersLogger + +logger.http.name = com.github.dockerjava.zerodep.shaded.org.apache.hc.client5.http.wire +logger.http.level = off \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3241a48375e..6b22c7961ae 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -71,6 +71,7 @@ metrics = "4.2.25" prometheus = "0.16.0" mysql = "8.0.23" postgresql = "42.6.0" +odps = "3.3.6" immutables-value = "2.10.0" selenium = "3.141.59" rauschig = "1.2.0" @@ -228,6 +229,7 @@ prometheus-dropwizard = { group = "io.prometheus", name = "simpleclient_dropwiza prometheus-servlet = { group = "io.prometheus", name = "simpleclient_servlet", version.ref = "prometheus" } mysql-driver = { group = "mysql", name = "mysql-connector-java", version.ref = "mysql" } postgresql-driver = { group = "org.postgresql", name = "postgresql", version.ref = "postgresql" } +odps-jdbc = { group = "com.aliyun.odps", name = "odps-jdbc", version.ref = "odps" } minikdc = { group = "org.apache.hadoop", name = "hadoop-minikdc", version.ref = "hadoop-minikdc"} immutables-value = { module = "org.immutables:value", version.ref = "immutables-value" } commons-cli = { group = "commons-cli", name = "commons-cli", version.ref = "commons-cli" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1f3efb49544..684c5032583 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -37,7 +37,8 @@ include( "catalogs:catalog-jdbc-doris", "catalogs:catalog-jdbc-mysql", "catalogs:catalog-jdbc-postgresql", - "catalogs:catalog-jdbc-oceanbase" + "catalogs:catalog-jdbc-oceanbase", + "catalogs:catalog-jdbc-maxcompute" ) include("catalogs:catalog-hadoop") include("catalogs:catalog-kafka") diff --git a/web/integration-test/build.gradle.kts b/web/integration-test/build.gradle.kts index 90ac2ad509c..d15fcbb6b66 100644 --- a/web/integration-test/build.gradle.kts +++ b/web/integration-test/build.gradle.kts @@ -62,6 +62,7 @@ tasks.test { dependsOn(":catalogs:catalog-jdbc-doris:jar", ":catalogs:catalog-jdbc-doris:runtimeJars") dependsOn(":catalogs:catalog-jdbc-mysql:jar", ":catalogs:catalog-jdbc-mysql:runtimeJars") dependsOn(":catalogs:catalog-jdbc-postgresql:jar", ":catalogs:catalog-jdbc-postgresql:runtimeJars") + dependsOn(":catalogs:catalog-jdbc-maxcompute:jar", ":catalogs:catalog-jdbc-maxcompute:runtimeJars") dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars") dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") dependsOn(":catalogs:catalog-kafka:jar", ":catalogs:catalog-kafka:runtimeJars")