From 31303a0edb2ff08c5eaf6f0c072dc01f8ee76de0 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Thu, 16 Mar 2023 11:48:07 -0700 Subject: [PATCH 01/18] Upgrade Transport to use API from Trino v406 --- .../transportable-udfs-example-udfs/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle index dd3d185e..a49709a5 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation('com.google.guava:guava:24.1-jre') implementation('org.apache.commons:commons-io:1.3.2') testImplementation('io.airlift:aircompressor:0.21') + testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.2') } // As the tasks of trinoDistThinJar and trinoTrinJar are from Transport plugin which is built by Gradle 7.5.1, From 94472c57564f08e02a7f139103e76f556838c6b0 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Thu, 16 Mar 2023 11:49:49 -0700 Subject: [PATCH 02/18] Upgrade Transport to use API from Trino v406 --- defaultEnvironment.gradle | 2 +- transportable-udfs-plugin/build.gradle | 2 +- .../transport/test/spi/SqlStdTester.java | 4 +- .../build.gradle | 1 + .../transport/test/trino/TrinoTester.java | 49 ++++++++++--- .../transport/trino/StdUdfWrapper.java | 69 ++++++++++--------- .../transport/trino/TrinoFactory.java | 37 ++++++---- .../trino/TestGetTypeVariableConstraints.java | 4 +- 8 files changed, 106 insertions(+), 62 deletions(-) diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index 3480cf21..9e53ffe8 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -10,7 +10,7 @@ subprojects { url "https://conjars.org/repo" } } - project.ext.setProperty('trino-version', '352') + project.ext.setProperty('trino-version', '406') project.ext.setProperty('airlift-slice-version', '0.39') project.ext.setProperty('spark-group', 'org.apache.spark') project.ext.setProperty('spark2-version', '2.3.0') diff --git a/transportable-udfs-plugin/build.gradle b/transportable-udfs-plugin/build.gradle index 3511f57c..cd6370c6 100644 --- a/transportable-udfs-plugin/build.gradle +++ b/transportable-udfs-plugin/build.gradle @@ -19,7 +19,7 @@ def writeVersionInfo = { file -> ant.propertyfile(file: file) { entry(key: "transport-version", value: version) entry(key: "hive-version", value: '1.2.2') - entry(key: "trino-version", value: '352') + entry(key: "trino-version", value: '406') entry(key: "spark_2.11-version", value: '2.3.0') entry(key: "spark_2.12-version", value: '3.1.1') entry(key: "scala-version", value: '2.11.8') diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java index 630c28fa..f1635ffd 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java @@ -25,7 +25,9 @@ public interface SqlStdTester extends StdTester { * @param expectedOutputData The expected output data from the function call * @param expectedOutputType The expected output type from the function call */ - void assertFunctionCall(String functionCallString, Object expectedOutputData, Object expectedOutputType); + default void assertFunctionCall(String functionCallString, Object expectedOutputData, Object expectedOutputType) { + throw new UnsupportedOperationException(); + }; default void check(TestCase testCase) { assertFunctionCall(getSqlFunctionCallGenerator().getSqlFunctionCallString(testCase.getFunctionCall()), diff --git a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle index 2aa191af..bebc8b3c 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle @@ -20,4 +20,5 @@ dependencies { // The io.airlift.slice dependency below has to match its counterpart in trino-root's pom.xml file // If not specified, an older version is picked up transitively from another dependency implementation(group: 'io.airlift', name: 'slice', version: project.ext.'airlift-slice-version') + implementation(group: 'org.assertj', name: 'assertj-core', version: '3.24.2') } \ No newline at end of file diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java index 2abc0619..9baaf5c6 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java @@ -7,11 +7,16 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.trino.metadata.BoundSignature; +import com.linkedin.transport.test.spi.TestCase; +import com.linkedin.transport.test.spi.types.TestType; +import io.trino.FeaturesConfig; +import io.trino.Session; +import io.trino.SessionTestUtils; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.metadata.SqlFunction; +import io.trino.spi.function.BoundSignature; import io.trino.metadata.FunctionBinding; -import io.trino.metadata.FunctionId; -import io.trino.operator.scalar.AbstractTestFunctions; -import io.trino.spi.type.Type; +import io.trino.spi.function.FunctionId; import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.udf.StdUDF; import com.linkedin.transport.api.udf.TopLevelStdUDF; @@ -19,32 +24,45 @@ import com.linkedin.transport.test.spi.SqlFunctionCallGenerator; import com.linkedin.transport.test.spi.SqlStdTester; import com.linkedin.transport.test.spi.ToPlatformTestOutputConverter; +import io.trino.spi.type.Type; +import io.trino.sql.query.QueryAssertions; +import io.trino.testing.LocalQueryRunner; +import io.trino.type.InternalTypeManager; +import java.util.ArrayList; import java.util.List; import java.util.Map; import static io.trino.type.UnknownType.UNKNOWN; +import static org.assertj.core.api.Assertions.*; -public class TrinoTester extends AbstractTestFunctions implements SqlStdTester { +public class TrinoTester implements SqlStdTester { private StdFactory _stdFactory; private SqlFunctionCallGenerator _sqlFunctionCallGenerator; private ToPlatformTestOutputConverter _toPlatformTestOutputConverter; + private Session _session; + private FeaturesConfig _featuresConfig; + private LocalQueryRunner _runner; + private QueryAssertions _queryAssertions; public TrinoTester() { _stdFactory = null; _sqlFunctionCallGenerator = new TrinoSqlFunctionCallGenerator(); _toPlatformTestOutputConverter = new ToTrinoTestOutputConverter(); + _session = SessionTestUtils.TEST_SESSION; + _featuresConfig = new FeaturesConfig(); + _runner = LocalQueryRunner.builder(_session).withFeaturesConfig(_featuresConfig).build(); + _queryAssertions = new QueryAssertions(_runner); } @Override public void setup( Map, List>> topLevelStdUDFClassesAndImplementations) { // Refresh Trino state during every setup call - initTestFunctions(); for (List> stdUDFImplementations : topLevelStdUDFClassesAndImplementations.values()) { for (Class stdUDF : stdUDFImplementations) { - registerScalarFunction(new TrinoTestStdUDFWrapper(stdUDF)); + _runner.addFunctions(new InternalFunctionBundle(new SqlFunction[]{new TrinoTestStdUDFWrapper(stdUDF)})); } } } @@ -59,7 +77,9 @@ public StdFactory getStdFactory() { ImmutableMap.of()); _stdFactory = new TrinoFactory( functionBinding, - this.functionAssertions.getMetadata()); + _runner.getMetadata(), + _runner.getFunctionManager(), + _session, InternalTypeManager.TESTING_TYPE_MANAGER); } return _stdFactory; } @@ -75,7 +95,16 @@ public ToPlatformTestOutputConverter getToPlatformTestOutputConverter() { } @Override - public void assertFunctionCall(String functionCallString, Object expectedOutputData, Object expectedOutputType) { - assertFunction(functionCallString, (Type) expectedOutputType, expectedOutputData); + public void check(TestCase testCase) { + String functionName = testCase.getFunctionCall().getFunctionName(); + List parameters = testCase.getFunctionCall().getParameters(); + List testTypes = testCase.getFunctionCall().getInferredParameterTypes(); + List functionArguments = new ArrayList<>(); + for (int i = 0; i < parameters.size(); ++i) { + functionArguments.add(_sqlFunctionCallGenerator.getFunctionCallArgumentString(parameters.get(i), testTypes.get(i))); + } + Object expectedOutputType = getPlatformType(testCase.getExpectedOutputType()); + QueryAssertions.ExpressionAssertProvider expressionAssertProvider = _queryAssertions.function(functionName, functionArguments); + assertThat(expressionAssertProvider).hasType((Type) expectedOutputType).isEqualTo(testCase.getExpectedOutput()); } } diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java index 2f5a24a9..5075d66c 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java @@ -8,7 +8,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.primitives.Booleans; import com.linkedin.transport.api.StdFactory; import com.linkedin.transport.api.data.PlatformData; import com.linkedin.transport.api.data.StdData; @@ -24,17 +23,18 @@ import com.linkedin.transport.api.udf.StdUDF8; import com.linkedin.transport.api.udf.TopLevelStdUDF; import com.linkedin.transport.typesystem.GenericTypeSignatureElement; -import io.trino.metadata.FunctionArgumentDefinition; import io.trino.metadata.FunctionBinding; -import io.trino.metadata.FunctionDependencies; -import io.trino.metadata.FunctionDependencyDeclaration; -import io.trino.metadata.FunctionKind; -import io.trino.metadata.FunctionMetadata; -import io.trino.metadata.Signature; +import io.trino.metadata.SignatureBinder; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionDependencies; +import io.trino.spi.function.FunctionDependencyDeclaration; +import io.trino.spi.function.FunctionKind; +import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.Signature; import io.trino.metadata.SqlScalarFunction; -import io.trino.metadata.TypeVariableConstraint; -import io.trino.operator.scalar.ChoicesScalarFunctionImplementation; -import io.trino.operator.scalar.ScalarFunctionImplementation; +import io.trino.spi.function.TypeVariableConstraint; +import io.trino.operator.scalar.ChoicesSpecializedSqlScalarFunction; +import io.trino.operator.scalar.SpecializedSqlScalarFunction; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.function.InvocationConvention; import io.trino.spi.type.ArrayType; @@ -56,10 +56,10 @@ import org.apache.commons.lang3.ClassUtils; import static com.linkedin.transport.trino.StdUDFUtils.quoteReservedKeywords; -import static io.trino.metadata.Signature.*; import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.*; import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.NULLABLE_RETURN; import static io.trino.spi.function.OperatorType.*; +import static io.trino.spi.function.TypeVariableConstraint.*; import static io.trino.sql.analyzer.TypeSignatureTranslator.parseTypeSignature; import static io.trino.util.Reflection.*; @@ -71,19 +71,18 @@ public abstract class StdUdfWrapper extends SqlScalarFunction { private static final int JITTER_FACTOR = 50; // to calculate jitter from delay protected StdUdfWrapper(StdUDF stdUDF) { - super(new FunctionMetadata( - new Signature(((TopLevelStdUDF) stdUDF).getFunctionName(), getTypeVariableConstraintsForStdUdf(stdUDF), - ImmutableList.of(), - parseTypeSignature(quoteReservedKeywords(stdUDF.getOutputParameterSignature()), - ImmutableSet.of()), stdUDF.getInputParameterSignatures() - .stream() - .map(typeSignature -> parseTypeSignature(quoteReservedKeywords(typeSignature), - ImmutableSet.of())) - .collect(Collectors.toList()), false), true, Booleans.asList(stdUDF.getNullableArguments()) - .stream() - .map(FunctionArgumentDefinition::new) - .collect(Collectors.toList()), false, false, ((TopLevelStdUDF) stdUDF).getFunctionDescription(), - FunctionKind.SCALAR)); + super(FunctionMetadata.builder(FunctionKind.SCALAR) + .nullable() + .nondeterministic() + .description(((TopLevelStdUDF) stdUDF).getFunctionDescription()) + .signature(Signature.builder() + .name(((TopLevelStdUDF) stdUDF).getFunctionName()) + .typeVariableConstraints(getTypeVariableConstraintsForStdUdf(stdUDF)) + .returnType(parseTypeSignature(quoteReservedKeywords(stdUDF.getOutputParameterSignature()), ImmutableSet.of())) + .argumentTypes(stdUDF.getInputParameterSignatures().stream() + .map(typeSignature -> parseTypeSignature(quoteReservedKeywords(typeSignature), ImmutableSet.of())).collect(Collectors.toList())) + .build()) + .build()); } @VisibleForTesting @@ -118,18 +117,20 @@ private void registerNestedDependencies(Type nestedType, FunctionDependencyDecla } @Override - public FunctionDependencyDeclaration getFunctionDependencies(FunctionBinding functionBinding) { + public FunctionDependencyDeclaration getFunctionDependencies(BoundSignature boundSignature) { FunctionDependencyDeclaration.FunctionDependencyDeclarationBuilder builder = FunctionDependencyDeclaration.builder(); - registerNestedDependencies(functionBinding.getBoundSignature().getReturnType(), builder); - List argumentTypes = functionBinding.getBoundSignature().getArgumentTypes(); + registerNestedDependencies(boundSignature.getReturnType(), builder); + List argumentTypes = boundSignature.getArgumentTypes(); argumentTypes.forEach(type -> registerNestedDependencies(type, builder)); return builder.build(); } @Override - public ScalarFunctionImplementation specialize(FunctionBinding functionBinding, FunctionDependencies functionDependencies) { + public SpecializedSqlScalarFunction specialize(BoundSignature boundSignature, FunctionDependencies functionDependencies) { + FunctionMetadata metadata = getFunctionMetadata(); + FunctionBinding functionBinding = SignatureBinder.bindFunction(metadata.getFunctionId(), metadata.getSignature(), boundSignature); StdFactory stdFactory = new TrinoFactory(functionBinding, functionDependencies); StdUDF stdUDF = getStdUDF(); stdUDF.init(stdFactory); @@ -141,17 +142,17 @@ public ScalarFunctionImplementation specialize(FunctionBinding functionBinding, - (new Random()).nextInt(initialJitterInt)); boolean[] nullableArguments = stdUDF.getAndCheckNullableArguments(); - return new ChoicesScalarFunctionImplementation( - functionBinding, + return new ChoicesSpecializedSqlScalarFunction( + boundSignature, NULLABLE_RETURN, getNullConventionForArguments(nullableArguments), - getMethodHandle(stdUDF, functionBinding, nullableArguments, requiredFilesNextRefreshTime)); + getMethodHandle(stdUDF, boundSignature, nullableArguments, requiredFilesNextRefreshTime)); } - private MethodHandle getMethodHandle(StdUDF stdUDF, FunctionBinding functionBinding, boolean[] nullableArguments, + private MethodHandle getMethodHandle(StdUDF stdUDF, BoundSignature boundSignature, boolean[] nullableArguments, AtomicLong requiredFilesNextRefreshTime) { - Type[] inputTypes = functionBinding.getBoundSignature().getArgumentTypes().toArray(new Type[0]); - Type outputType = functionBinding.getBoundSignature().getReturnType(); + Type[] inputTypes = boundSignature.getArgumentTypes().toArray(new Type[0]); + Type outputType = boundSignature.getReturnType(); // Generic MethodHandle for eval where all arguments are of type Object Class[] genericMethodHandleArgumentTypes = getMethodHandleArgumentTypes(inputTypes, nullableArguments, true); diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java index c26914e4..8f294d91 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java @@ -30,8 +30,10 @@ import com.linkedin.transport.trino.data.TrinoString; import com.linkedin.transport.trino.data.TrinoStruct; import io.airlift.slice.Slices; +import io.trino.Session; import io.trino.metadata.FunctionBinding; -import io.trino.metadata.FunctionDependencies; +import io.trino.metadata.FunctionManager; +import io.trino.spi.function.FunctionDependencies; import io.trino.metadata.Metadata; import io.trino.metadata.OperatorNotFoundException; import io.trino.spi.function.InvocationConvention; @@ -40,6 +42,8 @@ import io.trino.spi.type.MapType; import io.trino.spi.type.RowType; import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; +import io.trino.spi.type.TypeSignature; import java.lang.invoke.MethodHandle; import java.nio.ByteBuffer; import java.util.List; @@ -55,17 +59,27 @@ public class TrinoFactory implements StdFactory { final FunctionBinding functionBinding; final FunctionDependencies functionDependencies; final Metadata metadata; + final TypeManager typeManager; + final FunctionManager functionManager; + + final Session session; public TrinoFactory(FunctionBinding functionBinding, FunctionDependencies functionDependencies) { this.functionBinding = functionBinding; this.functionDependencies = functionDependencies; this.metadata = null; + this.typeManager = null; + this.functionManager = null; + this.session = null; } - public TrinoFactory(FunctionBinding functionBinding, Metadata metadata) { + public TrinoFactory(FunctionBinding functionBinding, Metadata metadata, FunctionManager functionManager, Session session, TypeManager typeManager) { this.functionBinding = functionBinding; this.functionDependencies = null; this.metadata = metadata; + this.typeManager = typeManager; + this.functionManager = functionManager; + this.session = session; } @Override @@ -137,25 +151,22 @@ public StdStruct createStruct(StdType stdType) { } @Override - public StdType createStdType(String typeSignature) { - if (metadata != null) { - return TrinoWrapper.createStdType(metadata.getType(applyBoundVariables( - parseTypeSignature(quoteReservedKeywords(typeSignature), ImmutableSet.of()), - functionBinding))); + public StdType createStdType(String typeSignatureStr) { + TypeSignature typeSignature = applyBoundVariables(parseTypeSignature(quoteReservedKeywords(typeSignatureStr), ImmutableSet.of()), functionBinding); + if (typeManager != null) { + return TrinoWrapper.createStdType(typeManager.getType(typeSignature)); } - return TrinoWrapper.createStdType(functionDependencies.getType( - applyBoundVariables(parseTypeSignature(quoteReservedKeywords(typeSignature), ImmutableSet.of()), - functionBinding))); + return TrinoWrapper.createStdType(functionDependencies.getType(typeSignature)); } public MethodHandle getOperatorHandle( OperatorType operatorType, List argumentTypes, InvocationConvention invocationConvention) throws OperatorNotFoundException { - if (metadata != null) { - return metadata.getScalarFunctionInvoker(metadata.resolveOperator(operatorType, argumentTypes), + if (functionManager != null) { + return functionManager.getScalarFunctionImplementation(metadata.resolveOperator(session, operatorType, argumentTypes), invocationConvention).getMethodHandle(); } - return functionDependencies.getOperatorInvoker(operatorType, argumentTypes, invocationConvention).getMethodHandle(); + return functionDependencies.getOperatorImplementation(operatorType, argumentTypes, invocationConvention).getMethodHandle(); } } diff --git a/transportable-udfs-trino/src/test/java/com/linkedin/transport/trino/TestGetTypeVariableConstraints.java b/transportable-udfs-trino/src/test/java/com/linkedin/transport/trino/TestGetTypeVariableConstraints.java index 6f2b49ef..33475f96 100644 --- a/transportable-udfs-trino/src/test/java/com/linkedin/transport/trino/TestGetTypeVariableConstraints.java +++ b/transportable-udfs-trino/src/test/java/com/linkedin/transport/trino/TestGetTypeVariableConstraints.java @@ -7,12 +7,12 @@ import com.google.common.collect.ImmutableList; import com.linkedin.transport.api.udf.StdUDF; -import io.trino.metadata.TypeVariableConstraint; +import io.trino.spi.function.TypeVariableConstraint; import java.util.List; import org.testng.Assert; import org.testng.annotations.Test; -import static io.trino.metadata.Signature.*; +import static io.trino.spi.function.TypeVariableConstraint.*; public class TestGetTypeVariableConstraints { From 4dd7decebf960e1b238b68561a4fca6586b9fc27 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Mon, 3 Apr 2023 16:52:17 -0700 Subject: [PATCH 03/18] PoC of Trino Connector --- settings.gradle | 1 + .../build.gradle | 1 + .../transport/test/trino/TrinoTester.java | 39 ++++++--- transportable-udfs-trino-plugin/build.gradle | 33 ++++++++ .../transport/trino/TransportConnector.java | 50 ++++++++++++ .../trino/TransportConnectorFactory.java | 79 +++++++++++++++++++ .../trino/TransportConnectorMetadata.java | 42 ++++++++++ .../trino/TransportFunctionProvider.java | 44 +++++++++++ .../transport/trino/TransportModule.java | 16 ++++ .../transport/trino/TransportPlugin.java | 18 +++++ .../trino/TransportTransactionHandle.java | 13 +++ .../trino/TransportUDFClassLoader.java | 45 +++++++++++ .../META-INF/services/io.trino.spi.Plugin | 1 + .../trino/plugin/TestTransportUdf.java | 24 ++++++ .../transport/trino/StdUdfWrapper.java | 27 ++++--- .../transport/trino/TrinoFactory.java | 25 +++--- 16 files changed, 420 insertions(+), 38 deletions(-) create mode 100644 transportable-udfs-trino-plugin/build.gradle create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java create mode 100644 transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin create mode 100644 transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java diff --git a/settings.gradle b/settings.gradle index a775c86e..9341981b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,6 +15,7 @@ def modules = [ 'transportable-udfs-spark_2.11', 'transportable-udfs-spark_2.12', 'transportable-udfs-trino', + 'transportable-udfs-trino-plugin', 'transportable-udfs-test:transportable-udfs-test-api', 'transportable-udfs-test:transportable-udfs-test-generic', 'transportable-udfs-test:transportable-udfs-test-hive', diff --git a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle index bebc8b3c..9711cde0 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle @@ -9,6 +9,7 @@ dependencies { implementation project(":transportable-udfs-test:transportable-udfs-test-api") implementation project(":transportable-udfs-test:transportable-udfs-test-spi") implementation project(":transportable-udfs-trino") + implementation project(":transportable-udfs-trino-plugin") implementation('com.google.guava:guava:24.1-jre') implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') { exclude 'group': 'com.google.collections', 'module': 'google-collections' diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java index 9baaf5c6..00de48d9 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java @@ -7,13 +7,20 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.linkedin.transport.test.spi.TestCase; import com.linkedin.transport.test.spi.types.TestType; +import com.linkedin.transport.trino.StdUdfWrapper; +import com.linkedin.transport.trino.TransportConnector; +import com.linkedin.transport.trino.TransportConnectorFactory; +import com.linkedin.transport.trino.TransportConnectorMetadata; +import com.linkedin.transport.trino.TransportFunctionProvider; import io.trino.FeaturesConfig; import io.trino.Session; -import io.trino.SessionTestUtils; -import io.trino.metadata.InternalFunctionBundle; -import io.trino.metadata.SqlFunction; +import io.trino.client.ClientCapabilities; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.function.BoundSignature; import io.trino.metadata.FunctionBinding; import io.trino.spi.function.FunctionId; @@ -24,13 +31,20 @@ import com.linkedin.transport.test.spi.SqlFunctionCallGenerator; import com.linkedin.transport.test.spi.SqlStdTester; import com.linkedin.transport.test.spi.ToPlatformTestOutputConverter; +import io.trino.spi.function.FunctionProvider; import io.trino.spi.type.Type; +import io.trino.sql.SqlPath; import io.trino.sql.query.QueryAssertions; import io.trino.testing.LocalQueryRunner; +import io.trino.testing.TestingSession; import io.trino.type.InternalTypeManager; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static io.trino.type.UnknownType.UNKNOWN; import static org.assertj.core.api.Assertions.*; @@ -50,7 +64,9 @@ public TrinoTester() { _stdFactory = null; _sqlFunctionCallGenerator = new TrinoSqlFunctionCallGenerator(); _toPlatformTestOutputConverter = new ToTrinoTestOutputConverter(); - _session = SessionTestUtils.TEST_SESSION; + SqlPath sqlPath = new SqlPath("LINKEDIN.TRANSPORT"); + _session = TestingSession.testSessionBuilder().setCatalog("LINKEDIN").setSchema("TRANSPORT").setPath(sqlPath).setClientCapabilities((Set) Arrays.stream( + ClientCapabilities.values()).map(Enum::toString).collect(ImmutableSet.toImmutableSet())).build(); _featuresConfig = new FeaturesConfig(); _runner = LocalQueryRunner.builder(_session).withFeaturesConfig(_featuresConfig).build(); _queryAssertions = new QueryAssertions(_runner); @@ -59,12 +75,19 @@ public TrinoTester() { @Override public void setup( Map, List>> topLevelStdUDFClassesAndImplementations) { + Map functions = new HashMap<>(); // Refresh Trino state during every setup call for (List> stdUDFImplementations : topLevelStdUDFClassesAndImplementations.values()) { for (Class stdUDF : stdUDFImplementations) { - _runner.addFunctions(new InternalFunctionBundle(new SqlFunction[]{new TrinoTestStdUDFWrapper(stdUDF)})); + StdUdfWrapper function = new TrinoTestStdUDFWrapper(stdUDF); + functions.put(function.getFunctionMetadata().getFunctionId(), function); } } + FunctionProvider functionProvider = new TransportFunctionProvider(functions); + ConnectorMetadata connectorMetadata = new TransportConnectorMetadata(functions); + Connector connector = new TransportConnector(connectorMetadata, functionProvider); + ConnectorFactory connectorFactory = new TransportConnectorFactory(connector); + _runner.createCatalog("LINKEDIN", connectorFactory, Collections.emptyMap()); } @Override @@ -75,11 +98,7 @@ public StdFactory getStdFactory() { new BoundSignature("test", UNKNOWN, ImmutableList.of()), ImmutableMap.of(), ImmutableMap.of()); - _stdFactory = new TrinoFactory( - functionBinding, - _runner.getMetadata(), - _runner.getFunctionManager(), - _session, InternalTypeManager.TESTING_TYPE_MANAGER); + _stdFactory = new TrinoFactory(functionBinding, _runner, InternalTypeManager.TESTING_TYPE_MANAGER); } return _stdFactory; } diff --git a/transportable-udfs-trino-plugin/build.gradle b/transportable-udfs-trino-plugin/build.gradle new file mode 100644 index 00000000..92f3741f --- /dev/null +++ b/transportable-udfs-trino-plugin/build.gradle @@ -0,0 +1,33 @@ +apply plugin: 'java' +apply plugin: 'distribution' + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(17)) +} + +dependencies { + implementation project(':transportable-udfs-api') + implementation project(":transportable-udfs-trino") + implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') { + exclude 'group': 'com.google.collections', 'module': 'google-collections' + } + implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version', classifier: 'tests') { + exclude 'group': 'com.google.collections', 'module': 'google-collections' + } + compileOnly(group:'io.trino', name: 'trino-spi', version: project.ext.'trino-version') +} + +distributions { + main { + contents { + from jar + from project.configurations.runtimeClasspath + } + } +} + +artifacts { + archives jar, distTar +} + +build.dependsOn distTar \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java new file mode 100644 index 00000000..210af4d2 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -0,0 +1,50 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionProvider; +import io.trino.spi.transaction.IsolationLevel; +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.*; + + +public class TransportConnector implements Connector { + private final ConnectorMetadata connectorMetadata; + private final FunctionProvider functionProvider; + + public TransportConnector(ConnectorMetadata connectorMetadata, FunctionProvider functionProvider) { + this.connectorMetadata = requireNonNull(connectorMetadata, "connector metadata is null"); + this.functionProvider = requireNonNull(functionProvider, "function provider is null"); + } + + public TransportConnector(Map functions) { + this.connectorMetadata = new TransportConnectorMetadata(functions); + this.functionProvider = new TransportFunctionProvider(functions); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transactionHandle) { + return this.connectorMetadata; + } + + @Override + public Optional getFunctionProvider() { + return Optional.of(this.functionProvider); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, + boolean autoCommit) { + return TransportTransactionHandle.INSTANCE; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java new file mode 100644 index 00000000..7a07e1a6 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -0,0 +1,79 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import io.airlift.log.Logger; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.function.FunctionId; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Objects.*; + + +public class TransportConnectorFactory implements ConnectorFactory { + + private static final String UDF_JAR_PATH = + "file:///Users/yiqding/workspace/trino/core/trino-server-main/plugin/transportable-udfs-trino-plugin-0.0.93/udf/"; + private static final Logger log = Logger.get(TransportConnectorFactory.class); + private Connector connector; + private final Class module; + + + public TransportConnectorFactory(Connector connector) { + this.connector = connector; + this.module = TransportModule.class; + } + + public TransportConnectorFactory(Class module) { + connector = null; + this.module = module; + } + + @Override + public String getName() { + return "TRANSPORT"; + } + + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) { + requireNonNull(config, "config is null"); + if (this.connector == null) { + ClassLoader classLoaderForUdfWrapper = StdUdfWrapper.class.getClassLoader(); + log.info(classLoaderForUdfWrapper.toString()); + ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); + log.info(classLoaderForFactory.toString()); + try { + List urlList = ImmutableList.of(new URL(UDF_JAR_PATH + "transportable-udfs-example-udfs.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-example-udfs-trino-dist-thin.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-api-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-trino-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-type-system-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-utils-0.0.93.jar")); + + TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, urlList); + Class udfClass = classLoaderForUdf.loadClass("com.linkedin.transport.examples.trino.ArrayElementAtFunction", false); + Object udfObj = udfClass.getConstructor().newInstance(); + log.info(udfObj.toString()); + Map functions = new HashMap<>(); + log.info("adding wrapper"); + StdUdfWrapper wrapper = (StdUdfWrapper) udfObj; + functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); + this.connector = new TransportConnector(functions); + } catch (Exception ex) { + log.error(ex); + throw new RuntimeException(ex.getCause()); + } + } + return this.connector; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java new file mode 100644 index 00000000..a8744a75 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java @@ -0,0 +1,42 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionDependencyDeclaration; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.SchemaFunctionName; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + + +public class TransportConnectorMetadata implements ConnectorMetadata { + private final Map functions; + + public TransportConnectorMetadata(Map functions) { + this.functions = functions; + } + + @Override + public FunctionDependencyDeclaration getFunctionDependencies(ConnectorSession session, FunctionId functionId, + BoundSignature boundSignature) { + return functions.get(functionId).getFunctionDependencies(boundSignature); + } + + @Override + public Collection getFunctions(ConnectorSession session, SchemaFunctionName name) { + return functions.values().stream().map(StdUdfWrapper::getFunctionMetadata).collect(Collectors.toList()); + } + + @Override + public FunctionMetadata getFunctionMetadata(ConnectorSession session, FunctionId functionId) { + return functions.get(functionId).getFunctionMetadata(); + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java new file mode 100644 index 00000000..d6678dda --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java @@ -0,0 +1,44 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.function.AggregationImplementation; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionDependencies; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionProvider; +import io.trino.spi.function.InvocationConvention; +import io.trino.spi.function.ScalarFunctionImplementation; +import io.trino.spi.function.WindowFunctionSupplier; +import java.util.Map; + + +public class TransportFunctionProvider implements FunctionProvider { + private final Map functions; + + public TransportFunctionProvider(Map functions) { + this.functions = functions; + } + + @Override + public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionId functionId, + BoundSignature boundSignature, FunctionDependencies functionDependencies, + InvocationConvention invocationConvention) { + return functions.get(functionId).getScalarFunctionImplementation(boundSignature, functionDependencies, invocationConvention); + } + + @Override + public AggregationImplementation getAggregationImplementation(FunctionId functionId, BoundSignature boundSignature, + FunctionDependencies functionDependencies) { + return null; + } + + @Override + public WindowFunctionSupplier getWindowFunctionSupplier(FunctionId functionId, BoundSignature boundSignature, + FunctionDependencies functionDependencies) { + return null; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java new file mode 100644 index 00000000..083b4961 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java @@ -0,0 +1,16 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.inject.Binder; +import com.google.inject.Module; + + +public class TransportModule implements Module { + @Override + public void configure(Binder binder) { + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java new file mode 100644 index 00000000..45dafe0e --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java @@ -0,0 +1,18 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.common.collect.ImmutableList; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + + +public class TransportPlugin implements Plugin { + @Override + public Iterable getConnectorFactories() { + return ImmutableList.of(new TransportConnectorFactory(TransportModule.class)); + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java new file mode 100644 index 00000000..e2535202 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java @@ -0,0 +1,13 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.ConnectorTransactionHandle; + + +public enum TransportTransactionHandle implements ConnectorTransactionHandle { + INSTANCE +} \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java new file mode 100644 index 00000000..c4f34bdd --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java @@ -0,0 +1,45 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; + + +public class TransportUDFClassLoader extends URLClassLoader { + private final ClassLoader parent; + + public TransportUDFClassLoader(ClassLoader parent, List urls) { + super(urls.toArray(new URL[0])); + this.parent = parent; + } + + @Override + public Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + // Check if class is in the loaded classes cache + Class cachedClass = findLoadedClass(name); + if (cachedClass != null) { + return resolveClass(cachedClass, resolve); + } + + if (name.endsWith("StdUdfWrapper") || name.endsWith("TopLevelStdUDF") || name.endsWith("StdUDF")) { + return resolveClass(parent.loadClass(name), resolve); + } + + // Look for class locally + return super.loadClass(name, resolve); + } + } + + private Class resolveClass(Class clazz, boolean resolve) { + if (resolve) { + resolveClass(clazz); + } + return clazz; + } +} \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin b/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin new file mode 100644 index 00000000..b5fd55d8 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin @@ -0,0 +1 @@ +com.linkedin.transport.trino.TransportPlugin \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java new file mode 100644 index 00000000..88df2150 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java @@ -0,0 +1,24 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino.plugin; + +import com.linkedin.transport.trino.TransportPlugin; +import io.trino.server.testing.TestingTrinoServer; +import java.sql.Statement; +import org.testng.annotations.BeforeClass; + + +public class TestTransportUdf { + private TestingTrinoServer server; + private Statement statement; + + @BeforeClass + public void setupServer() throws Exception { + server = TestingTrinoServer.create(); + server.installPlugin(new TransportPlugin()); + server.createCatalog("LINKEDIN", "TRANSPORT"); + } +} diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java index 5075d66c..7228c89c 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java @@ -30,11 +30,10 @@ import io.trino.spi.function.FunctionDependencyDeclaration; import io.trino.spi.function.FunctionKind; import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.ScalarFunctionImplementation; import io.trino.spi.function.Signature; -import io.trino.metadata.SqlScalarFunction; import io.trino.spi.function.TypeVariableConstraint; import io.trino.operator.scalar.ChoicesSpecializedSqlScalarFunction; -import io.trino.operator.scalar.SpecializedSqlScalarFunction; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.function.InvocationConvention; import io.trino.spi.type.ArrayType; @@ -65,13 +64,15 @@ // Suppressing argument naming convention for the evalInternal methods @SuppressWarnings({"checkstyle:regexpsinglelinejava"}) -public abstract class StdUdfWrapper extends SqlScalarFunction { +public abstract class StdUdfWrapper { private static final int DEFAULT_REFRESH_INTERVAL_DAYS = 1; private static final int JITTER_FACTOR = 50; // to calculate jitter from delay - protected StdUdfWrapper(StdUDF stdUDF) { - super(FunctionMetadata.builder(FunctionKind.SCALAR) + private final FunctionMetadata functionMetadata; + + public StdUdfWrapper(StdUDF stdUDF) { + this.functionMetadata = FunctionMetadata.builder(FunctionKind.SCALAR) .nullable() .nondeterministic() .description(((TopLevelStdUDF) stdUDF).getFunctionDescription()) @@ -82,7 +83,11 @@ protected StdUdfWrapper(StdUDF stdUDF) { .argumentTypes(stdUDF.getInputParameterSignatures().stream() .map(typeSignature -> parseTypeSignature(quoteReservedKeywords(typeSignature), ImmutableSet.of())).collect(Collectors.toList())) .build()) - .build()); + .build(); + } + + public FunctionMetadata getFunctionMetadata() { + return this.functionMetadata; } @VisibleForTesting @@ -116,7 +121,6 @@ private void registerNestedDependencies(Type nestedType, FunctionDependencyDecla } } - @Override public FunctionDependencyDeclaration getFunctionDependencies(BoundSignature boundSignature) { FunctionDependencyDeclaration.FunctionDependencyDeclarationBuilder builder = FunctionDependencyDeclaration.builder(); @@ -127,10 +131,9 @@ public FunctionDependencyDeclaration getFunctionDependencies(BoundSignature boun return builder.build(); } - @Override - public SpecializedSqlScalarFunction specialize(BoundSignature boundSignature, FunctionDependencies functionDependencies) { - FunctionMetadata metadata = getFunctionMetadata(); - FunctionBinding functionBinding = SignatureBinder.bindFunction(metadata.getFunctionId(), metadata.getSignature(), boundSignature); + public ScalarFunctionImplementation getScalarFunctionImplementation(BoundSignature boundSignature, + FunctionDependencies functionDependencies, InvocationConvention invocationConvention) { + FunctionBinding functionBinding = SignatureBinder.bindFunction(functionMetadata.getFunctionId(), functionMetadata.getSignature(), boundSignature); StdFactory stdFactory = new TrinoFactory(functionBinding, functionDependencies); StdUDF stdUDF = getStdUDF(); stdUDF.init(stdFactory); @@ -146,7 +149,7 @@ public SpecializedSqlScalarFunction specialize(BoundSignature boundSignature, Fu boundSignature, NULLABLE_RETURN, getNullConventionForArguments(nullableArguments), - getMethodHandle(stdUDF, boundSignature, nullableArguments, requiredFilesNextRefreshTime)); + getMethodHandle(stdUDF, boundSignature, nullableArguments, requiredFilesNextRefreshTime)).getScalarFunctionImplementation(invocationConvention); } private MethodHandle getMethodHandle(StdUDF stdUDF, BoundSignature boundSignature, boolean[] nullableArguments, diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java index 8f294d91..b06ae75e 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java @@ -30,11 +30,8 @@ import com.linkedin.transport.trino.data.TrinoString; import com.linkedin.transport.trino.data.TrinoStruct; import io.airlift.slice.Slices; -import io.trino.Session; import io.trino.metadata.FunctionBinding; -import io.trino.metadata.FunctionManager; import io.trino.spi.function.FunctionDependencies; -import io.trino.metadata.Metadata; import io.trino.metadata.OperatorNotFoundException; import io.trino.spi.function.InvocationConvention; import io.trino.spi.function.OperatorType; @@ -44,6 +41,7 @@ import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; +import io.trino.testing.LocalQueryRunner; import java.lang.invoke.MethodHandle; import java.nio.ByteBuffer; import java.util.List; @@ -58,28 +56,22 @@ public class TrinoFactory implements StdFactory { final FunctionBinding functionBinding; final FunctionDependencies functionDependencies; - final Metadata metadata; final TypeManager typeManager; - final FunctionManager functionManager; - - final Session session; + final LocalQueryRunner queryRunner; public TrinoFactory(FunctionBinding functionBinding, FunctionDependencies functionDependencies) { this.functionBinding = functionBinding; this.functionDependencies = functionDependencies; - this.metadata = null; this.typeManager = null; - this.functionManager = null; - this.session = null; + this.queryRunner = null; } - public TrinoFactory(FunctionBinding functionBinding, Metadata metadata, FunctionManager functionManager, Session session, TypeManager typeManager) { + // for test only + public TrinoFactory(FunctionBinding functionBinding, LocalQueryRunner queryRunner, TypeManager typeManager) { this.functionBinding = functionBinding; this.functionDependencies = null; - this.metadata = metadata; + this.queryRunner = queryRunner; this.typeManager = typeManager; - this.functionManager = functionManager; - this.session = session; } @Override @@ -163,8 +155,9 @@ public MethodHandle getOperatorHandle( OperatorType operatorType, List argumentTypes, InvocationConvention invocationConvention) throws OperatorNotFoundException { - if (functionManager != null) { - return functionManager.getScalarFunctionImplementation(metadata.resolveOperator(session, operatorType, argumentTypes), + if (queryRunner != null && queryRunner.getFunctionManager() != null) { + return queryRunner.getFunctionManager() + .getScalarFunctionImplementation(queryRunner.getMetadata().resolveOperator(queryRunner.getDefaultSession(), operatorType, argumentTypes), invocationConvention).getMethodHandle(); } return functionDependencies.getOperatorImplementation(operatorType, argumentTypes, invocationConvention).getMethodHandle(); From af417d8cb4d8152e6dac2cf0d53397cb529de2da Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Mon, 3 Apr 2023 16:52:17 -0700 Subject: [PATCH 04/18] PoC of Trino Connector --- settings.gradle | 1 + .../build.gradle | 1 + .../transport/test/trino/TrinoTester.java | 39 ++++++-- transportable-udfs-trino-plugin/build.gradle | 33 +++++++ .../transport/trino/TransportConnector.java | 50 ++++++++++ .../trino/TransportConnectorFactory.java | 97 +++++++++++++++++++ .../trino/TransportConnectorMetadata.java | 42 ++++++++ .../trino/TransportFunctionProvider.java | 44 +++++++++ .../transport/trino/TransportModule.java | 16 +++ .../transport/trino/TransportPlugin.java | 18 ++++ .../trino/TransportTransactionHandle.java | 13 +++ .../trino/TransportUDFClassLoader.java | 45 +++++++++ .../META-INF/services/io.trino.spi.Plugin | 1 + .../trino/plugin/TestTransportUdf.java | 24 +++++ .../transport/trino/StdUdfWrapper.java | 27 +++--- .../transport/trino/TrinoFactory.java | 25 ++--- 16 files changed, 438 insertions(+), 38 deletions(-) create mode 100644 transportable-udfs-trino-plugin/build.gradle create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java create mode 100644 transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin create mode 100644 transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java diff --git a/settings.gradle b/settings.gradle index a775c86e..9341981b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,6 +15,7 @@ def modules = [ 'transportable-udfs-spark_2.11', 'transportable-udfs-spark_2.12', 'transportable-udfs-trino', + 'transportable-udfs-trino-plugin', 'transportable-udfs-test:transportable-udfs-test-api', 'transportable-udfs-test:transportable-udfs-test-generic', 'transportable-udfs-test:transportable-udfs-test-hive', diff --git a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle index bebc8b3c..9711cde0 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-trino/build.gradle @@ -9,6 +9,7 @@ dependencies { implementation project(":transportable-udfs-test:transportable-udfs-test-api") implementation project(":transportable-udfs-test:transportable-udfs-test-spi") implementation project(":transportable-udfs-trino") + implementation project(":transportable-udfs-trino-plugin") implementation('com.google.guava:guava:24.1-jre') implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') { exclude 'group': 'com.google.collections', 'module': 'google-collections' diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java index 9baaf5c6..00de48d9 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java @@ -7,13 +7,20 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.linkedin.transport.test.spi.TestCase; import com.linkedin.transport.test.spi.types.TestType; +import com.linkedin.transport.trino.StdUdfWrapper; +import com.linkedin.transport.trino.TransportConnector; +import com.linkedin.transport.trino.TransportConnectorFactory; +import com.linkedin.transport.trino.TransportConnectorMetadata; +import com.linkedin.transport.trino.TransportFunctionProvider; import io.trino.FeaturesConfig; import io.trino.Session; -import io.trino.SessionTestUtils; -import io.trino.metadata.InternalFunctionBundle; -import io.trino.metadata.SqlFunction; +import io.trino.client.ClientCapabilities; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.function.BoundSignature; import io.trino.metadata.FunctionBinding; import io.trino.spi.function.FunctionId; @@ -24,13 +31,20 @@ import com.linkedin.transport.test.spi.SqlFunctionCallGenerator; import com.linkedin.transport.test.spi.SqlStdTester; import com.linkedin.transport.test.spi.ToPlatformTestOutputConverter; +import io.trino.spi.function.FunctionProvider; import io.trino.spi.type.Type; +import io.trino.sql.SqlPath; import io.trino.sql.query.QueryAssertions; import io.trino.testing.LocalQueryRunner; +import io.trino.testing.TestingSession; import io.trino.type.InternalTypeManager; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static io.trino.type.UnknownType.UNKNOWN; import static org.assertj.core.api.Assertions.*; @@ -50,7 +64,9 @@ public TrinoTester() { _stdFactory = null; _sqlFunctionCallGenerator = new TrinoSqlFunctionCallGenerator(); _toPlatformTestOutputConverter = new ToTrinoTestOutputConverter(); - _session = SessionTestUtils.TEST_SESSION; + SqlPath sqlPath = new SqlPath("LINKEDIN.TRANSPORT"); + _session = TestingSession.testSessionBuilder().setCatalog("LINKEDIN").setSchema("TRANSPORT").setPath(sqlPath).setClientCapabilities((Set) Arrays.stream( + ClientCapabilities.values()).map(Enum::toString).collect(ImmutableSet.toImmutableSet())).build(); _featuresConfig = new FeaturesConfig(); _runner = LocalQueryRunner.builder(_session).withFeaturesConfig(_featuresConfig).build(); _queryAssertions = new QueryAssertions(_runner); @@ -59,12 +75,19 @@ public TrinoTester() { @Override public void setup( Map, List>> topLevelStdUDFClassesAndImplementations) { + Map functions = new HashMap<>(); // Refresh Trino state during every setup call for (List> stdUDFImplementations : topLevelStdUDFClassesAndImplementations.values()) { for (Class stdUDF : stdUDFImplementations) { - _runner.addFunctions(new InternalFunctionBundle(new SqlFunction[]{new TrinoTestStdUDFWrapper(stdUDF)})); + StdUdfWrapper function = new TrinoTestStdUDFWrapper(stdUDF); + functions.put(function.getFunctionMetadata().getFunctionId(), function); } } + FunctionProvider functionProvider = new TransportFunctionProvider(functions); + ConnectorMetadata connectorMetadata = new TransportConnectorMetadata(functions); + Connector connector = new TransportConnector(connectorMetadata, functionProvider); + ConnectorFactory connectorFactory = new TransportConnectorFactory(connector); + _runner.createCatalog("LINKEDIN", connectorFactory, Collections.emptyMap()); } @Override @@ -75,11 +98,7 @@ public StdFactory getStdFactory() { new BoundSignature("test", UNKNOWN, ImmutableList.of()), ImmutableMap.of(), ImmutableMap.of()); - _stdFactory = new TrinoFactory( - functionBinding, - _runner.getMetadata(), - _runner.getFunctionManager(), - _session, InternalTypeManager.TESTING_TYPE_MANAGER); + _stdFactory = new TrinoFactory(functionBinding, _runner, InternalTypeManager.TESTING_TYPE_MANAGER); } return _stdFactory; } diff --git a/transportable-udfs-trino-plugin/build.gradle b/transportable-udfs-trino-plugin/build.gradle new file mode 100644 index 00000000..92f3741f --- /dev/null +++ b/transportable-udfs-trino-plugin/build.gradle @@ -0,0 +1,33 @@ +apply plugin: 'java' +apply plugin: 'distribution' + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(17)) +} + +dependencies { + implementation project(':transportable-udfs-api') + implementation project(":transportable-udfs-trino") + implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') { + exclude 'group': 'com.google.collections', 'module': 'google-collections' + } + implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version', classifier: 'tests') { + exclude 'group': 'com.google.collections', 'module': 'google-collections' + } + compileOnly(group:'io.trino', name: 'trino-spi', version: project.ext.'trino-version') +} + +distributions { + main { + contents { + from jar + from project.configurations.runtimeClasspath + } + } +} + +artifacts { + archives jar, distTar +} + +build.dependsOn distTar \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java new file mode 100644 index 00000000..210af4d2 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -0,0 +1,50 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTransactionHandle; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionProvider; +import io.trino.spi.transaction.IsolationLevel; +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.*; + + +public class TransportConnector implements Connector { + private final ConnectorMetadata connectorMetadata; + private final FunctionProvider functionProvider; + + public TransportConnector(ConnectorMetadata connectorMetadata, FunctionProvider functionProvider) { + this.connectorMetadata = requireNonNull(connectorMetadata, "connector metadata is null"); + this.functionProvider = requireNonNull(functionProvider, "function provider is null"); + } + + public TransportConnector(Map functions) { + this.connectorMetadata = new TransportConnectorMetadata(functions); + this.functionProvider = new TransportFunctionProvider(functions); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transactionHandle) { + return this.connectorMetadata; + } + + @Override + public Optional getFunctionProvider() { + return Optional.of(this.functionProvider); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, + boolean autoCommit) { + return TransportTransactionHandle.INSTANCE; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java new file mode 100644 index 00000000..c0853c80 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -0,0 +1,97 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; +import io.airlift.log.Logger; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.function.FunctionId; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Objects.*; + + +public class TransportConnectorFactory implements ConnectorFactory { + + private static final String UDF_JAR_PATH = + "file:///Users/yiqding/workspace/trino/core/trino-server-main/plugin/transportable-udfs-trino-plugin-0.0.93/udf/"; + private static final Logger log = Logger.get(TransportConnectorFactory.class); + private Connector connector; + private final Class module; + + + public TransportConnectorFactory(Connector connector) { + this.connector = connector; + this.module = TransportModule.class; + } + + public TransportConnectorFactory(Class module) { + connector = null; + this.module = module; + } + + @Override + public String getName() { + return "TRANSPORT"; + } + + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) { + requireNonNull(config, "config is null"); + if (this.connector == null) { + ClassLoader classLoaderForUdfWrapper = StdUdfWrapper.class.getClassLoader(); + log.info(classLoaderForUdfWrapper.toString()); + ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); + log.info(classLoaderForFactory.toString()); + try { + List classPathList = ImmutableList.of("com.linkedin.transport.examples.trino.ArrayElementAtFunction", + "com.linkedin.transport.examples.trino.ArrayFillFunction", + "com.linkedin.transport.examples.trino.BinaryDuplicateFunction", + "com.linkedin.transport.examples.trino.BinaryObjectSizeFunction", + "com.linkedin.transport.examples.trino.FileLookupFunction", + "com.linkedin.transport.examples.trino.MapFromTwoArraysFunction", + "com.linkedin.transport.examples.trino.MapKeySetFunction", + "com.linkedin.transport.examples.trino.MapValuesFunction", + "com.linkedin.transport.examples.trino.NestedMapFromTwoArraysFunction", + "com.linkedin.transport.examples.trino.NumericAddDoubleFunction", + "com.linkedin.transport.examples.trino.NumericAddFloatFunction", + "com.linkedin.transport.examples.trino.NumericAddIntFunction", + "com.linkedin.transport.examples.trino.NumericAddLongFunction", + "com.linkedin.transport.examples.trino.StructCreateByIndexFunction", + "com.linkedin.transport.examples.trino.StructCreateByNameFunction"); + + List urlList = ImmutableList.of(new URL(UDF_JAR_PATH + "transportable-udfs-example-udfs.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-example-udfs-trino-dist-thin.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-api-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-trino-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-type-system-0.0.93.jar"), + new URL(UDF_JAR_PATH + "transportable-udfs-utils-0.0.93.jar")); + + TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, urlList); + Map functions = new HashMap<>(); + for (String classPath : classPathList) { + Class udfClass = classLoaderForUdf.loadClass(classPath, false); + Object udfObj = udfClass.getConstructor().newInstance(); + log.info(udfObj.toString()); + log.info("adding wrapper"); + StdUdfWrapper wrapper = (StdUdfWrapper) udfObj; + functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); + } + this.connector = new TransportConnector(functions); + } catch (Exception ex) { + log.error(ex); + throw new RuntimeException(ex.getCause()); + } + } + return this.connector; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java new file mode 100644 index 00000000..a8744a75 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java @@ -0,0 +1,42 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionDependencyDeclaration; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.SchemaFunctionName; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + + +public class TransportConnectorMetadata implements ConnectorMetadata { + private final Map functions; + + public TransportConnectorMetadata(Map functions) { + this.functions = functions; + } + + @Override + public FunctionDependencyDeclaration getFunctionDependencies(ConnectorSession session, FunctionId functionId, + BoundSignature boundSignature) { + return functions.get(functionId).getFunctionDependencies(boundSignature); + } + + @Override + public Collection getFunctions(ConnectorSession session, SchemaFunctionName name) { + return functions.values().stream().map(StdUdfWrapper::getFunctionMetadata).collect(Collectors.toList()); + } + + @Override + public FunctionMetadata getFunctionMetadata(ConnectorSession session, FunctionId functionId) { + return functions.get(functionId).getFunctionMetadata(); + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java new file mode 100644 index 00000000..d6678dda --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java @@ -0,0 +1,44 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.function.AggregationImplementation; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionDependencies; +import io.trino.spi.function.FunctionId; +import io.trino.spi.function.FunctionProvider; +import io.trino.spi.function.InvocationConvention; +import io.trino.spi.function.ScalarFunctionImplementation; +import io.trino.spi.function.WindowFunctionSupplier; +import java.util.Map; + + +public class TransportFunctionProvider implements FunctionProvider { + private final Map functions; + + public TransportFunctionProvider(Map functions) { + this.functions = functions; + } + + @Override + public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionId functionId, + BoundSignature boundSignature, FunctionDependencies functionDependencies, + InvocationConvention invocationConvention) { + return functions.get(functionId).getScalarFunctionImplementation(boundSignature, functionDependencies, invocationConvention); + } + + @Override + public AggregationImplementation getAggregationImplementation(FunctionId functionId, BoundSignature boundSignature, + FunctionDependencies functionDependencies) { + return null; + } + + @Override + public WindowFunctionSupplier getWindowFunctionSupplier(FunctionId functionId, BoundSignature boundSignature, + FunctionDependencies functionDependencies) { + return null; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java new file mode 100644 index 00000000..083b4961 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java @@ -0,0 +1,16 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.inject.Binder; +import com.google.inject.Module; + + +public class TransportModule implements Module { + @Override + public void configure(Binder binder) { + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java new file mode 100644 index 00000000..45dafe0e --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java @@ -0,0 +1,18 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import com.google.common.collect.ImmutableList; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; + + +public class TransportPlugin implements Plugin { + @Override + public Iterable getConnectorFactories() { + return ImmutableList.of(new TransportConnectorFactory(TransportModule.class)); + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java new file mode 100644 index 00000000..e2535202 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java @@ -0,0 +1,13 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.spi.connector.ConnectorTransactionHandle; + + +public enum TransportTransactionHandle implements ConnectorTransactionHandle { + INSTANCE +} \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java new file mode 100644 index 00000000..2128d816 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java @@ -0,0 +1,45 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; + + +public class TransportUDFClassLoader extends URLClassLoader { + private final ClassLoader parent; + + public TransportUDFClassLoader(ClassLoader parent, List urls) { + super(urls.toArray(new URL[0])); + this.parent = parent; + } + + @Override + public Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + // Check if class is in the loaded classes cache + Class cachedClass = findLoadedClass(name); + if (cachedClass != null) { + return resolveClass(cachedClass, resolve); + } + + if (name.endsWith("StdUdfWrapper") || name.startsWith("com.linkedin.transport.api")) { + return resolveClass(parent.loadClass(name), resolve); + } + + // Look for class locally + return super.loadClass(name, resolve); + } + } + + private Class resolveClass(Class clazz, boolean resolve) { + if (resolve) { + resolveClass(clazz); + } + return clazz; + } +} \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin b/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin new file mode 100644 index 00000000..b5fd55d8 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/resources/META-INF/services/io.trino.spi.Plugin @@ -0,0 +1 @@ +com.linkedin.transport.trino.TransportPlugin \ No newline at end of file diff --git a/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java new file mode 100644 index 00000000..88df2150 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java @@ -0,0 +1,24 @@ +/** + * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino.plugin; + +import com.linkedin.transport.trino.TransportPlugin; +import io.trino.server.testing.TestingTrinoServer; +import java.sql.Statement; +import org.testng.annotations.BeforeClass; + + +public class TestTransportUdf { + private TestingTrinoServer server; + private Statement statement; + + @BeforeClass + public void setupServer() throws Exception { + server = TestingTrinoServer.create(); + server.installPlugin(new TransportPlugin()); + server.createCatalog("LINKEDIN", "TRANSPORT"); + } +} diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java index 5075d66c..7228c89c 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUdfWrapper.java @@ -30,11 +30,10 @@ import io.trino.spi.function.FunctionDependencyDeclaration; import io.trino.spi.function.FunctionKind; import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.ScalarFunctionImplementation; import io.trino.spi.function.Signature; -import io.trino.metadata.SqlScalarFunction; import io.trino.spi.function.TypeVariableConstraint; import io.trino.operator.scalar.ChoicesSpecializedSqlScalarFunction; -import io.trino.operator.scalar.SpecializedSqlScalarFunction; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.function.InvocationConvention; import io.trino.spi.type.ArrayType; @@ -65,13 +64,15 @@ // Suppressing argument naming convention for the evalInternal methods @SuppressWarnings({"checkstyle:regexpsinglelinejava"}) -public abstract class StdUdfWrapper extends SqlScalarFunction { +public abstract class StdUdfWrapper { private static final int DEFAULT_REFRESH_INTERVAL_DAYS = 1; private static final int JITTER_FACTOR = 50; // to calculate jitter from delay - protected StdUdfWrapper(StdUDF stdUDF) { - super(FunctionMetadata.builder(FunctionKind.SCALAR) + private final FunctionMetadata functionMetadata; + + public StdUdfWrapper(StdUDF stdUDF) { + this.functionMetadata = FunctionMetadata.builder(FunctionKind.SCALAR) .nullable() .nondeterministic() .description(((TopLevelStdUDF) stdUDF).getFunctionDescription()) @@ -82,7 +83,11 @@ protected StdUdfWrapper(StdUDF stdUDF) { .argumentTypes(stdUDF.getInputParameterSignatures().stream() .map(typeSignature -> parseTypeSignature(quoteReservedKeywords(typeSignature), ImmutableSet.of())).collect(Collectors.toList())) .build()) - .build()); + .build(); + } + + public FunctionMetadata getFunctionMetadata() { + return this.functionMetadata; } @VisibleForTesting @@ -116,7 +121,6 @@ private void registerNestedDependencies(Type nestedType, FunctionDependencyDecla } } - @Override public FunctionDependencyDeclaration getFunctionDependencies(BoundSignature boundSignature) { FunctionDependencyDeclaration.FunctionDependencyDeclarationBuilder builder = FunctionDependencyDeclaration.builder(); @@ -127,10 +131,9 @@ public FunctionDependencyDeclaration getFunctionDependencies(BoundSignature boun return builder.build(); } - @Override - public SpecializedSqlScalarFunction specialize(BoundSignature boundSignature, FunctionDependencies functionDependencies) { - FunctionMetadata metadata = getFunctionMetadata(); - FunctionBinding functionBinding = SignatureBinder.bindFunction(metadata.getFunctionId(), metadata.getSignature(), boundSignature); + public ScalarFunctionImplementation getScalarFunctionImplementation(BoundSignature boundSignature, + FunctionDependencies functionDependencies, InvocationConvention invocationConvention) { + FunctionBinding functionBinding = SignatureBinder.bindFunction(functionMetadata.getFunctionId(), functionMetadata.getSignature(), boundSignature); StdFactory stdFactory = new TrinoFactory(functionBinding, functionDependencies); StdUDF stdUDF = getStdUDF(); stdUDF.init(stdFactory); @@ -146,7 +149,7 @@ public SpecializedSqlScalarFunction specialize(BoundSignature boundSignature, Fu boundSignature, NULLABLE_RETURN, getNullConventionForArguments(nullableArguments), - getMethodHandle(stdUDF, boundSignature, nullableArguments, requiredFilesNextRefreshTime)); + getMethodHandle(stdUDF, boundSignature, nullableArguments, requiredFilesNextRefreshTime)).getScalarFunctionImplementation(invocationConvention); } private MethodHandle getMethodHandle(StdUDF stdUDF, BoundSignature boundSignature, boolean[] nullableArguments, diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java index 8f294d91..b06ae75e 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java @@ -30,11 +30,8 @@ import com.linkedin.transport.trino.data.TrinoString; import com.linkedin.transport.trino.data.TrinoStruct; import io.airlift.slice.Slices; -import io.trino.Session; import io.trino.metadata.FunctionBinding; -import io.trino.metadata.FunctionManager; import io.trino.spi.function.FunctionDependencies; -import io.trino.metadata.Metadata; import io.trino.metadata.OperatorNotFoundException; import io.trino.spi.function.InvocationConvention; import io.trino.spi.function.OperatorType; @@ -44,6 +41,7 @@ import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; +import io.trino.testing.LocalQueryRunner; import java.lang.invoke.MethodHandle; import java.nio.ByteBuffer; import java.util.List; @@ -58,28 +56,22 @@ public class TrinoFactory implements StdFactory { final FunctionBinding functionBinding; final FunctionDependencies functionDependencies; - final Metadata metadata; final TypeManager typeManager; - final FunctionManager functionManager; - - final Session session; + final LocalQueryRunner queryRunner; public TrinoFactory(FunctionBinding functionBinding, FunctionDependencies functionDependencies) { this.functionBinding = functionBinding; this.functionDependencies = functionDependencies; - this.metadata = null; this.typeManager = null; - this.functionManager = null; - this.session = null; + this.queryRunner = null; } - public TrinoFactory(FunctionBinding functionBinding, Metadata metadata, FunctionManager functionManager, Session session, TypeManager typeManager) { + // for test only + public TrinoFactory(FunctionBinding functionBinding, LocalQueryRunner queryRunner, TypeManager typeManager) { this.functionBinding = functionBinding; this.functionDependencies = null; - this.metadata = metadata; + this.queryRunner = queryRunner; this.typeManager = typeManager; - this.functionManager = functionManager; - this.session = session; } @Override @@ -163,8 +155,9 @@ public MethodHandle getOperatorHandle( OperatorType operatorType, List argumentTypes, InvocationConvention invocationConvention) throws OperatorNotFoundException { - if (functionManager != null) { - return functionManager.getScalarFunctionImplementation(metadata.resolveOperator(session, operatorType, argumentTypes), + if (queryRunner != null && queryRunner.getFunctionManager() != null) { + return queryRunner.getFunctionManager() + .getScalarFunctionImplementation(queryRunner.getMetadata().resolveOperator(queryRunner.getDefaultSession(), operatorType, argumentTypes), invocationConvention).getMethodHandle(); } return functionDependencies.getOperatorImplementation(operatorType, argumentTypes, invocationConvention).getMethodHandle(); From 13588afbe163a9638e144ac553819e34db32cb27 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Thu, 20 Apr 2023 16:49:53 -0700 Subject: [PATCH 05/18] address comments and remove unreachable artifactory repo --- defaultEnvironment.gradle | 3 --- .../linkedin/transport/trino/TransportConnectorFactory.java | 6 ++---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index f0017e6e..8771dee2 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -6,9 +6,6 @@ subprojects { repositories { mavenCentral() jcenter() - maven { - url "https://conjars.org/repo" - } } project.ext.setProperty('trino-version', '406') project.ext.setProperty('airlift-slice-version', '0.44') diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java index 64474fca..3f871732 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -17,7 +17,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +27,7 @@ public class TransportConnectorFactory implements ConnectorFactory { - private static final String TRANSPORT_UDF_MP = "/transport-udf-mp"; + private static final String TRANSPORT_UDF_REPO = "/transport-udf-repo"; private static final Logger log = Logger.get(TransportConnectorFactory.class); private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> { @@ -76,9 +75,8 @@ public Connector create(String catalogName, Map config, Connecto private List getUDFJarUrls() { String workingDir = System.getProperty("user.dir"); - String udfDir = workingDir + TRANSPORT_UDF_MP; + String udfDir = workingDir + TRANSPORT_UDF_REPO; File[] udfSubDirs = new File(udfDir).listFiles(File::isDirectory); - log.info(Arrays.toString(udfSubDirs)); List urlList = new ArrayList<>(); for (File subDirPath : udfSubDirs) { getUDFJarUrlFromDir(subDirPath, urlList); From 82286fb4aabb1a48991b7d7bc7964cd73cfb3e4e Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Fri, 21 Apr 2023 10:52:06 -0700 Subject: [PATCH 06/18] address comments --- .../trino/TransportConnectorFactory.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java index 3f871732..27dc76d7 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -17,17 +17,20 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ServiceLoader; +import java.util.stream.Collectors; import static java.util.Objects.*; public class TransportConnectorFactory implements ConnectorFactory { - private static final String TRANSPORT_UDF_REPO = "/transport-udf-repo"; + private static final String DEFAULT_TRANSPORT_UDF_REPO = "/transport-udf-repo"; + private static final String TRANSPORT_UDF_REPO_CONFIG_NAME = "transport.udf.repo"; private static final Logger log = Logger.get(TransportConnectorFactory.class); private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> { @@ -59,8 +62,8 @@ public Connector create(String catalogName, Map config, Connecto requireNonNull(config, "config is null"); if (this.connector == null) { ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); - List allUrlList = getUDFJarUrls(); - TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, allUrlList); + List jarUrlList = getUDFJarUrls(config); + TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); Map functions = new HashMap<>(); @@ -73,18 +76,15 @@ public Connector create(String catalogName, Map config, Connecto return this.connector; } - private List getUDFJarUrls() { - String workingDir = System.getProperty("user.dir"); - String udfDir = workingDir + TRANSPORT_UDF_REPO; - File[] udfSubDirs = new File(udfDir).listFiles(File::isDirectory); - List urlList = new ArrayList<>(); - for (File subDirPath : udfSubDirs) { - getUDFJarUrlFromDir(subDirPath, urlList); - } - return urlList; + private List getUDFJarUrls(Map config) { + String udfDir = config.containsKey(TRANSPORT_UDF_REPO_CONFIG_NAME) + ? config.get(TRANSPORT_UDF_REPO_CONFIG_NAME) : DEFAULT_TRANSPORT_UDF_REPO; + File[] udfSubDirs = new File(System.getProperty("user.dir") + udfDir).listFiles(File::isDirectory); + return Arrays.stream(udfSubDirs).flatMap(e -> getUDFJarUrlFromDir(e).stream()).collect(Collectors.toList()); } - private void getUDFJarUrlFromDir(File path, List urlList) { + private List getUDFJarUrlFromDir(File path) { + List urlList = new ArrayList<>(); File[] files = path.listFiles(TRANSPORT_UDF_JAR_FILTER); for (File file : files) { try { @@ -94,5 +94,6 @@ private void getUDFJarUrlFromDir(File path, List urlList) { throw new RuntimeException(ex); } } + return urlList; } } From 04c68e3245ce21d02b3ababbb3a4c2e81832614b Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Mon, 24 Apr 2023 10:11:45 -0700 Subject: [PATCH 07/18] fix the issue of Maven repo --- defaultEnvironment.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index 8771dee2..106f3d53 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -6,6 +6,9 @@ subprojects { repositories { mavenCentral() jcenter() + maven { + url "https://conjars.wensel.net/repo" + } } project.ext.setProperty('trino-version', '406') project.ext.setProperty('airlift-slice-version', '0.44') From 2722cbe51ccd700db19273a099315940fafd2b77 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Mon, 24 Apr 2023 12:12:46 -0700 Subject: [PATCH 08/18] fixing conjar repo issue in transportable-udfs-examples --- transportable-udfs-examples/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transportable-udfs-examples/build.gradle b/transportable-udfs-examples/build.gradle index f02a88f3..a367f892 100644 --- a/transportable-udfs-examples/build.gradle +++ b/transportable-udfs-examples/build.gradle @@ -32,7 +32,7 @@ subprojects { repositories { mavenCentral() maven { - url "https://conjars.org/repo" + url "https://conjars.wensel.net/repo" } } } From 97f87770e57878f421dbf85e91b1cdbf14f3b595 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Tue, 25 Apr 2023 12:54:44 -0700 Subject: [PATCH 09/18] address comments --- .../transport/test/trino/TrinoTester.java | 13 +++- .../transport/trino/TransportConnector.java | 5 +- .../trino/TransportConnectorFactory.java | 70 ++++++++----------- .../trino/TransportConnectorMetadata.java | 7 +- .../trino/TransportFunctionProvider.java | 2 +- .../transport/trino/TransportPlugin.java | 4 +- .../trino/TransportTransactionHandle.java | 2 +- .../trino/TransportUDFClassLoader.java | 5 +- .../trino/plugin/TestTransportUdf.java | 24 ------- .../linkedin/transport/trino/StdUDFUtils.java | 8 +-- 10 files changed, 57 insertions(+), 83 deletions(-) delete mode 100644 transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java index 0e1dec78..1dca926f 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java @@ -13,13 +13,13 @@ import com.linkedin.transport.test.spi.types.TestType; import com.linkedin.transport.trino.StdUdfWrapper; import com.linkedin.transport.trino.TransportConnector; -import com.linkedin.transport.trino.TransportConnectorFactory; import com.linkedin.transport.trino.TransportConnectorMetadata; import com.linkedin.transport.trino.TransportFunctionProvider; import io.trino.FeaturesConfig; import io.trino.Session; import io.trino.client.ClientCapabilities; import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.function.BoundSignature; @@ -87,7 +87,16 @@ public void setup( FunctionProvider functionProvider = new TransportFunctionProvider(functions); ConnectorMetadata connectorMetadata = new TransportConnectorMetadata(functions); Connector connector = new TransportConnector(connectorMetadata, functionProvider); - ConnectorFactory connectorFactory = new TransportConnectorFactory(connector); + ConnectorFactory connectorFactory = new ConnectorFactory() { + @Override + public String getName() { + return "TRANSPORT"; + } + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) { + return connector; + } + };; _runner.createCatalog("LINKEDIN", connectorFactory, Collections.emptyMap()); } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java index 210af4d2..7a1df50f 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -28,8 +28,7 @@ public TransportConnector(ConnectorMetadata connectorMetadata, FunctionProvider } public TransportConnector(Map functions) { - this.connectorMetadata = new TransportConnectorMetadata(functions); - this.functionProvider = new TransportFunctionProvider(functions); + this(new TransportConnectorMetadata(functions), new TransportFunctionProvider(functions)); } @Override diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java index 27dc76d7..b1ac5b8a 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -1,12 +1,11 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ package com.linkedin.transport.trino; import com.google.common.collect.ImmutableList; -import com.google.inject.Module; import io.airlift.log.Logger; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -16,6 +15,8 @@ import java.io.FileFilter; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -29,28 +30,12 @@ public class TransportConnectorFactory implements ConnectorFactory { - private static final String DEFAULT_TRANSPORT_UDF_REPO = "/transport-udf-repo"; + private static final String DEFAULT_TRANSPORT_UDF_REPO = "transport-udf-repo"; private static final String TRANSPORT_UDF_REPO_CONFIG_NAME = "transport.udf.repo"; private static final Logger log = Logger.get(TransportConnectorFactory.class); - private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> { - return file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs"); - }; - - private Connector connector; - private final Class module; - - - // test only - public TransportConnectorFactory(Connector connector) { - this.connector = connector; - this.module = TransportModule.class; - } - - public TransportConnectorFactory(Class module) { - this.connector = null; - this.module = module; - } + private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> + file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs"); @Override public String getName() { @@ -60,37 +45,40 @@ public String getName() { @Override public Connector create(String catalogName, Map config, ConnectorContext context) { requireNonNull(config, "config is null"); - if (this.connector == null) { - ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); - List jarUrlList = getUDFJarUrls(config); - TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); - ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); - List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); - Map functions = new HashMap<>(); - for (StdUdfWrapper wrapper : stdUdfWrappers) { - log.info("Loading Transport UDF class: " + wrapper.getFunctionMetadata().getFunctionId().toString()); - functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); - } - this.connector = new TransportConnector(functions); + ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); + List jarUrlList = getUDFJarUrls(config); + log.info("The URLs of Transport UDF jars: " + jarUrlList); + TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); + ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); + List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); + Map functions = new HashMap<>(); + for (StdUdfWrapper wrapper : stdUdfWrappers) { + log.info("Loading Transport UDF class: " + wrapper.getFunctionMetadata().getFunctionId().toString()); + functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); } - return this.connector; + return new TransportConnector(functions); } - private List getUDFJarUrls(Map config) { - String udfDir = config.containsKey(TRANSPORT_UDF_REPO_CONFIG_NAME) - ? config.get(TRANSPORT_UDF_REPO_CONFIG_NAME) : DEFAULT_TRANSPORT_UDF_REPO; - File[] udfSubDirs = new File(System.getProperty("user.dir") + udfDir).listFiles(File::isDirectory); + private static List getUDFJarUrls(Map config) { + String udfDir = config.getOrDefault(TRANSPORT_UDF_REPO_CONFIG_NAME, DEFAULT_TRANSPORT_UDF_REPO); + if (!Paths.get(udfDir).isAbsolute()) { + Path workingDirPath = Paths.get("").toAbsolutePath(); + udfDir = Paths.get(workingDirPath.toString(), udfDir).toString(); + } + File[] udfSubDirs = new File(udfDir).listFiles(File::isDirectory); return Arrays.stream(udfSubDirs).flatMap(e -> getUDFJarUrlFromDir(e).stream()).collect(Collectors.toList()); } - private List getUDFJarUrlFromDir(File path) { + private static List getUDFJarUrlFromDir(File path) { List urlList = new ArrayList<>(); File[] files = path.listFiles(TRANSPORT_UDF_JAR_FILTER); for (File file : files) { try { - urlList.add(file.toURI().toURL()); + if (file != null) { + urlList.add(file.toURI().toURL()); + } } catch (MalformedURLException ex) { - log.error("Fail to parsing the URL of the given jar file", ex.getMessage()); + log.error("Fail to parsing the URL of the given jar file ", ex); throw new RuntimeException(ex); } } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java index 40e018a3..c847b09c 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -32,8 +32,9 @@ public FunctionDependencyDeclaration getFunctionDependencies(ConnectorSession se @Override public Collection getFunctions(ConnectorSession session, SchemaFunctionName name) { - return functions.values().stream().filter(e -> e.getFunctionMetadata().getCanonicalName().equals(name.getFunctionName())) - .map(StdUdfWrapper::getFunctionMetadata).collect(Collectors.toList()); + return functions.values().stream().map(StdUdfWrapper::getFunctionMetadata) + .filter(e -> e.getCanonicalName().equals(name.getFunctionName())) + .collect(Collectors.toList()); } @Override diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java index d6678dda..db7e8cce 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java index 45dafe0e..0713c281 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -13,6 +13,6 @@ public class TransportPlugin implements Plugin { @Override public Iterable getConnectorFactories() { - return ImmutableList.of(new TransportConnectorFactory(TransportModule.class)); + return ImmutableList.of(new TransportConnectorFactory()); } } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java index e2535202..44b95ba0 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java index 2128d816..a92ed567 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -27,7 +27,8 @@ public Class loadClass(String name, boolean resolve) throws ClassNotFoundExce return resolveClass(cachedClass, resolve); } - if (name.endsWith("StdUdfWrapper") || name.startsWith("com.linkedin.transport.api")) { + if (name.equals("com.linkedin.transport.trino.StdUdfWrapper") + || name.startsWith("com.linkedin.transport.api")) { return resolveClass(parent.loadClass(name), resolve); } diff --git a/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java deleted file mode 100644 index 88df2150..00000000 --- a/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/plugin/TestTransportUdf.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Copyright 2019 LinkedIn Corporation. All rights reserved. - * Licensed under the BSD-2 Clause license. - * See LICENSE in the project root for license information. - */ -package com.linkedin.transport.trino.plugin; - -import com.linkedin.transport.trino.TransportPlugin; -import io.trino.server.testing.TestingTrinoServer; -import java.sql.Statement; -import org.testng.annotations.BeforeClass; - - -public class TestTransportUdf { - private TestingTrinoServer server; - private Statement statement; - - @BeforeClass - public void setupServer() throws Exception { - server = TestingTrinoServer.create(); - server.installPlugin(new TransportPlugin()); - server.createCatalog("LINKEDIN", "TRANSPORT"); - } -} diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUDFUtils.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUDFUtils.java index a30189a4..bd7e2579 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUDFUtils.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/StdUDFUtils.java @@ -32,10 +32,10 @@ private StdUDFUtils() { "CURRENT_CATALOG", "CURRENT_DATE", "CURRENT_PATH", "CURRENT_ROLE", "CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "DEALLOCATE", "DELETE", "DESCRIBE", "DISTINCT", "DROP", "ELSE", "END", "ESCAPE", "EXCEPT", "EXECUTE", "EXISTS", "EXTRACT", "FALSE", "FOR", "FROM", "FULL", "GROUP", "GROUPING", - "HAVING", "IN", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JOIN", "LEFT", "LIKE", "LISTAGG", "LOCALTIME", - "LOCALTIMESTAMP", "NATURAL", "NORMALIZE", "NOT", "NULL", "ON", "OR", "ORDER", "OUTER", "PREPARE", "RECURSIVE", - "RIGHT", "ROLLUP", "SELECT", "SKIP", "TABLE", "THEN", "TRUE", "UESCAPE", "UNION", "UNNEST", "USING", "VALUES", - "WHEN", "WHERE", "WITH"); + "HAVING", "IN", "INNER", "INSERT", "INTERSECT", "INTO", "IS", "JSON_ARRAY", "JSON_EXISTS", "JSON_OBJECT", + "JSON_QUERY", "JSON_VALUE", "JOIN", "LEFT", "LIKE", "LISTAGG", "LOCALTIME", "LOCALTIMESTAMP", "NATURAL", "NORMALIZE", + "NOT", "NULL", "ON", "OR", "ORDER", "OUTER", "PREPARE", "RECURSIVE", "RIGHT", "ROLLUP", "SELECT", "SKIP", + "TABLE", "THEN", "TRIM", "TRUE", "UESCAPE", "UNION", "UNNEST", "USING", "VALUES", "WHEN", "WHERE", "WITH"); /** * Quote the reserved keywords which might appear as field names in the type signatures From 29b067facd549faad0e776f7280a8766c64a9a73 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Tue, 25 Apr 2023 13:41:37 -0700 Subject: [PATCH 10/18] address comments --- .../transport/trino/TransportConfig.java | 23 ++++++ .../transport/trino/TransportConnector.java | 66 ++++++++++++++++- .../trino/TransportConnectorFactory.java | 74 ++++--------------- .../transport/trino/TransportModule.java | 7 +- 4 files changed, 108 insertions(+), 62 deletions(-) create mode 100644 transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java new file mode 100644 index 00000000..e494a044 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java @@ -0,0 +1,23 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.airlift.configuration.Config; + + +public class TransportConfig { + private String transportUdfRepo; + + public String getTransportUdfRepo() { + return transportUdfRepo; + } + + @Config("transport.udf.repo") + public TransportConfig setTransportUdfRepo(String transportUdfRepo) { + this.transportUdfRepo = transportUdfRepo; + return this; + } +} diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java index 7a1df50f..a424917c 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -5,6 +5,8 @@ */ package com.linkedin.transport.trino; +import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorSession; @@ -12,13 +14,33 @@ import io.trino.spi.function.FunctionId; import io.trino.spi.function.FunctionProvider; import io.trino.spi.transaction.IsolationLevel; +import java.io.File; +import java.io.FileFilter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.ServiceLoader; +import java.util.stream.Collectors; +import javax.inject.Inject; +import org.apache.bval.util.StringUtils; import static java.util.Objects.*; public class TransportConnector implements Connector { + + private static final Logger log = Logger.get(TransportConnector.class); + private static final String DEFAULT_TRANSPORT_UDF_REPO = "transport-udf-repo"; + private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> + file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs"); + private final ConnectorMetadata connectorMetadata; private final FunctionProvider functionProvider; @@ -27,8 +49,22 @@ public TransportConnector(ConnectorMetadata connectorMetadata, FunctionProvider this.functionProvider = requireNonNull(functionProvider, "function provider is null"); } - public TransportConnector(Map functions) { - this(new TransportConnectorMetadata(functions), new TransportFunctionProvider(functions)); + @Inject + public TransportConnector(TransportConfig config) { + ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); + List jarUrlList = getUDFJarUrls(config); + log.info("The URLs of Transport UDF jars: " + jarUrlList); + TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); + ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); + List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); + Map functions = new HashMap<>(); + for (StdUdfWrapper wrapper : stdUdfWrappers) { + log.info("Loading Transport UDF class: " + wrapper.getFunctionMetadata().getFunctionId().toString()); + functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); + } + + this.connectorMetadata = new TransportConnectorMetadata(functions); + this.functionProvider = new TransportFunctionProvider(functions); } @Override @@ -46,4 +82,30 @@ public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel boolean autoCommit) { return TransportTransactionHandle.INSTANCE; } + + private static List getUDFJarUrls(TransportConfig config) { + String udfDir = StringUtils.isBlank(config.getTransportUdfRepo()) ? DEFAULT_TRANSPORT_UDF_REPO : config.getTransportUdfRepo(); + if (!Paths.get(udfDir).isAbsolute()) { + Path workingDirPath = Paths.get("").toAbsolutePath(); + udfDir = Paths.get(workingDirPath.toString(), udfDir).toString(); + } + File[] udfSubDirs = new File(udfDir).listFiles(File::isDirectory); + return Arrays.stream(udfSubDirs).flatMap(e -> getUDFJarUrlFromDir(e).stream()).collect(Collectors.toList()); + } + + private static List getUDFJarUrlFromDir(File path) { + List urlList = new ArrayList<>(); + File[] files = path.listFiles(TRANSPORT_UDF_JAR_FILTER); + for (File file : files) { + try { + if (file != null) { + urlList.add(file.toURI().toURL()); + } + } catch (MalformedURLException ex) { + log.error("Fail to parsing the URL of the given jar file ", ex); + throw new RuntimeException(ex); + } + } + return urlList; + } } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java index b1ac5b8a..f8fbdd49 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -5,38 +5,19 @@ */ package com.linkedin.transport.trino; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.function.FunctionId; -import java.io.File; -import java.io.FileFilter; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.ServiceLoader; -import java.util.stream.Collectors; +import static io.trino.plugin.base.Versions.*; import static java.util.Objects.*; public class TransportConnectorFactory implements ConnectorFactory { - - private static final String DEFAULT_TRANSPORT_UDF_REPO = "transport-udf-repo"; - private static final String TRANSPORT_UDF_REPO_CONFIG_NAME = "transport.udf.repo"; - private static final Logger log = Logger.get(TransportConnectorFactory.class); - - private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> - file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs"); - @Override public String getName() { return "TRANSPORT"; @@ -45,43 +26,18 @@ public String getName() { @Override public Connector create(String catalogName, Map config, ConnectorContext context) { requireNonNull(config, "config is null"); - ClassLoader classLoaderForFactory = TransportConnectorFactory.class.getClassLoader(); - List jarUrlList = getUDFJarUrls(config); - log.info("The URLs of Transport UDF jars: " + jarUrlList); - TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); - ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); - List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); - Map functions = new HashMap<>(); - for (StdUdfWrapper wrapper : stdUdfWrappers) { - log.info("Loading Transport UDF class: " + wrapper.getFunctionMetadata().getFunctionId().toString()); - functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); - } - return new TransportConnector(functions); - } + checkSpiVersion(context, this); - private static List getUDFJarUrls(Map config) { - String udfDir = config.getOrDefault(TRANSPORT_UDF_REPO_CONFIG_NAME, DEFAULT_TRANSPORT_UDF_REPO); - if (!Paths.get(udfDir).isAbsolute()) { - Path workingDirPath = Paths.get("").toAbsolutePath(); - udfDir = Paths.get(workingDirPath.toString(), udfDir).toString(); - } - File[] udfSubDirs = new File(udfDir).listFiles(File::isDirectory); - return Arrays.stream(udfSubDirs).flatMap(e -> getUDFJarUrlFromDir(e).stream()).collect(Collectors.toList()); - } + // A plugin is not required to use Guice; it is just very convenient + Bootstrap app = new Bootstrap( + new TypeDeserializerModule(context.getTypeManager()), + new TransportModule()); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); - private static List getUDFJarUrlFromDir(File path) { - List urlList = new ArrayList<>(); - File[] files = path.listFiles(TRANSPORT_UDF_JAR_FILTER); - for (File file : files) { - try { - if (file != null) { - urlList.add(file.toURI().toURL()); - } - } catch (MalformedURLException ex) { - log.error("Fail to parsing the URL of the given jar file ", ex); - throw new RuntimeException(ex); - } - } - return urlList; + return injector.getInstance(TransportConnector.class); } } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java index 083b4961..75510f00 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java @@ -1,5 +1,5 @@ /** - * Copyright 2019 LinkedIn Corporation. All rights reserved. + * Copyright 2023 LinkedIn Corporation. All rights reserved. * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ @@ -7,10 +7,15 @@ import com.google.inject.Binder; import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.*; public class TransportModule implements Module { @Override public void configure(Binder binder) { + binder.bind(TransportConnector.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(TransportConfig.class); } } From f795e24a4518192225aef503f9c2f73c4854e98f Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Tue, 25 Apr 2023 16:38:42 -0700 Subject: [PATCH 11/18] address comments --- .../trino/TransportUDFClassLoader.java | 11 ++++++++ .../transport/trino/TransportPluginTest.java | 26 ++++++++++++++++++ .../transport-udf-1-trino-dist-thin.jar | Bin 0 -> 5595 bytes .../transport-udf-1-trino/transport-udf-1.jar | Bin 0 -> 9956 bytes .../transport-udf-2-trino-dist-thin.jar | Bin 0 -> 4848 bytes .../transport-udf-2-trino/transport-udf-2.jar | Bin 0 -> 9695 bytes 6 files changed, 37 insertions(+) create mode 100644 transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/TransportPluginTest.java create mode 100644 transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1-trino-dist-thin.jar create mode 100644 transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1.jar create mode 100644 transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2-trino-dist-thin.jar create mode 100644 transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2.jar diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java index a92ed567..fae72dfb 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportUDFClassLoader.java @@ -10,6 +10,17 @@ import java.util.List; +/** + * The approach of Trino plugin is used to dynamically load UDF classes into Trino server. + * The infrastructure classes of this Trino plugin (e.g. TransportConnectorFactory, TransportConnector and so on) are + * loaded by PluginClassLoader defined in Trino during the initialization of plugins. In current implementation of Trino, + * only the URLs of the jars immediately under the directory where TransportPlugin deployed are passed into PluginClassLoader. + * However, the jars containing actual UDF classes cannot be deployed in the same directory. As the URLs of jars with actual UDF classes + * are not visible to PluginClassLoader, PluginClassLoader cannot be used to load UDF classes. + * Therefore, TransportUDFClassLoader is built with the URLs to the jars with UDF classes to load all UDF classes inside those jars. + * Also, PluginClassLoader is used as the parent of TransportClassLoader. It can help prevent TransportUDFClassLoader load some base classes + * defined in Transport (e.g. com.linkedin.transport.trino.StdUdfWrapper) which have been already loaded by PluginClassLoader + */ public class TransportUDFClassLoader extends URLClassLoader { private final ClassLoader parent; diff --git a/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/TransportPluginTest.java b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/TransportPluginTest.java new file mode 100644 index 00000000..b684a033 --- /dev/null +++ b/transportable-udfs-trino-plugin/src/test/java/com/linkedin/transport/trino/TransportPluginTest.java @@ -0,0 +1,26 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.trino; + +import io.trino.server.testing.TestingTrinoServer; +import io.trino.spi.Plugin; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; + + +public class TransportPluginTest { + + @Test + public void testTransportPluginInitialization() { + TestingTrinoServer server = TestingTrinoServer.create(); + Plugin plugin = new TransportPlugin(); + server.installPlugin(plugin); + server.createCatalog("LINKEDIN", "TRANSPORT"); + Assert.assertTrue(getOnlyElement(plugin.getConnectorFactories()) instanceof TransportConnectorFactory); + } +} diff --git a/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1-trino-dist-thin.jar b/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1-trino-dist-thin.jar new file mode 100644 index 0000000000000000000000000000000000000000..7a459c158307fbc56817266ea0ad041dce963f5d GIT binary patch literal 5595 zcmb7I2{@G7A0GQw24fxj77fO(RAMX>*@mQSU1LmRm@H$smXN)sC?z7YWofxt@4a@} ziLAMnEQyMwD~j|N|8KZaQ+aeeGjlx8`TgGSocDWw%Rw2_(X)a;%*-HW=d0ErV9-*& zfWZQMObx6M3Pxr~MLH1bn}tOov~LB01p>f_`|E|K2s0z3!7(eiDRRNIz0Jf}0p7ja zSOM12-dSv^^hEX9i{~$Bshk1!o2c}_9OH53JHW5k?EY^J0q^dHb;i4IyO@u9F^PzF z_xAK4lD0Jy`?(p`4}Hdyfc4(ie$Nio$i}Z2FknkSva0~(M?HyxbYEG-n5yaq(rm6vb&l+*Bm z+h5i7k;yi)*L?>esZlgH%W~~eAI@qfE182ovdtLO_1d!v(nrFqQBtxZUbQ}nVq>Hz zjia)51rs%_2su3+Ir!~^U8{d;S6isX`tTAR*H?=5n5KYsteJ}DWIP{dNrG#|`H z~tg#ZSJ)E zbn-)y7DNC0%@ba}#q^6Z@8g`n$BW4yC)*#iXPVfn4~oyfWU#Um75m$J@>Rhom(_^= z(_y6`p7=uEEZ&p%v(Rs8H42jEOCYDhP4TBfG|5I)i@7!Nix7fz-mK`iIv`*6ziR}L z6#|f@yJNBtL?YVXfPg)NbtfT6+fwB>PoWAdLT_~b(FH$*vPheI_iOixy$)GxN6n>@ zKhmvOG`%fS77^>IFHsHp_wmj4{-yuPoICc$riD`|P=!PX7fI@jyvM`SYY3Yy!ah2+#FJZ><@W|a^x;$>OT-*okJx8Ql_NB(fa6RIEGnud3Wh;@-!`CD1 zx;B(HEDJR-5%Babpf{6z+q5browD?_qeV9+eIaehKd}+QzlQ6<(SM-fd!xeM3=ri7 z5ETj#l}VQIx26y!Jq0f=wd~cP5Vg!tZ`Nz$K8 zVb8*+dln`$e`>E-HpvVaSyX6$OVmHJVOI?Sq5=V;5IZKy%;yZ2h{qtDosHaoz*C(L zM-ZD~#yQzw>Cs9Hb`9}&5>8HPQhOTM$nQ!^EMPed~%&F zs6j_rtHLM=y?6RByqUnfDb6J;qKM((@;#-Jggilo<5TRHLMASIs9%=IU>Sa+7`k9s zUZ`!x!dDfmT7tq%t>A)2vEv};wL|Y2^r{rEi?tX|xgBQB>zRrc0Y}-lNp+GL7Q}LQ zjbPbNOb?=E$JkUw!)oyd=|F zfi7-7lS0$41-~tt^gVo>2-LwZuFZ^AnLBEYRz>NY?5H(-SXn-$;X0=@ltu12V`Hg& zZ)RUcYJBXwLXtt|1@JRHfrIr8rAdZ-xmvknWh+*078E1d-Sy*8;Sw>v6K}AW|h>t$|z(1%xvT3Y)O)jSoEH=Smc?UeN z)j$z4xuM;pY9ErRx&Yw}NV@^IM}tzHmTLdOH@> zMw-(1@g(3eXc9G1K$fCNN^V_J4wZgcLWJjy`uY&*l=FPt6JB^e2Zg7n2ce%&1W0r@ zh(5clVrGn~R4a-eOdVVrZErgpR1-ow<n57did_MU8AxS-Kc*ek*0& z0VCHnW#Fxd+lBL2h4=qik1B(lMn}}*HH`yK7eu4)Yjr2Kss(C(Vgj#Rz=*03=@G46 zEJ~!wZt=odI*RH4tb!_1vy`HN&<9=D6byFK9K|ND(RI7xyeR>DAo>m*t(w&TTa2Y` z{@Qq=>fw4{^3XEPw@fjzp1MZS)UV~JElrttxZ}RhQ52$#<8&$Igx@?;bD$xd3Bi>$ z{)CdF5`Am?ik`~bO>}-KC%o*S6EqTmH%5}Sa+J5wfGkG}Bk{F?{h`7uZ02&X(}h+; z7)28F2;rrJ%&{ClPtK@n?rXskw2!krbx`-wbkbp#iEIx7>8;y@56EUY)f=tdrJv=Q zxAhXQ3T5}Gf0vr+)gDjt_X>|E)LW6ISz!F`m{+vMYx9IhKH4jV z3n=^O9f`Z679`N+V#s5@P zQ7sVk{;T+`4h3Q~OS>;xJZW&*Tegc($@btnf4zEA5?0?%b#DsF4K!4alqab_dQ6)p#qp_bEV`4e*w#He}Ri2YMAp!i2)%IyBh zuSZowPM^L7$Y|}XO!x>GGs|KrDf(FR#R$`CLt!}Vb3hSMjW?O4@%W?IE4dKVnUHfM zW^rkC0mc?SWQ|#|Qw-Sq(s=^BGcH}H9x`gErhfh|HOU#Mhft-a@xMes6dCa!2SS!4h;ivji&~4QE??IJDVvj*7jb z`)I)|6T@Mwuay$?YH4(&H)uWJVft%XYc29}f~6)QINeXkW11JQYa9;A=-vNZbq$t8 zRH*ZA3Ny5TNU5k%64MJtYYXwb6z09{dl7dgV(hIfyr1QS_=LHKR#Os_3y_$GU7^kP zcAhvCD7SD9qS4Bni0Bm5bm4kWiFer#!_+-9_+?ynM@;gnz5J9MD`|%ZTN!3?_DMWm za2h;xUZB>d`%;_FEa#KAWj6fca|cGR-eMWfI7(`vi!&=eksdN)WBj8a2L z-`lq~cTY3T@u1%9%2z}&Q8lq#fQ~ksSZ^%R7muN8+$a|94c4!NfR$4Ky@!5i0kov> zFYP9Lvx5X?KnDjuMshyxeEI|t?dgdnrgkD&X_a?Hj#oub3&Su74hO$56R1@3p6qL3 z6Vp5nrEdwMXXk=TxFS9vlGOd^9PX-v-$z(RCUiEqjt{e`H{I~&qBD2s9wbGFf69A0 z>m0;8)NY>hH$uoR{DYX4p;oNO+Vcj9aQ9BuGga-s>(

seNqDJr&8hl0SwrrlIAg z`*}YP>GlPabL0+s4HJwNL7cfiGTfheqt!=l0hG)QHsG?G~ak-A;G0U9U@xN8O}t z;SB*V(auh7E8u?DvQfiPHyc}Up^U!{_oLQ>8j-qa-a?dP-YMd?B{Vf4b?vtW*azJ2 z{pyK+UkOs5 literal 0 HcmV?d00001 diff --git a/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1.jar b/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-1-trino/transport-udf-1.jar new file mode 100644 index 0000000000000000000000000000000000000000..805b5ce80f0763413d6496a9f363f146b39b6f9c GIT binary patch literal 9956 zcmb7K1y~eX(_TRkVQG*qQA#?bkrr6GK~R?NC6-2!S`YzYX^@m9r9(<;=@gI-B?YOK zP!N%Sz4yC%<$l-C|Ln8z>@zd(*>iTzJ7;DzL6{eD0RS8vz$Ht#J^(#1zP+HwW%N@` zL05)LSzVC_6QKE%U}XTt4h>p>3jHMeyP%qky0W5zjxM*F;=bBIzbc4}djucE#W6H6 zT&2zXTHx*K(kjL|V`%;FsFZ&<26Y2FySqT#pj;l7*6#n9!~w;i@B&)C;UWNV{TF&p zR#31d7z*aLcZWFFz=tE+d5OYyr4-hUJ_G!UCuC(Ng1t!y`BF?P#na<0uTCsA&Q6>L zlM#-7gadm~r+e26ZDi$T0*=YFDY%AA;hW=cD7f%mokdXEX=Y;XEbF zltYzl#QtCCaa|Oz8`}hu(SSz2P^=m4j&ee1@E8-fcLJ;@_b}*7R_=Cz!k=PeXwyXF zASoR{nE@dTt=Va1`+<;H5!T}I)5^H!_msOIu9;&z8DLb;M@ch$I>3_~)k|BZU|UE9 zoi#=>eGs(7O%jZ^jbCBc`%LwOrK+}J;TH>wUL$}7#OWW4i2nNf;^Y5E#?j8%!OGIk z`Io{J=Y_xT)-Tl<|6R?>8|>ubX!XyHzDlMAroaRM<}U#NQoo@uYv&Ah^U*N3x3Ylh z*!fy1dN^A^?I6zF7LH(d_jKJm8l*C0U-Lqp?I=2fycO_ps~JnZMy}$T*C=DMFh2Uw z2HdWWcJ@l!aAf|<;osJe`C*#DN@CWuyfC4uc4ftVWe zm1ZJF@zI8#?8RlBKG3D9Yb(6y*mf$-Z6PGQbRa%$=55GXFVvqnbV(~Q8bR(w?O|XC z@?WWG;G&)=g4WHcYoJ|UseEaM z7kx!9yC<}MY%K%aOsD~*D3Yak+K_+GtZm`Eg?8RJN*k)O-0{q)931BXHBUrJA6M%Eak) zq`t=GyUeq9eD5g1UM`p4tzVM6SF#G>K8!thP}JzpIYVgHUqYdvGbO`F=4L*2sH?n! zc+*aLW+Dj=sndH{LqaVK*Kf1|k*|ZQ zhxXx~ z&!j;rE?l@_tCjqrQuh*#7)(9pi?}$ACrrX3opee|WJ}iU&QcC+GMJY1De0|dU?K&| zB94-_E4<2(PnTc49&3ceAQo~V5uR6!6g^#b@TP2x<#|=!7;d^;v^WsL1gKj)<=mfT zz1G4A4cVHSle)V9hRABFyR}Uis8~LkEi^MqO^C`)Y!M6FFCpT7d-Zf*4-&%Esa;XV zE9Nm&+ewD!!D5^Yf_hSI@YaTEUQdniQcl_O8R?QHK?}DSzh#W zP5!cw*Rx>$u(h=S#(KgkovfGCa@o5|-k*w-ou^aY{CvR644RJYp{d{^4getW8!ji~ z<_7jraI|u=a)!!4e|ESUJ=1RvCpm`ESdEp9!BIeVD~l_XTweMyB?eqj&{M!Y6B zRPH$XPBgoC9W^Tp^ycU+t#NxHZaJ%Z76?~%l^b%3CW2K?O4vZNn%plO^t$z6B5V5N zWI`659>*t1dF3G``mY;olH#FT5?s2(?~rQFv1`&|Iua|l=XmQ)`DznxY~x-COMU2& zqwi)lzRMr^DlEwK@b!%y_(PJN+UHXyKBbHfy#hiS%42MOsaH#O+%F=Juv$kuSCjS+ zJ!&P!J(_mb%>+XK_05{YX~da3b9OrSJ>0{n9g*=I5fbZpUMH^F zZEt7g=b&5NiZkzqs?U^2*X3@>F??e3%ZZ*|jsQc)$xR-SX&5!7x=AFs5oo_pkHgKX zvBQ>JM+ox+Q=darkYg7!0%^;mE+(SpAO(WC$PC7pji}qnt!&!B{WE=WHh1LwE#&Kj zp-galw*hUwIc{)6qJ<8xnwGX=TcyP6q*RNhW|8P6;GK=Upo*FRrMfSjDWAs(vfsaB zPDssSq+q)-7K$lt*IDnVN>@HSw10;)Gi> z9g{R)W-qJ}49+`Q$T%b<+d2d;SOqu8th04y^v+#|!bqXFUva?kn@?eVSE+h2euzJSTL~(^LcqAq5gOhG7w04fJPn%BoPA~oJON*a^b>%Jq z0K8~_68Q~ZQulDOaW>0%S@uUIq@`enpi^w++yqLXl)MOl7%1Dc_dWJXJHFG*#)^WP4HN zSKEoX^ggNsWKXVy)@ORe=$ujZ{(xck2d!MAAkln1TRkZEYCc{=&8NZFfc>7L=c-hK z-FfB21bRLeXD>eH3JzrS(S@h>Vm_rSVFATuvm@py*>-v^cZuXj)vaD12w?*>SvvOn)y3u792%$oQM)ohXBZ1y<#A(i%}QOFJ0YTLl56WF)3lh*=I z&;3rI@6i1PGL<9oQO|l@e;J=XiiktG2OUw;%H0M1wO3~sZ{ff~bsk0yO)r+Jao^vV zNf=b=)?n7O5i?IdQX6yXLY0uzbOlOTm%l94M@_%}`lSOfbxS^LF*e#m!kX=n zp*Ffd+05`H|3U8O&D~3U@t212<)R}JDU0TGGtLsT4eQE%zsjV>qdc_d6@TX zJ_p?(&WE+Jy~d`!^q6Vpqr@yRwo_ULsNFTBFw(e+s=x&oX4)5nBA}!WEvIZ%aTA@xj{sT$aB z+c3sytmB!ssz~zoW zKwc$oNGj$f<%;Xoi$u%Q_Nq0wv%Pz8&K+Wi=@zQd?r`Tf!|WgDlsv@4-0@tPUGrC% zjhNT2HQ@+~C@7HTX|;d(FfoQGjpbTYGGh~Aqa58F`ph!QA(T1}JHYC^v|JU?9$+pp zRyq?8GA6B>78S5**-aC$8GgEZtHmD+MWivRwCqWZtP{Y%^t#RaB$VvRSPv5gD2qilwBNHMe0Gvet88?P?Gl(Fdvvc#yfv-1vM04-< zUO|qfZVPi7_?Cq-OMrPBb2D1RjS>LCTYy9+SH|E(b2fqwmD-SPKLVtxX#rOmRS|{6Tl{SKa^_8X z127|-+O$ctXU~&5a*hpSp=s?7y05`zh9X#oyU$H`#_hVLeGm5(>#K%63x?*x!Fw-M zg`*Zt1E^SL?>!JIZ`rcHhJ5n4Jlj1hyD`Du?(~eT9q=m3z0HOUYQUcF^@X=wLGOJd zpWofVvh@IqcKNQ5P(PNPXO?dRm6JXZkHg2XX}lfH$sK^l&C54kFUJ8L&U~o~0VgMN zKt$U1coL=+$uL`W{akZ)6=?bdOdLR?&A)#nZUOyP=elacrkDSctxmg z=fuOnP|)d!o@$c3wz8HwN^^t0fQMaf;Fj6ckjH$(_NiQZf`*?M(?Paz7j4aZK}yAf z3f3O&&Q~HTHy(42Ewr{suk#dF_<{N7D`1>BJ<5tMFsEK zCi6J!rWO7A(`3KInXtZOfBPEBox~z%WHr~j`U0b9cFCRU)KXTp50Vf;rxE`BMC8lU zpf}Ye97#Fr;(B)V>WL=dlFkDDaZ>%4AjZ~FAm>CYE|<7L;6p1ja^JlJ!-MAnlnEbu z^;c60RA9$}Pp{uN(Ox;7H(lh*RT(~_7pChM+Gn-Lu*@T!&pJp6NTK&Qr7%jg zIm?>2JUR@!S-!Uw`zF5QVl1o-=pV(O;Qz!=rr3V!W;rJHaE0^-)9ooy za03`(^ZBbiCI#PM`QEKdDXEn$seGJ3d2Xcq6Do^ve-J4hqRTQ_Y-d@=o$_5q_ot0$ zWg9Y+ceUHtxu&XRnO0;7SFML%j2;q4sPvNi=rz#$4Cf9z?VM#<`n}{~oY&IR(9F3o z3p|Dnn_xFyg0Q*Fdu-P7ZqzJ_U2{yM?lgf2`WHI0d8824?{Fheq21d=CW+uyDSu1X zgs(P=GIXE-J!)lpKVw!~8@T1QW%pabr}8|+s-vm4H@ahT#&H03582_Wht{@VShK^F zb_QBxLcq)q_5mZv{4(O%NbYX!iX+gRev#jYjiW`ks}0W))w)a9^YWJ2o$QS6DQ#Q| zzvYu%!SMBXSs`-b%{zh^LvKv3AcGZ=8TB>E^=fs^wYYdM+$6RHaut@_mC`3+$#$%% z&#Db@gyeA21-)FTb{X?XfN=(p-6dQ^8JM)H0~8`rsufa9ws~JHQ)UZ?9e;AYJDN9p zQOqI$yau`Ya1B?uNA~{bFeb<&*HaaklyS7EtmRuYwv8j`g}1YZf#EYfbcv zG4et}lv1Us9fD9&^8_9)jkAt6kED(=|D@Hk)>{@5jixp1Nt|7R=>qN_#os&?-A_OM zy5DaA&_Hnp19z-=BTtxLeK5ONl?<|w3dV>KQ)x@WqI)cs2;~&Woj@>;V+ZTA?FaxR zz0!if&spd`v_phLK1B#S?ycCDc5dI_={2Jh4SVe{ZDm+&{L$jLur|JiXl}KI-V!IcQ4WrEiY;xFQzV zi@D)JjTH+^r+rB046QiejU8JEb zhd&LE9<4ApLzC)0pZiT0tT^C}vkC8O4c;%>HJkmdC@XVy%?sQ>^$X(m#qFW+V?0ET zJsC~2qByXsB53_8r-sY*AZC8fE1ukAUEt0;rL&cx%44;gF%?c=cK62lk9w$4EVbSk z7edPSq0QXI!UUHMM7KAt69v}jRMhEML3tyj7Y62EgTgSI0bjKBJI9RL={*;ETX^lkoH4Y zB%-3S*WRFpx*l~w9YrYOwixeF5OmQ!OT_)%Z4^bp(ujaDjRL>9X##TS*^8E)3`m3D zr^%KFpQ&0gq@pY^Ce3@g%PG5Wuyvq{NM2sKO#-s_mJGCd9{K7$ehmi;FHD%!8?Ttp z7k+2eGZf9kC+*=)PBoO= zXGwA&sL;H-_aZmvhHs=hHJrf;IoFopju8YiDzW*95M02H}y89|=IFBuLpujnZr zJ^Z*A(dfPDGR#X#q*U6${er5Qa=obQxb?vah&!E`R9@!|erFq}ul~j&Fz}(+ zl|IHhA|5{Cj7r^|yt(^_0d6(I?W9({#a<9po_Y)0LY_RLU$Uprqrn8$%iSAct0!4p zm8BBz=!>&s>F7VD|73m!S<@ZvGsf)I*+ zLD&U14qK`<3|p$b8@i1x)pkX6qP2w1t*g9l>oT`Ogg85w`gn=yVnIo{f}99Ev4%`T zw4o3&mmbfWDr?LkZ!auxp-?bJ5w}%bHWKcTDd4D zI9*$y;$Aai3(_+vO5hKfqk~034Ipz<&2qlS50vw9FbwsstBxD(*vF+Dw37JdYGn}S zDoGWIk$-{E$}!EqS-j-CeX7VJhJByl2Ifi6vndOqx4f>;T9A`=D>c+e6I5bCp;^QO zTo&m}L0{P-+w}luMF$Q^>fu*IxhW$O@%M2)(s9X(fR@XzsK2F+QoMaZW6>^^Dx<9` z7ty*eSnO3{$wu{+IQB&HNWkgMTMr^ zcS&dGF1=DoxgM2hYF0&4^X~-tKd4#J5d!{I7U*TjCW? zceEL@>t+Bw(Ul82Gf%5tJ#Dzm^^NI1+2wv`G+7~3e(AS7Xd?=rY372l7N z^91S39hG?_nbazFjOM17n}Nd>ZqhFk>0sjxt2L^(8z3QEraH%S+trfyQkJ+NHFP6< zk5rCEXUexzmtFPnOhC%RB_yFDD+I_(%Mt_&iFt$M!d|+Qb*Z#QDg;(NTcDO_lzmBj zeV!q!^*52aIt5%W=n9hN-f?Azv@oY}8wB#O$tgV3z4bP14j2y|XxIv=4Gtg%y5|EN z=BtqNET-$A7e=OKp1acouTv=aBG0lnge<%>*&5TXkzN+}|H z$JV>Z0&AEuGFNJ|F31&@5a1}y(PS9_%XT|{%}Z8bEN4)UkT|20>#ZwyRL_bZ7o1kK zDD2`*>j?+<*vBq`VDldBRXj0j@!fn6KNEkQ#0l`A#xduRccf`k`ke} zi@^)Ks(fWa0i6O8lx(#;#wiT7$fdgStj0AmkWA|1NR$H9tAT~jw5~DY(fY|XNm9!; z7yLHPPfd8uG{zqw7V1<8)e~o{U|BoJk3m9aW|K59m zwuBx;fNhy?@83HSG(kVBo$o~WCj4#W2C}1HzpMSNFX6m}Z_?-k;E(aG{ov2>4cHU? z@w_m%i1KI30D{@6od{I?0I}HRuPDK043zFAM!0>2LJ&Q&Yovl=G#H zA1J$Mm-~H`e-t{-1D#JA{{Z@m{s;a((0@xIpT|0%ko$o}K>WY4ewm&-4|V>={sYPo zeK7w1&-&;6{XEk78_^FWbM$HSzmWdresmt>e3bJ8WS{nbgZ%AJ_s^K;?@<4ZL;i&N q;~Z!F7jign;yk1Ln61#infQ}mG(lKsZv_BI(GOkpCttKU0Puekl4p1T literal 0 HcmV?d00001 diff --git a/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2-trino-dist-thin.jar b/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2-trino-dist-thin.jar new file mode 100644 index 0000000000000000000000000000000000000000..a98746301414beed61e1db6de735c401c8a899e3 GIT binary patch literal 4848 zcmdT|dpJ~U7as;CL=BC*(kZu+FfO5TnPJSBaVr&KjKLH$V+>M+(V<*ZQl(pMVCdBlCtEN8zr@M~tnp-z?+Cy35%|}~ok%)Pa3+#g zbcV7!)2KL-7lllvtq9xva~Rv8>=zbk>la-EEfa-Tx@l!VX|?#i z@Poz`wA6R5W9N(@fkFEk8BvcDh3IP9HBIqy-?G1UJ4suKcS2-BRkfJR}ZVIxriV&hXTg5V79L7_*N` zuP{2Sq@1PGS!JeDc0fHmQFWk|!kdEl-HeVzl#0HWYBLBKpu{u%61b;oXUq6S^}Xu3 zD@?EAuihNfQjhO3pH+XK`-+$bMa>4+aTz!Le&df`x!is=Jx%NYEhX^&#fQ=jMxWdt zTFEfEhyF62MHhvYfBHZZX+G49M%!uj?G7L2>aLL4?5ig1TG+2@-DUJ_Lgk)#KEuSq zYllXM>M34ZMyvP9q?=X4ss#_khpzX}JSrIUr>F8index(@hfr!e35`UF@QRVRjXq~ zqk7|MhExKMMlkTh;5-QIGPPi;_Y2J*Ja52Z@mWWl92a7yOBV-+LSE9Hbd{M`t}#i) z?os#S(@fOgarn!>zGMA5J65lGvibBrwWPL!yMmw`#@#|h`#a}R9y?u?-}r2l6o+H2 zpAxX@k(?U7bw?hjO)4Q^<9ab}^*lvy)f1|D^ov5h$34-+wMuS6q zdMZ9c1ll<|XEQ%>q(k1RG066jq*hd^#9VP4W%uYF2%k57YGHQ4v*B`eb9=(xt#15f zj+t61s@}+);5_xP&djPaifMy~i*n!0q@*YG4mZ}?(;IhoHF>U0xm5gWkZbx-Z6h9> zrcOIg_M~BD6xLg)@gZB2`w|Z)oRio-p6+;b|Bb)aQ;YZGIL2Xc7`miI;L)`!^cJq$ zHBNkHUeKbbRP>0}6Bq+MJYlz2U)odq~>fDeRGndmK%mMVt$aL5M-F|!3 znY<0DGv%GqCJD3n{1eF|kE#B3Tj>|CY=OoxH*}K@>)thR^uEq*4@opkl26dqm2%B7 zaeR}u*K*=YhD~hRwo0W+c_|y3&)Ey7Iie=l)VJ_wi$&h$f702}tx0o`f?dCwtG(&C z+jvE!xBH;(oLE8auzJA*$@uI2pSv7d=6E&Kx_bndXVwYB=u<}jP2EEg{HzEx_Jpd! zaPAfg5OKzFSXk?x!fTgyL|Al3ghq^VKR&Ue0)7qUYgLX|Fvb3oJi~E1&eNw)6k<>q zMs%I+zueo}`*BV#2=9LQkQ3uWb*zNR2YLTIyUa3i;|z_PCx$cSh43Hg#v`w?<xh-J zVGhkR{FfE--+`CQL)a=)&&s8XKrR`tI+vCgJ|xMRaDrVZGb~W~0+dWLg5*!ze?Vw6 zYBF=rIhml(LWN&@5~23LdK9}0%q!`|y1f#7Yb=(NExzl(v`U5AT>m}4q?oo|Uyw25 zPGL>^pY$S~3xDX(Se7BYV7y=tyWK;tDZgwdwT;%RdF(9Rkb@m*gmz)Fbhm;DAzK`( zRM+wUYM7U$?NSd*xgRBKx@lsjpt-%dUFArA5Z7m=z!Nw8MAw-OAT+_PhGU3%l`SdW zM=&o8;$IH#1HVvA(x^AmdCP5hLn*%WY%Mrr*C$7#_>-D%XL8$aM^2R_YWh;E+gp>p zJcl+JjvY)a+rpFca>rK+s7g!wZ`?_C4@v~sX-qH^*yIz zi)P_vF{;|3UA*~-j;j(iN||S#h{Ece1iZ_q4%H!^y6zi8F-lsdurUUf1@o_a4bqDi}&E0MB z5D{yE#4+2b^x_}{QhME-ogcLSy=AAolV!-W5N?XNr_Q^phxK2K9l?VI=<{Up;T=u! zZc;hbc&-)>!9EqIEdZXZD(c6T?>XWerm}dv1q^{(W_g7BwP+^9iJg>JI!fwNE zhuyrE5q?`3kWFeQ6UBJw(m%3!{92LU<&ihiLpxr~SU5C{^WIVIo`x=$4Nz*S7v2c0 zmpX7S0@a6z|3TL&nS9uT26*}cV(b180rb<#UwUQbrS=zC0qv)<70vm8vx^NCN1+g? z+0nb#0c8_@O7NmiZsF!z&#p5B8+KK1SHr1RwJ@J!Ciw5j^q+cx@=^FRRH!d$W3TM7 z^*Tcr%wrU8l|Cu)t5B?`jPViBQI+tQ7Z;E!R!P=>zTW0~;16%1g~=CMjxy>yazo#a zM1db<78H6;BJ*B$onk99|Nn2p!6^*>`9uW>1eg$rsn7adQr8@{9L7Fhv7A|!6|4Z< zmtfyaTkH{7-azXAW2~WS(OASjz>+m$t%P8WSc}F&Bm+wp;>#vl5y~347L5h11D4C6 zOZ#udAe;XUvG}Rxe;#CM7+dLWSi{(&u^wqJkSITSq@`$!qZxZl_R;K1)U4n$tB(1T zL5-b?ea`qwB@15d<1asazn?nTIoaFgubgeb4qDwOTlwfHyiP z*!!2SR1#}fOSSxi{-B#-CuKi`zmhfqf3K|Wi>~HKX6I$`d`W)RDY9s+%}?k1rhF`& pDeQRmD*PovR&dD=@xYs`@`U+cGI0ZYh*dy@K;hgVkoi9~?Qb*k<{$t7 literal 0 HcmV?d00001 diff --git a/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2.jar b/transportable-udfs-trino-plugin/transport-udf-repo/transport-udf-2-trino/transport-udf-2.jar new file mode 100644 index 0000000000000000000000000000000000000000..3a2be2c8985d0656f237c8e43e5d65b96742d5a7 GIT binary patch literal 9695 zcma)CWmsIvwnaklV2!)GI|&4L3(~mLc;hZXLV(~9T!Xtya19pR-642z59EX>M4*|FjZ@_;C$_dF!iHj+#Fv*FZ$_@0($}lpGBFiw+4Gj!eDX~njuWYVu zLOlX|$p6WT^WR{O_6F7twl?;Tj84WT4u6?MG4ZIu=tKO(69@>(f5-!a91V;O91WN( z9Bizcl?M?zSddz70idl4*rVL{?TRjm?r10mVJ!Q@X6%co%ZpAlY(g()+{}f?UhB0` zG|S^4TD-1qk2!drUSa9&{Az3xVVK4caw_MIaKB{Cu78sX8m?mPSOA%oAP@>1^S%Hv z7#V7{X;9EmsmM>d#P0~+;++k))Gm>v$`hvBwXbF01WzG&&n~d;^G%Ue4XYQl&jxVJ z4rnX{qOT^|tTYRGxV&+OB74d*kr+w8_lLG9x-B2qPQtU)JrvV~~H3u`;)|1R0xK|6?%rA*xjf4Pd?ztSi>{1?J|9-255V6L+PZ zXhFx(0>!RtKXZJ`!MV6q>t4nXY7v+`hLUJUxf+ybSxe!ws0r81Z@NV^inOXf((O^2 zbB1->k287X8jYXC7Os}eoaefzHB_3o254~<)t)7l3Z;q^I2x#Z?bjY>FVSwda`itW zmVygSMNP0dU^G{A6Pm6rx_;X>*N%IH zXAUtm@X1(A1v8^>SZOZZ*n0$VveO+FzmGa zr^gtr(oD1Z7Pp-x-YU5ug4?kZUMOd)D;QMy3?7OZX&E#|?*kzFRC@|E#l!oPj2tQ% zvi3KD7B#tf!ICvXzCl%tw(OHr(^DUvX|Yv=;0O!(G@QC>1LEYJN-~TX58P>l=4cuN z&O%o>(bXkW_q%jToLfwe<2ro0m8S1)2|RLh=UftM;=}ocLm26_hUa_Dw-|U%X+FF& zTIm$TC_L!xPC6brCEkSfB(?H(_*Q@*ERCyC0B1Zrf+*vfs)qs(r@g_PDV?%R%UU{1 z#naNE6JE8lknHyKm>e5Tdvj)s_}RK@xVsEQ8EKz};tqa{(#l6D+hxTK?6Vg;f)TYD zw@_!vEd!is71vLgWS?rf2V|zwMT69fATWx zdzaT!TDZCWoXIDcG&S$AI24P3=Ek0SGb&Kf5wzp|yoW{i@OQ8{LHbT>5O%)M+fbJ8 zVR@*J_{HmS0IuPu+?lhZ+mKA6M_1`DD-}GK%Wj^=7Q*wr?*f$FG4=3@3-Z1fifP1A z$g9h2C;xa7qbQ1U(rVr$7ijVgQS810gj|KUbp0OhNlhxnS&qE0sRHF$v+@U#)RFr4 zBZwc}k}sjRk#aCI?h@Z^;k*uxl7~5%ogkhGe}c;zh<~5*9s5@m2JzQ6+Wh!edXQJa zf1zH=j`mJQjw1FT14odso4f%S^s9^w0<|3GMFICy@iP^w=nJ`#SelUFU9B|^f8q5N zU3*_tZdwXXmkx0t0j9=@WRd?j`UF~VCv+z}^NV8_oFQpxi&*mSTPZff zY5a}e-|qHoAWSkqECCjHr{ABK!!Q_E1mzguG4_ASVz9qb&jfA+ zDpabA@rS{h7w42SVG`vYnIojM&d&-vt=^wi{->QoTn z#eJESh~ati2Zcqms?V#NWFsr=PjRK8k1}GARqz+iB)Oxt(pRF!6Wh%7O{h57H*N+p z9pK|wg*6;wntW$NbfU!;Cke911ChMN=lXi*Pl+L6;i%~^JKYBf;k~NsUU-;@UA5*M zL1^jiA;yI)S3h_1Qs90=(z`!j&Mz?@kF3E^mZtM0&v^BfEqpw#lE=cRbA4wUaA#T4 zK)IbW@+zF8g%XC-Iv}irce2kI} z7$h$l2QOP|yvjk>vmVb$vh8a+kxiFvoj3&xpNB63yIM+adb(9Fkt>RVbrx+^3Z!f5 zQQaf}F^VP$9@im$2wv3-MhOIF>G`5Ep8d62xw;t!^R@Zxayl5XvCi`tCzbCn5kv-- z3~Wzzu)A(9j5<{HT&mI$X`J@X;$!Gy4F`y{nT{fm8Vm9f8vuuRJskS^ZmFT#r?EF z5jBfA)y!R@dm|%PK+e7pWuir>3gJBT#`qX_W>{cf&LYrr)lEAC2oF(*5bG}9y-XFO z((=~-yIfzFz1gxG>Cg%)54Q~U>Cea|V1y;@ zRa}K50<)is-bHih^7#wrbl9Y?Yahe&digORqZ~^UIP@JGL(mc%QiMNz6Ds(RWA``v z0cyrYcc{Q4NP~H_!d0%=hpPf&T4O~p;SoEGuQqbOjWCRw$lq?nOL_1$VNqU*n(cTxUKhoRG#KRWQ~*y22q^C-0kZ;nNLm$A0g` zwQ&z&p9{tgj}O0Gx|T^fUgu3_3=mvjlV!e0xoo;^BHp^2yE`u<_J(44V-FCi!xf_P zo29WfVDgK{+>51=H`HY;yD)?d;m_9H>msS#y=ur@+U`PM3MIi75(u3~Xe8*C#&PLz zl_9AF#@4LK%VA4y_5(R}SO+|A0t|8qwUW-0K+nxJ*vFoco;$YBD$axoUKShR4W$UO zL@~hoV-+J8ZV%(Ota7$8+*cR$$BbrYfpII9QCU zDF&;CT$Z@a$2~u{>STBx02=dU*J`LTp28aOgscFtCfw`pqc1049RaX%v5KYbzf%Z*e zEU2f_g^kXC^yXe;rBXpCY3w6NY^6Uumn)k*HUYmY)Z2IrB}OWfxMQV?0yR?YJ}c%~ zk|O#z19@DAPnitI!8s;@MvKH;Nde1}$vLZP1if)$0XmNPD#Px1C3pfrd-RrBCS<`& ztRLnEAua@PRf17lQBm-rC}ehLHZMxGRVZyQNJST`L{oQpieApft<0?5Vg|%-na}m| zHv1w~g)wWDBW)ETdRLV!et}x1j|BlQSe-Hi6xACRAC1Sf79D`TG znVctQ71}d@mE5z~06D!Ji$k&)F(vzKj*4I1;j5Nqsq;|4D2!HoI8nS^OSl#?wKcdK zK8PFgJ%dPV#zds3^h)>lRHGD;`&~i(HT8n+;yTJc0#1ZFkcJi}zg(q7;sPCJ^L}=G z;bcJ8oXnsES78lpd9)d>EB&6!yMBT%93Jg~z{x(VG2#Aw87e;KG2b1RQS3;r&(=fD zhw*LNYezF*&1&v$u?z9j_r6X?u$Sc(EFzm$tQ+&@f5BPj>?MH@rH%_{2l;)ofG-vZ z(@^VhyO%-|(oawjtRelbLY6%{shz$_JE&uOD9!J86(y!VMD9hAjDcMpcS=}MjRw%V5 z33uC_ip4%!gK|M2-08dh3~HZ4w7W}jYPcsbjkbxtl64jmNttPyVme>&HxVTQOjoe| zX`nZ{FGl+J8}_ER(Hl!k4pA51V71vFX`1m&eDZuV`mSPigy;qe=kkI-1#Yb<()H-_ z5To&{PK@#v>KDH7*8%#G^*CLC0{U;vDQm>5s?&YANv(oC-9d=75*LeuXwP@VsU&8l zbWLIsXg|Xy;(b7AZ-2>(+O2vVh+m#aPPVj6bc`<{{;i8#6n0L`h7gM|!IYuQEpxh0 zRbyKw>@Kw3P5=eXQf*Ar(JSrFM~!QGCv{Hm2KLuFtW)Q!s_}z#@%;NPQU>Iv408On z{QB8N`pA-u<4>q)7FB3U3X_nzW@M-^WJ>zXDNeYzi`c_K_sLEfchJ4;(i?%*+tr-I zBi8o1aJiz_?(cUAprwE`a_!~4-QTx4^tw10IQ0I0gYRR`107_*9;Jh8H~m6s}`mF#Ks2bz<1&7sAbxDjMEvP<+9v&Dk0Qu9`6vp)s)#py1pPVgp3esSJx8JOTNO6k+!jEKA^ zhxd)bfT{}Is;ri2`d~Smv#+ghTJkm3LUQESWmX*xV%j>9vD5xj z3kg%-Q6^uGoC}bKesfkq=2>ccDNE*g{v3%pbX&U}}iZKO76DTc&Z_sqQGN*>7puhEs-3A2>F z+dLB=YZJlHTeYmru1!{#Pr?!F_{4oR^+_%iXNKdLFvt_Pld4vc<<*n-)!ON+zZG1{S;Q3zM^-(7eJ>IUB zNTMO6_8n_B4At<9NIosI&y8(YiP4{cyR{HJ=i$_I=vy%xL2zqY_9;plAW0xWvQ~XD zg04w{Qh`T<_t`fLINo`vj~H^$V6bJpy%4lxh6pzmKGQNsU-Y~1gK@TgYiCMHd>Az; z1(@N;2i@90{h?d9Aw+aY9d)C29My)y9CDo=uSemuE!@0%$9ydf${UNb0Nt9O^s?h{ zj3lOjljOtR-oz)Y<#!@`MF5H@wn}w?WV*o-c*W^8%_kDux2=(1ORRTdkgmmYZMh)IxrE=sEY+dbd zk|Ls(Tj_B9kYif@QAeyvN^Cd^nm`SbL%}-eHg;l$U6=G6U~nJ$l#KLS=>7U_N*;+> zRRqvT<-=t6vkYHNIm)OLuT>m)q!+oe#FxcoH`Hg`ET|oy)uCVcK9%*IJbpR^?-^l$ zz6F!E-cho2W@`hDE#mrZn0SY}<#5{Tjpigys+5u6^y~ONB=!%fQW^1iaP=73W=Fjx z43v%^hEH3i9vaBpZ-{U2J5-^^pZ?xJB5mhf_{gW@ja)mmG8H9CKY$s@8%2uK? zkk`{cf^$K&(1TZ7*y$;~dTOqY9AkR^dLDaS1T5YZM=^c!aOF(z+9^Yee!}c5#=;H* znE0Gmq+s;IJ&t_A#!aFw?hMSm?a@-b-%?JI_ynlz90PWO&GH*Y)Rl9)_ zzH1%$auIH^7LX>x-7vLT`Sn8Ny7%IMD1N~vxYV+LWNU-p1`A(dA9@}k$F?{CjzTCG z77K}60k`;*X^(Sbux8z&1>@>0!e*imaSo%6&8ZtX z4T8&-S6~<3mC2y=b2g9IoFZr3EZ~f{h7_7Q4y-yoy@-6Rr7_{;2|s7q;$kMCZHDR8 z6tTyLcSRQ>N2ukRy{@V~dYq)$(#@^G;bgVWn!{I`nCdKATjfzr4j}AYOjs%pYhnB* zf@Vw3#d+abX4)$<92CsV^6c#b_l5v=%c|1#6FvkM+`{jM(xWkn45LV~4sVmbys12N z*PUL)0(+#%blACSX`bsc$5>1hW^_lMT;S9WWYE^s-1_eq=5g8~E3CvSc||1^@p6<0 zf0=o9HQ(F78bxaNX-Bq27>V=rteCK$?*?;RfuI}OWRl=cJjG@WlbS2H@R=D-fpnI3 z@_Ap6G~($QW-HQ7HmTUl35c@@A?b4nsj#yN(drf<>JSQYw~RCN{hr=^x)S9mwg_(X zNh4_OPAv-DrS9j(Ic@zElgrhfbm{lFiKgUNMIx?@?XCE;IN8ZP9(|N_eUxL@h*?Zu z0=2VY)jm)ud>|almDFX9)iW$qardvSyx0L@hV-yVAPsT| zKqN$aIapGrq7l;Emq|I8NLw(`na<9Q0hLW|u8Td>*^a})WhgYs>FHVQtL^6T5t+*! zs*mW~=;q@TRXHaehy&prfihIdEP2t{%&*#MuG`apl@w}153;W?5D-((ARt8lg`~)X z92`N$|J=-#sLeZIs1n@Ea}8RLfob0Yp;bicDVtHh8fwwa;L*fjnVC0FgDV;5tt$bGpg0C zfN1ZC&0Dt5HHXMDqc7o$&DeRb;8qoG3x7ZB761p~OWl$iZf3HQ%MBj%HS6k_McEIAaYR zWr@gwotuDlv?H{P2kI9}%qYg$M!3=LxTu*p52l ztURlv1CW!J?o7za*@CB)j=p)PDH9+@D55Ic$22_yN12>aV`!?#oDC))&O=&nA;8NX z)oX%@UVX1QV`vf}GW;yp`SLm<$J^Ljj^U$#^=9BX%%Wp6i8r|sa8!Pe_KnMB+?-D1 zdzau~#+lC}RFbsv#po`k)aW{2jme@UdwYv`P?GWX1;Rg*yv_`11y&$7ssdK>XWnIp zW(LDB6uJ_yN*_V%V;*=a)7s?EgeC+dsw>%{l@&g0h_O?6s#OPb*9k7zD5`3AB&+Dr zEp^P(EwzI%57&Z&FOLK|15$!;sOJiBgJ)x^%<4Vm-Y#?M?SAixB8A!NF_-)p!ERJl=Q>iKLp&URb=P zgK6J8jN-2;Gra<;?h9sL296__#8vgW`52sh7w9~REda#5nIa_CjE#tn>pP+-5e|a0 z!?>dZxgw!MrHJh0S^A7;GUv`4?nSb&M#u;xG^ zE<~<#zfA$aUfP5YOoG-EY|y1_t`O$ikaB^Shpj~5_n^0Yw*+YBhnqxDp8q$HVFzxG5A|EhCvEUj&>b*#fwtowE&N;x+X-c-h&*g-RH6!X2cPobE>cRF zJ&i2IkLsIGCk6en3Ip~2GmXO>QO-5$%0-zS0#xtqm`1Z`)?Ur~HWS?8^2Y5IN%?GWA*!`XK9ktW;O^>K!;Yj|GfH88h9;)o}U3 zfaDqVk`I-pooNR?(=OCBBc3m=ADwYL{Du5HX~ul`ZDMW((zCL$v2?Q4Gx_}x_Akf6 z!NuklNDsjeWi7-H|G@uDa`?}gF#87E8rv7PRkr{51t6gS(0}h!ctCm>s1OG~YUMv& z3yLzorag8o{0ROrvVCbEK7X1W{>#Jg7~x0s9|nmZR)ycj4`SzspFa_Po{RrxTzHJ~ z*tqZqPPT99KcM_#V|Wbt1LBW~J#2XVK7LHB1M)W)z&~aDZ~^=_eq?<4J>#b*;Ga@| zcmn=R#dw&*56i*N)SoNC{7)bxe_;IQ9!mc`$WOn(W3TX?W7{eN`-ru==(_=Dk(hwz6a{x*I* sQVb-2qjirV9t-3jAlbeKe+TiKTq?@IJpB=j`tTxuDD!h4!XY634=ER}4gdfE literal 0 HcmV?d00001 From a414767d8f5fa3e0db67669e41ce19260a46603a Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Tue, 25 Apr 2023 16:52:12 -0700 Subject: [PATCH 12/18] address comments --- transportable-udfs-trino-plugin/build.gradle | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/transportable-udfs-trino-plugin/build.gradle b/transportable-udfs-trino-plugin/build.gradle index 92f3741f..1e27568d 100644 --- a/transportable-udfs-trino-plugin/build.gradle +++ b/transportable-udfs-trino-plugin/build.gradle @@ -8,12 +8,8 @@ java { dependencies { implementation project(':transportable-udfs-api') implementation project(":transportable-udfs-trino") - implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') { - exclude 'group': 'com.google.collections', 'module': 'google-collections' - } - implementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version', classifier: 'tests') { - exclude 'group': 'com.google.collections', 'module': 'google-collections' - } + implementation (group:'io.trino', name: 'trino-plugin-toolkit', version: project.ext.'trino-version') + testImplementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') compileOnly(group:'io.trino', name: 'trino-spi', version: project.ext.'trino-version') } From d5260de2b788f5f8630f644737a766933c6046ba Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Tue, 25 Apr 2023 17:24:44 -0700 Subject: [PATCH 13/18] address comments --- transportable-udfs-trino-plugin/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/transportable-udfs-trino-plugin/build.gradle b/transportable-udfs-trino-plugin/build.gradle index 1e27568d..8870147e 100644 --- a/transportable-udfs-trino-plugin/build.gradle +++ b/transportable-udfs-trino-plugin/build.gradle @@ -13,6 +13,7 @@ dependencies { compileOnly(group:'io.trino', name: 'trino-spi', version: project.ext.'trino-version') } +// packaging as a shaded jar following the guideline from Trino plugin distributions { main { contents { From 75e2e3a335c80b40d8d383f5ab7f6e73d6fb16de Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Wed, 26 Apr 2023 01:19:38 -0700 Subject: [PATCH 14/18] address comments --- .../examples/TestBinaryDuplicateFunction.java | 7 +-- .../TestBinaryObjectSizeFunction.java | 2 +- .../examples/TestFileLookupFunction.java | 5 +- .../TestNestedMapFromTwoArraysFunction.java | 47 +++++++------------ .../transport/test/AbstractStdUDFTest.java | 4 ++ .../transport/test/spi/SqlStdTester.java | 2 +- transportable-udfs-trino-plugin/build.gradle | 3 +- .../transport/trino/TransportConnector.java | 5 +- 8 files changed, 36 insertions(+), 39 deletions(-) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java index 747cef6c..2919f6ed 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java @@ -25,6 +25,7 @@ // the input byte array in Slice with an initial 32 byes capacity, while the execution of the query without where clause does not trigger // the code of VariableWidthBlockBuilder.writeByte() and create the input byte array in Slice with the actual capacity of the content. // Therefore, the outputs from both queries are different. + public class TestBinaryDuplicateFunction extends AbstractStdUDFTest { @Override protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { @@ -33,7 +34,7 @@ protected Map, List>> ge @Test public void testBinaryDuplicateASCII() { - if (!Boolean.valueOf(System.getProperty("trinoTest"))) { + if (!isTrinoTest()) { StdTester tester = getTester(); testBinaryDuplicateStringHelper(tester, "bar", "barbar"); testBinaryDuplicateStringHelper(tester, "", ""); @@ -43,7 +44,7 @@ public void testBinaryDuplicateASCII() { @Test public void testBinaryDuplicateUnicode() { - if (!Boolean.valueOf(System.getProperty("trinoTest"))) { + if (!isTrinoTest()) { StdTester tester = getTester(); testBinaryDuplicateStringHelper(tester, "こんにちは世界", "こんにちは世界こんにちは世界"); testBinaryDuplicateStringHelper(tester, "\uD83D\uDE02", "\uD83D\uDE02\uD83D\uDE02"); @@ -58,7 +59,7 @@ private void testBinaryDuplicateStringHelper(StdTester tester, String input, Str @Test public void testBinaryDuplicate() { - if (!Boolean.valueOf(System.getProperty("trinoTest"))) { + if (!isTrinoTest()) { StdTester tester = getTester(); testBinaryDuplicateHelper(tester, new byte[]{1, 2, 3}, new byte[]{1, 2, 3, 1, 2, 3}); testBinaryDuplicateHelper(tester, new byte[]{-1, -2, -3}, new byte[]{-1, -2, -3, -1, -2, -3}); diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java index 2bd2d580..7ac89276 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java @@ -33,7 +33,7 @@ protected Map, List>> ge @Test public void tesBinaryObjectSize() { - if (!Boolean.valueOf(System.getProperty("trinoTest"))) { + if (!isTrinoTest()) { StdTester tester = getTester(); ByteBuffer argTest1 = ByteBuffer.wrap("foo".getBytes()); ByteBuffer argTest2 = ByteBuffer.wrap("".getBytes()); diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestFileLookupFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestFileLookupFunction.java index 892a1dda..38963e3e 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestFileLookupFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestFileLookupFunction.java @@ -36,9 +36,12 @@ public void testFileLookup() { public void testFileLookupFailNull() { try { StdTester tester = getTester(); + // in case of Trino, the execution of a query with UDF to check a null value in a file + // does not result in a NullPointerException, but returns a null value tester.check(functionCall("file_lookup", resource("file_lookup_function/sample"), null), null, "boolean"); } catch (NullPointerException ex) { - Assert.assertFalse(Boolean.valueOf(System.getProperty("trinoTest"))); + // in case of Hive and Spark, the execution of a query with UDF to check a null value in a file results in a NullPointerException + Assert.assertFalse(isTrinoTest()); } } } diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNestedMapFromTwoArraysFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNestedMapFromTwoArraysFunction.java index c23638f3..08654178 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNestedMapFromTwoArraysFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestNestedMapFromTwoArraysFunction.java @@ -25,37 +25,24 @@ protected Map, List>> ge @Test public void testNestedMapUnionFunction() { + // in case of Trino v406, the output of the query with UDF "udf_map_from_two_arrays" is "array(array(map(...))) + // in case of Hive and Spark, the output of the query with UDF "udf_map_from_two_arrays" is "array(row(map(...))) StdTester tester = getTester(); - if (Boolean.valueOf(System.getProperty("trinoTest"))) { - tester.check( - functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")))), - array(array(map(1, "a", 2, "b"))), - "array(row(map(integer,varchar)))"); - tester.check( - functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")), row(array(11, 12), array("aa", "bb")))), - array(array(map(1, "a", 2, "b")), array(map(11, "aa", 12, "bb"))), - "array(row(map(integer,varchar)))"); - tester.check( - functionCall("nested_map_from_two_arrays", - array(row(array(array(1), array(2)), array(array("a"), array("b"))))), - array(array(map(array(1), array("a"), array(2), array("b")))), - "array(row(map(array(integer),array(varchar))))"); - } else { - tester.check( - functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")))), - array(row(map(1, "a", 2, "b"))), - "array(row(map(integer,varchar)))"); - tester.check( - functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")), row(array(11, 12), array("aa", "bb")))), - array(row(map(1, "a", 2, "b")), row(map(11, "aa", 12, "bb"))), - "array(row(map(integer,varchar)))"); - tester.check( - functionCall("nested_map_from_two_arrays", - array(row(array(array(1), array(2)), array(array("a"), array("b"))))), - array(row(map(array(1), array("a"), array(2), array("b")))), - "array(row(map(array(integer),array(varchar))))"); - } - + tester.check( + functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")))), + isTrinoTest() ? array(array(map(1, "a", 2, "b"))) : array(row(map(1, "a", 2, "b"))), + "array(row(map(integer,varchar)))"); + tester.check( + functionCall("nested_map_from_two_arrays", array(row(array(1, 2), array("a", "b")), row(array(11, 12), array("aa", "bb")))), + isTrinoTest() ? array(array(map(1, "a", 2, "b")), array(map(11, "aa", 12, "bb"))) + : array(row(map(1, "a", 2, "b")), row(map(11, "aa", 12, "bb"))), + "array(row(map(integer,varchar)))"); + tester.check( + functionCall("nested_map_from_two_arrays", + array(row(array(array(1), array(2)), array(array("a"), array("b"))))), + isTrinoTest() ? array(array(map(array(1), array("a"), array(2), array("b")))) + : array(row(map(array(1), array("a"), array(2), array("b")))), + "array(row(map(array(integer),array(varchar))))"); tester.check( functionCall("nested_map_from_two_arrays", array(row(array(1), array("a", "b")))), null, "array(row(map(integer,varchar)))"); diff --git a/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java b/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java index 7dbc3d34..4d2a7892 100644 --- a/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java +++ b/transportable-udfs-test/transportable-udfs-test-api/src/main/java/com/linkedin/transport/test/AbstractStdUDFTest.java @@ -110,6 +110,10 @@ protected static String resource(String relativeResourcePath) { return filePath; } + protected boolean isTrinoTest() { + return Boolean.valueOf(System.getProperty("trinoTest")); + } + private void validateTopLevelStdUDFClassesAndImplementations( Map, List>> topLevelStdUDFClassesAndImplementations) { topLevelStdUDFClassesAndImplementations.forEach((topLevelStdUDFClass, stdUDFImplementationClasses) -> { diff --git a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java index f1635ffd..000320b5 100644 --- a/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java +++ b/transportable-udfs-test/transportable-udfs-test-spi/src/main/java/com/linkedin/transport/test/spi/SqlStdTester.java @@ -27,7 +27,7 @@ public interface SqlStdTester extends StdTester { */ default void assertFunctionCall(String functionCallString, Object expectedOutputData, Object expectedOutputType) { throw new UnsupportedOperationException(); - }; + } default void check(TestCase testCase) { assertFunctionCall(getSqlFunctionCallGenerator().getSqlFunctionCallString(testCase.getFunctionCall()), diff --git a/transportable-udfs-trino-plugin/build.gradle b/transportable-udfs-trino-plugin/build.gradle index 8870147e..b5fd7092 100644 --- a/transportable-udfs-trino-plugin/build.gradle +++ b/transportable-udfs-trino-plugin/build.gradle @@ -8,8 +8,7 @@ java { dependencies { implementation project(':transportable-udfs-api') implementation project(":transportable-udfs-trino") - implementation (group:'io.trino', name: 'trino-plugin-toolkit', version: project.ext.'trino-version') - testImplementation(group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') + implementation (group:'io.trino', name: 'trino-main', version: project.ext.'trino-version') compileOnly(group:'io.trino', name: 'trino-spi', version: project.ext.'trino-version') } diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java index a424917c..35b84138 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -39,7 +39,10 @@ public class TransportConnector implements Connector { private static final Logger log = Logger.get(TransportConnector.class); private static final String DEFAULT_TRANSPORT_UDF_REPO = "transport-udf-repo"; private static final FileFilter TRANSPORT_UDF_JAR_FILTER = (file) -> - file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs"); + file.isFile() && file.getName().endsWith(".jar") && !file.getName().startsWith("transportable-udfs-api") + && !file.getName().startsWith("transportable-udfs-trino") + && !file.getName().startsWith("transportable-udfs-type-system") + && !file.getName().startsWith("transportable-udfs-utils"); private final ConnectorMetadata connectorMetadata; private final FunctionProvider functionProvider; From d1ff33a4ba16b1773eeae4ef1f79b989dafa7955 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Wed, 26 Apr 2023 02:05:03 -0700 Subject: [PATCH 15/18] address comments --- .../trino/TrinoTestFunctionDependencies.java | 87 +++++++++++++++++++ .../transport/test/trino/TrinoTester.java | 4 +- .../transport/trino/TrinoFactory.java | 22 ----- 3 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTestFunctionDependencies.java diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTestFunctionDependencies.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTestFunctionDependencies.java new file mode 100644 index 00000000..94b49c0a --- /dev/null +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTestFunctionDependencies.java @@ -0,0 +1,87 @@ +/** + * Copyright 2023 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.transport.test.trino; + +import io.trino.spi.function.FunctionDependencies; +import io.trino.spi.function.FunctionNullability; +import io.trino.spi.function.InvocationConvention; +import io.trino.spi.function.OperatorType; +import io.trino.spi.function.QualifiedFunctionName; +import io.trino.spi.function.ScalarFunctionImplementation; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; +import io.trino.spi.type.TypeSignature; +import io.trino.testing.LocalQueryRunner; +import java.util.List; + + +public class TrinoTestFunctionDependencies implements FunctionDependencies { + private final TypeManager typeManager; + private final LocalQueryRunner queryRunner; + + public TrinoTestFunctionDependencies(TypeManager typeManager, LocalQueryRunner queryRunner) { + this.typeManager = typeManager; + this.queryRunner = queryRunner; + } + + @Override + public Type getType(TypeSignature typeSignature) { + return typeManager.getType(typeSignature); + } + + @Override + public FunctionNullability getFunctionNullability(QualifiedFunctionName name, List parameterTypes) { + return null; + } + + @Override + public FunctionNullability getOperatorNullability(OperatorType operatorType, List parameterTypes) { + return null; + } + + @Override + public FunctionNullability getCastNullability(Type fromType, Type toType) { + return null; + } + + @Override + public ScalarFunctionImplementation getScalarFunctionImplementation(QualifiedFunctionName name, + List parameterTypes, InvocationConvention invocationConvention) { + return null; + } + + @Override + public ScalarFunctionImplementation getScalarFunctionImplementationSignature(QualifiedFunctionName name, + List parameterTypes, InvocationConvention invocationConvention) { + return null; + } + + @Override + public ScalarFunctionImplementation getOperatorImplementation(OperatorType operatorType, List parameterTypes, + InvocationConvention invocationConvention) { + return queryRunner.getFunctionManager() + .getScalarFunctionImplementation(queryRunner.getMetadata().resolveOperator(queryRunner.getDefaultSession(), operatorType, parameterTypes), + invocationConvention); + } + + @Override + public ScalarFunctionImplementation getOperatorImplementationSignature(OperatorType operatorType, + List parameterTypes, InvocationConvention invocationConvention) { + return null; + } + + @Override + public ScalarFunctionImplementation getCastImplementation(Type fromType, Type toType, + InvocationConvention invocationConvention) { + return null; + } + + @Override + public ScalarFunctionImplementation getCastImplementationSignature(TypeSignature fromType, TypeSignature toType, + InvocationConvention invocationConvention) { + return null; + } +} diff --git a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java index 1dca926f..be6f58e4 100644 --- a/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java +++ b/transportable-udfs-test/transportable-udfs-test-trino/src/main/java/com/linkedin/transport/test/trino/TrinoTester.java @@ -66,7 +66,7 @@ public TrinoTester() { _sqlFunctionCallGenerator = new TrinoSqlFunctionCallGenerator(); _toPlatformTestOutputConverter = new ToTrinoTestOutputConverter(); SqlPath sqlPath = new SqlPath("LINKEDIN.TRANSPORT"); - _session = TestingSession.testSessionBuilder().setCatalog("LINKEDIN").setSchema("TRANSPORT").setPath(sqlPath).setClientCapabilities((Set) Arrays.stream( + _session = TestingSession.testSessionBuilder().setPath(sqlPath).setClientCapabilities((Set) Arrays.stream( ClientCapabilities.values()).map(Enum::toString).collect(ImmutableSet.toImmutableSet())).build(); _featuresConfig = new FeaturesConfig(); _runner = LocalQueryRunner.builder(_session).withFeaturesConfig(_featuresConfig).build(); @@ -108,7 +108,7 @@ public StdFactory getStdFactory() { new BoundSignature("test", UNKNOWN, ImmutableList.of()), ImmutableMap.of(), ImmutableMap.of()); - _stdFactory = new TrinoFactory(functionBinding, _runner, InternalTypeManager.TESTING_TYPE_MANAGER); + _stdFactory = new TrinoFactory(functionBinding, new TrinoTestFunctionDependencies(InternalTypeManager.TESTING_TYPE_MANAGER, _runner)); } return _stdFactory; } diff --git a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java index b06ae75e..0fa4d89e 100644 --- a/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java +++ b/transportable-udfs-trino/src/main/java/com/linkedin/transport/trino/TrinoFactory.java @@ -39,9 +39,7 @@ import io.trino.spi.type.MapType; import io.trino.spi.type.RowType; import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; -import io.trino.testing.LocalQueryRunner; import java.lang.invoke.MethodHandle; import java.nio.ByteBuffer; import java.util.List; @@ -56,22 +54,10 @@ public class TrinoFactory implements StdFactory { final FunctionBinding functionBinding; final FunctionDependencies functionDependencies; - final TypeManager typeManager; - final LocalQueryRunner queryRunner; public TrinoFactory(FunctionBinding functionBinding, FunctionDependencies functionDependencies) { this.functionBinding = functionBinding; this.functionDependencies = functionDependencies; - this.typeManager = null; - this.queryRunner = null; - } - - // for test only - public TrinoFactory(FunctionBinding functionBinding, LocalQueryRunner queryRunner, TypeManager typeManager) { - this.functionBinding = functionBinding; - this.functionDependencies = null; - this.queryRunner = queryRunner; - this.typeManager = typeManager; } @Override @@ -145,9 +131,6 @@ public StdStruct createStruct(StdType stdType) { @Override public StdType createStdType(String typeSignatureStr) { TypeSignature typeSignature = applyBoundVariables(parseTypeSignature(quoteReservedKeywords(typeSignatureStr), ImmutableSet.of()), functionBinding); - if (typeManager != null) { - return TrinoWrapper.createStdType(typeManager.getType(typeSignature)); - } return TrinoWrapper.createStdType(functionDependencies.getType(typeSignature)); } @@ -155,11 +138,6 @@ public MethodHandle getOperatorHandle( OperatorType operatorType, List argumentTypes, InvocationConvention invocationConvention) throws OperatorNotFoundException { - if (queryRunner != null && queryRunner.getFunctionManager() != null) { - return queryRunner.getFunctionManager() - .getScalarFunctionImplementation(queryRunner.getMetadata().resolveOperator(queryRunner.getDefaultSession(), operatorType, argumentTypes), - invocationConvention).getMethodHandle(); - } return functionDependencies.getOperatorImplementation(operatorType, argumentTypes, invocationConvention).getMethodHandle(); } } From b18c82e3769d623fad06ff52cd529f91ac3b5afe Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Wed, 26 Apr 2023 10:04:48 -0700 Subject: [PATCH 16/18] add some comments --- .../transport/examples/TestBinaryDuplicateFunction.java | 2 +- .../transport/examples/TestBinaryObjectSizeFunction.java | 1 + .../java/com/linkedin/transport/trino/TransportConfig.java | 4 ++++ .../com/linkedin/transport/trino/TransportConnector.java | 5 +++++ .../linkedin/transport/trino/TransportConnectorFactory.java | 6 +++++- .../transport/trino/TransportConnectorMetadata.java | 6 +++++- .../linkedin/transport/trino/TransportFunctionProvider.java | 6 +++++- .../java/com/linkedin/transport/trino/TransportModule.java | 6 +++++- .../java/com/linkedin/transport/trino/TransportPlugin.java | 6 +++++- .../transport/trino/TransportTransactionHandle.java | 6 +++++- 10 files changed, 41 insertions(+), 7 deletions(-) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java index 2919f6ed..076ef67a 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryDuplicateFunction.java @@ -25,7 +25,7 @@ // the input byte array in Slice with an initial 32 byes capacity, while the execution of the query without where clause does not trigger // the code of VariableWidthBlockBuilder.writeByte() and create the input byte array in Slice with the actual capacity of the content. // Therefore, the outputs from both queries are different. - +// TODO: https://github.com/linkedin/transport/issues/131 public class TestBinaryDuplicateFunction extends AbstractStdUDFTest { @Override protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java index 7ac89276..3e11c355 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/src/test/java/com/linkedin/transport/examples/TestBinaryObjectSizeFunction.java @@ -25,6 +25,7 @@ // the input byte array in Slice with an initial 32 byes capacity, while the execution of the query without where clause does not trigger // the code of VariableWidthBlockBuilder.writeByte() and create the input byte array in Slice with the actual capacity of the content. // Therefore, the outputs from both queries are different. +// TODO: https://github.com/linkedin/transport/issues/131 public class TestBinaryObjectSizeFunction extends AbstractStdUDFTest { @Override protected Map, List>> getTopLevelStdUDFClassesAndImplementations() { diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java index e494a044..3d7d62d5 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConfig.java @@ -8,6 +8,10 @@ import io.airlift.configuration.Config; +/** + * This class defines the configuration which is used by Trino plugin to load UDF classes in Trino server + * following the development guideline in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportConfig { private String transportUdfRepo; diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java index 35b84138..b71dffa8 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -34,6 +34,11 @@ import static java.util.Objects.*; +/** + * This class implements the interface of Connector from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportConnector implements Connector { private static final Logger log = Logger.get(TransportConnector.class); diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java index f8fbdd49..fc7de696 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorFactory.java @@ -16,7 +16,11 @@ import static io.trino.plugin.base.Versions.*; import static java.util.Objects.*; - +/** + * This class implements the interface of ConnectorFactory from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportConnectorFactory implements ConnectorFactory { @Override public String getName() { diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java index c847b09c..137d442e 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnectorMetadata.java @@ -16,7 +16,11 @@ import java.util.Map; import java.util.stream.Collectors; - +/** + * This class implements the interface of ConnectorMetadata from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportConnectorMetadata implements ConnectorMetadata { private final Map functions; diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java index db7e8cce..79409bef 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportFunctionProvider.java @@ -15,7 +15,11 @@ import io.trino.spi.function.WindowFunctionSupplier; import java.util.Map; - +/** + * This class implements the interface of FunctionProvider from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportFunctionProvider implements FunctionProvider { private final Map functions; diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java index 75510f00..a964a2cf 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportModule.java @@ -11,7 +11,11 @@ import static io.airlift.configuration.ConfigBinder.*; - +/** + * This class implements the interface of Module from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportModule implements Module { @Override public void configure(Binder binder) { diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java index 0713c281..270b6a60 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportPlugin.java @@ -9,7 +9,11 @@ import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; - +/** + * This class implements the interface of Plugin from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public class TransportPlugin implements Plugin { @Override public Iterable getConnectorFactories() { diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java index 44b95ba0..1e886540 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportTransactionHandle.java @@ -7,7 +7,11 @@ import io.trino.spi.connector.ConnectorTransactionHandle; - +/** + * This class implements the interface of ConnectorTransactionHandle from Trino SPI as a part of Trino plugin + * to load UDF classes in Trino server following the development guideline + * in https://trino.io/docs/current/develop/spi-overview.html + */ public enum TransportTransactionHandle implements ConnectorTransactionHandle { INSTANCE } \ No newline at end of file From 8d38fc53f0f6ec2109274bdab07b63a96651ea88 Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Wed, 26 Apr 2023 16:42:58 -0700 Subject: [PATCH 17/18] remove conjar repo --- defaultEnvironment.gradle | 3 --- transportable-udfs-examples/build.gradle | 3 --- 2 files changed, 6 deletions(-) diff --git a/defaultEnvironment.gradle b/defaultEnvironment.gradle index 106f3d53..8771dee2 100644 --- a/defaultEnvironment.gradle +++ b/defaultEnvironment.gradle @@ -6,9 +6,6 @@ subprojects { repositories { mavenCentral() jcenter() - maven { - url "https://conjars.wensel.net/repo" - } } project.ext.setProperty('trino-version', '406') project.ext.setProperty('airlift-slice-version', '0.44') diff --git a/transportable-udfs-examples/build.gradle b/transportable-udfs-examples/build.gradle index a367f892..fa7d26de 100644 --- a/transportable-udfs-examples/build.gradle +++ b/transportable-udfs-examples/build.gradle @@ -31,9 +31,6 @@ subprojects { } repositories { mavenCentral() - maven { - url "https://conjars.wensel.net/repo" - } } } From 213178aaf96a8dc7c6b77b1620c9d8ad1a8560eb Mon Sep 17 00:00:00 2001 From: Yiqiang Ding Date: Thu, 27 Apr 2023 07:48:49 -0700 Subject: [PATCH 18/18] address comments --- .../com/linkedin/transport/trino/TransportConnector.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java index b71dffa8..b6a10cff 100644 --- a/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java +++ b/transportable-udfs-trino-plugin/src/main/java/com/linkedin/transport/trino/TransportConnector.java @@ -6,6 +6,7 @@ package com.linkedin.transport.trino; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorMetadata; @@ -22,7 +23,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,12 +65,13 @@ public TransportConnector(TransportConfig config) { TransportUDFClassLoader classLoaderForUdf = new TransportUDFClassLoader(classLoaderForFactory, jarUrlList); ServiceLoader serviceLoader = ServiceLoader.load(StdUdfWrapper.class, classLoaderForUdf); List stdUdfWrappers = ImmutableList.copyOf(serviceLoader); - Map functions = new HashMap<>(); + ImmutableMap.Builder functionIdStdUdfWrapperBuilder = ImmutableMap.builder(); for (StdUdfWrapper wrapper : stdUdfWrappers) { log.info("Loading Transport UDF class: " + wrapper.getFunctionMetadata().getFunctionId().toString()); - functions.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); + functionIdStdUdfWrapperBuilder.put(wrapper.getFunctionMetadata().getFunctionId(), wrapper); } + Map functions = functionIdStdUdfWrapperBuilder.build(); this.connectorMetadata = new TransportConnectorMetadata(functions); this.functionProvider = new TransportFunctionProvider(functions); }