-
Notifications
You must be signed in to change notification settings - Fork 187
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Register "passthrough" UDFs with correct ordinal return type (#541)
* register spark groot udf with correct return type * test showcase * extend + refactor unit test --------- Co-authored-by: Limian Zhang <raylmzhang@gmail.com>
- Loading branch information
Showing
6 changed files
with
177 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
coral-schema/src/test/java/com/linkedin/coral/hive/hive2rel/CoralTestUDFReturnSecondArg.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| /** | ||
| * Copyright 2021-2024 LinkedIn Corporation. All rights reserved. | ||
| * Licensed under the BSD-2 Clause license. | ||
| * See LICENSE in the project root for license information. | ||
| */ | ||
| package com.linkedin.coral.hive.hive2rel; | ||
|
|
||
| import org.apache.hadoop.hive.ql.exec.Description; | ||
| import org.apache.hadoop.hive.ql.exec.UDFArgumentException; | ||
| import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; | ||
| import org.apache.hadoop.hive.ql.metadata.HiveException; | ||
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; | ||
| import org.apache.hadoop.hive.serde2.objectinspector.*; | ||
|
|
||
|
|
||
| @Description(name = "return_second_arg_struct_udf", | ||
| value = "_FUNC_(string, struct) - Returns the second argument (struct) as-is") | ||
| public class CoralTestUDFReturnSecondArg extends GenericUDF { | ||
|
|
||
| private transient ObjectInspector stringOI; | ||
| private transient StructObjectInspector structOI; | ||
|
|
||
| @Override | ||
| public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { | ||
| // Check the number of arguments | ||
| if (arguments.length != 2) { | ||
| throw new UDFArgumentLengthException( | ||
| "return_struct_udf() requires exactly two arguments: a string and a struct."); | ||
| } | ||
|
|
||
| // Validate the first argument (string) | ||
| if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE || ((PrimitiveObjectInspector) arguments[0]) | ||
| .getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING) { | ||
| throw new UDFArgumentException("The first argument must be a string."); | ||
| } | ||
|
|
||
| // Validate the second argument (struct) | ||
| if (arguments[1].getCategory() != ObjectInspector.Category.STRUCT) { | ||
| throw new UDFArgumentException("The second argument must be a struct."); | ||
| } | ||
|
|
||
| // Initialize ObjectInspectors | ||
| stringOI = arguments[0]; | ||
| structOI = (StructObjectInspector) arguments[1]; | ||
|
|
||
| // Return the ObjectInspector for the struct (second argument) | ||
| return structOI; | ||
| } | ||
|
|
||
| @Override | ||
| public Object evaluate(DeferredObject[] arguments) throws HiveException { | ||
| // Simply return the second argument as-is | ||
| Object structObj = arguments[1].get(); | ||
| return structObj; | ||
| } | ||
|
|
||
| @Override | ||
| public String getDisplayString(String[] children) { | ||
| return "return_struct_udf(" + children[0] + ", " + children[1] + ")"; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
45 changes: 45 additions & 0 deletions
45
coral-schema/src/test/resources/base-complex-mixed-nullabilities.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| { | ||
| "type": "record", | ||
| "name": "OuterRecord", | ||
| "fields": [ | ||
| { | ||
| "name": "innerRecord", | ||
| "type": { | ||
| "type": "record", | ||
| "name": "InnerRecord", | ||
| "fields": [ | ||
| { | ||
| "name": "String_Field_Non_Nullable", | ||
| "type": "string" | ||
| }, | ||
| { | ||
| "name": "String_Field_Nullable", | ||
| "type": [ "string", "null" ] | ||
| }, | ||
| { | ||
| "name" : "Int_Field_Non_Nullable", | ||
| "type" : "int" | ||
| }, | ||
| { | ||
| "name" : "Int_Field_Nullable", | ||
| "type" : [ "int", "null" ] | ||
| }, | ||
| { | ||
| "name" : "Array_Col_Non_Nullable", | ||
| "type" : { | ||
| "type" : "array", | ||
| "items" : "string" | ||
| } | ||
| }, | ||
| { | ||
| "name" : "Array_Col_Nullable", | ||
| "type" : [ "null", { | ||
| "type" : "array", | ||
| "items" : [ "null", "string" ] | ||
| } ] | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } |
38 changes: 38 additions & 0 deletions
38
...c/test/resources/testPreserveNullabilitiesAfterApplyingOrdinalReturnTypeUDF-expected.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| { | ||
| "type" : "record", | ||
| "name" : "innerfield_with_udf", | ||
| "namespace" : "default.innerfield_with_udf", | ||
| "fields" : [ { | ||
| "name" : "innerRecord", | ||
| "type" : { | ||
| "type" : "record", | ||
| "name" : "InnerRecord", | ||
| "namespace" : "default.innerfield_with_udf.innerfield_with_udf", | ||
| "fields" : [ { | ||
| "name" : "String_Field_Non_Nullable", | ||
| "type" : "string" | ||
| }, { | ||
| "name" : "String_Field_Nullable", | ||
| "type" : [ "string", "null" ] | ||
| }, { | ||
| "name" : "Int_Field_Non_Nullable", | ||
| "type" : "int" | ||
| }, { | ||
| "name" : "Int_Field_Nullable", | ||
| "type" : [ "int", "null" ] | ||
| }, { | ||
| "name" : "Array_Col_Non_Nullable", | ||
| "type" : { | ||
| "type" : "array", | ||
| "items" : "string" | ||
| } | ||
| }, { | ||
| "name" : "Array_Col_Nullable", | ||
| "type" : [ "null", { | ||
| "type" : "array", | ||
| "items" : [ "null", "string" ] | ||
| } ] | ||
| } ] | ||
| } | ||
| } ] | ||
| } |