Skip to content

Commit

Permalink
DRILL-2385: Count on complex objects failed with missing function imp…
Browse files Browse the repository at this point in the history
…lementation - added MapHolder, ListHolder; - added testCountComplexObjects() unit test.
  • Loading branch information
vdiravka authored and parthchandra committed Jun 19, 2016
1 parent 3209886 commit f86c4fa
Show file tree
Hide file tree
Showing 11 changed files with 201 additions and 6 deletions.
21 changes: 21 additions & 0 deletions exec/java-exec/src/main/codegen/data/CountAggrTypes.tdd
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,59 @@
countFunctionsInput: [
"Bit",
"NullableBit",
"RepeatedBit",
"Int",
"NullableInt",
"RepeatedInt",
"BigInt",
"NullableBigInt",
"RepeatedBigInt",
"Float4",
"NullableFloat4",
"RepeatedFloat4",
"Float8",
"NullableFloat8",
"RepeatedFloat8",
"Date",
"NullableDate",
"RepeatedDate",
"TimeStamp",
"NullableTimeStamp",
"RepeatedTimeStamp",
"Time",
"NullableTime",
"RepeatedTime",
"IntervalDay",
"NullableIntervalDay",
"RepeatedIntervalDay",
"IntervalYear",
"NullableIntervalYear",
"RepeatedIntervalYear",
"Interval",
"NullableInterval",
"RepeatedInterval",
"VarChar",
"NullableVarChar",
"RepeatedVarChar",
"VarBinary"
"NullableVarBinary",
"RepeatedVarBinary"
"Decimal9",
"NullableDecimal9",
"RepeatedDecimal9",
"Decimal18",
"NullableDecimal18",
"RepeatedDecimal18",
"Decimal28Sparse",
"NullableDecimal28Sparse",
"RepeatedDecimal28Sparse",
"Decimal38Sparse",
"NullableDecimal38Sparse"
"RepeatedDecimal38Sparse",
"List"
"RepeatedList",
"Map"
"RepeatedMap"
]
}

Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ public class ResolverTypePrecedence {
precedenceMap.put(MinorType.INTERVALDAY, i+= 2);
precedenceMap.put(MinorType.INTERVALYEAR, i+= 2);
precedenceMap.put(MinorType.INTERVAL, i+= 2);
precedenceMap.put(MinorType.MAP, i += 2);
precedenceMap.put(MinorType.LIST, i += 2);
precedenceMap.put(MinorType.UNION, i += 2);

MAX_IMPLICIT_CAST_COST = i;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,8 @@ private static void initTypeRules() {
rule.add(MinorType.FIXEDBINARY);
rules.put(MinorType.VARBINARY, rule);

rules.put(MinorType.MAP, Sets.newHashSet(MinorType.MAP));
rules.put(MinorType.LIST, Sets.newHashSet(MinorType.LIST));
rules.put(MinorType.UNION, Sets.newHashSet(MinorType.UNION));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.drill.exec.fn.impl;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.PlanTestBase;
Expand All @@ -28,6 +29,7 @@
import org.junit.Test;

import java.util.List;
import java.util.Map;

public class TestAggregateFunctions extends BaseTestQuery {

Expand Down Expand Up @@ -525,4 +527,36 @@ public void testPushFilterDown() throws Exception {
PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPatterns);
}

@Test // DRILL-2385: count on complex objects failed with missing function implementation
public void testCountComplexObjects() throws Exception {
final String query = "select count(t.%s) %s from cp.`complex/json/complex.json` t";
Map<String, String> objectsMap = Maps.newHashMap();
objectsMap.put("COUNT_BIG_INT_REPEATED", "sia");
objectsMap.put("COUNT_FLOAT_REPEATED", "sfa");
// TODO: can be uncommented after fixing DRILL-4664
// objectsMap.put("COUNT_MAP_REPEATED", "soa");
// objectsMap.put("COUNT_MAP_REQUIRED", "oooi");
objectsMap.put("COUNT_LIST_REPEATED", "odd");
objectsMap.put("COUNT_LIST_OPTIONAL", "sia");

for (String object: objectsMap.keySet()) {
String optionSetting = "";
if (object.equals("COUNT_LIST_OPTIONAL")) {
// if `exec.enable_union_type` parameter is true then BIGINT<REPEATED> object is converted to LIST<OPTIONAL> one
optionSetting = "alter session set `exec.enable_union_type`=true";
}
try {
testBuilder()
.sqlQuery(query, objectsMap.get(object), object)
.optionSettingQueriesForTestQuery(optionSetting)
.unOrdered()
.baselineColumns(object)
.baselineValues(3L)
.go();
} finally {
test("ALTER SESSION RESET `exec.enable_union_type`");
}
}
}

}
30 changes: 30 additions & 0 deletions exec/java-exec/src/test/resources/complex/json/complex.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{"sia":[1, 11, 101, 1001],
"sfa":[0.0, 1.01, 10.222, 10.0006789],
"soa":[{"in":1},{"in":1,"fl":1.12345}, {"in":1, "fl":10.12345, "nul":null}, {"in":1, "fl":10.6789, "nul":null, "bool":true, "str":"here is a string at row 1"}],
"oooi":{"oa":{"oab":{"oabc":1}}},
"odd": [
[[1],[],[3]],
[],
[[5]]
]
}
{"sia":[2, 12, 102, 1002],
"sfa":[0.0, 2.01, 20.222, 20.0006789],
"soa":[{"in":2},{"in":2,"fl":2.12345}, {"in":2, "fl":20.12345, "nul":"not null"}, {"in":2, "fl":20.6789, "nul":null, "bool":false, "str":"here is a string at row 2"}],
"oooi":{"oa":{"oab":{"oabc":2}}},
"odd": [
[[1],[],[3]],
[],
[[5]]
]
}
{"sia":[3, 13, 103, 1003],
"sfa":[0.0, 3.01, 30.222, 30.0006789],
"soa":[{"in":3},{"in":3,"fl":3.12345}, {"in":3, "fl":30.12345, "nul":"not null"}, {"in":3, "fl":30.6789, "nul":"not null", "bool":true, "str":"here is a string at row 3"}],
"oooi":{"oa":{"oab":{"oabc":3}}},
"odd": [
[[1],[],[3]],
[],
[[5]]
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
[],
[[5]]
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
[],
[{"val": [7]}]
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.expr.holders;

import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.vector.complex.reader.FieldReader;

public class ListHolder implements ValueHolder {
public static final TypeProtos.MajorType TYPE = Types.optional(TypeProtos.MinorType.LIST);
public FieldReader reader;
public int isSet;

public TypeProtos.MajorType getType() {
return TYPE;
}

public boolean isSet() {
return isSet == 1;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.expr.holders;

import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.vector.complex.reader.FieldReader;

public class MapHolder implements ValueHolder {
public static final TypeProtos.MajorType TYPE = Types.required(TypeProtos.MinorType.MAP);
public FieldReader reader;

public TypeProtos.MajorType getType() {
return TYPE;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,23 @@
*/
package org.apache.drill.exec.expr.holders;

import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.vector.complex.ListVector;

public final class RepeatedListHolder implements ValueHolder{
public int start;
public int end;

public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.LIST);

public TypeProtos.MajorType getType() {return TYPE;}

/** The first index (inclusive) into the Vector. **/
public int start;

/** The last index (exclusive) into the Vector. **/
public int end;

/** The Vector holding the actual values. **/
public ListVector vector;

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,28 @@
*/
package org.apache.drill.exec.expr.holders;

import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.complex.MapVector;

import java.util.LinkedHashSet;

public final class RepeatedMapHolder implements ValueHolder{
public int start;
public int end;

public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.MAP);

// public final LinkedHashSet<ValueHolder> children = null;

public TypeProtos.MajorType getType() {return TYPE;}

/** The first index (inclusive) into the Vector. **/
public int start;

/** The last index (exclusive) into the Vector. **/
public int end;

/** The Vector holding the actual values. **/
public MapVector vector;

}

0 comments on commit f86c4fa

Please sign in to comment.