-
Notifications
You must be signed in to change notification settings - Fork 80
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Optimizations for BigDecimal and BigInteger parquet reads (#5638)
- Loading branch information
1 parent
83f62db
commit b900e46
Showing
47 changed files
with
659 additions
and
425 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
...rc/main/java/io/deephaven/parquet/base/materializers/BigDecimalFromBytesMaterializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
// | ||
// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
// | ||
package io.deephaven.parquet.base.materializers; | ||
|
||
import io.deephaven.parquet.base.PageMaterializer; | ||
import io.deephaven.parquet.base.PageMaterializerFactory; | ||
import io.deephaven.util.codec.ObjectCodec; | ||
import org.apache.parquet.column.values.ValuesReader; | ||
|
||
import java.math.BigDecimal; | ||
|
||
public class BigDecimalFromBytesMaterializer extends ObjectMaterializerBase<BigDecimal> implements PageMaterializer { | ||
|
||
public static final class Factory implements PageMaterializerFactory { | ||
|
||
final ObjectCodec<BigDecimal> codec; | ||
|
||
public Factory(ObjectCodec<BigDecimal> codec) { | ||
this.codec = codec; | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { | ||
return new BigDecimalFromBytesMaterializer(dataReader, (BigDecimal) nullValue, numValues, codec); | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { | ||
return new BigDecimalFromBytesMaterializer(dataReader, numValues, codec); | ||
} | ||
} | ||
|
||
private final ValuesReader dataReader; | ||
private final ObjectCodec<BigDecimal> codec; | ||
|
||
private BigDecimalFromBytesMaterializer(ValuesReader dataReader, int numValues, | ||
ObjectCodec<BigDecimal> codec) { | ||
this(dataReader, null, numValues, codec); | ||
} | ||
|
||
private BigDecimalFromBytesMaterializer(ValuesReader dataReader, BigDecimal nullValue, int numValues, | ||
ObjectCodec<BigDecimal> codec) { | ||
super(nullValue, new BigDecimal[numValues]); | ||
this.dataReader = dataReader; | ||
this.codec = codec; | ||
} | ||
|
||
@Override | ||
public void fillValues(int startIndex, int endIndex) { | ||
for (int ii = startIndex; ii < endIndex; ii++) { | ||
final byte[] bytes = dataReader.readBytes().getBytes(); | ||
data[ii] = codec.decode(bytes, 0, bytes.length); | ||
} | ||
} | ||
} |
55 changes: 55 additions & 0 deletions
55
.../src/main/java/io/deephaven/parquet/base/materializers/BigDecimalFromIntMaterializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
// | ||
// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
// | ||
// ****** AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY | ||
// ****** Edit BigDecimalFromLongMaterializer and run "./gradlew replicatePageMaterializers" to regenerate | ||
// | ||
// @formatter:off | ||
package io.deephaven.parquet.base.materializers; | ||
|
||
import io.deephaven.parquet.base.PageMaterializer; | ||
import io.deephaven.parquet.base.PageMaterializerFactory; | ||
import org.apache.parquet.column.values.ValuesReader; | ||
|
||
import java.math.BigDecimal; | ||
|
||
public class BigDecimalFromIntMaterializer extends ObjectMaterializerBase<BigDecimal> implements PageMaterializer { | ||
|
||
public static final class Factory implements PageMaterializerFactory { | ||
final int scale; | ||
|
||
public Factory(final int scale) { | ||
this.scale = scale; | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { | ||
return new BigDecimalFromIntMaterializer(dataReader, (BigDecimal) nullValue, numValues, scale); | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { | ||
return new BigDecimalFromIntMaterializer(dataReader, numValues, scale); | ||
} | ||
}; | ||
|
||
private final ValuesReader dataReader; | ||
private final int scale; | ||
|
||
private BigDecimalFromIntMaterializer(ValuesReader dataReader, int numValues, int scale) { | ||
this(dataReader, null, numValues, scale); | ||
} | ||
|
||
private BigDecimalFromIntMaterializer(ValuesReader dataReader, BigDecimal nullValue, int numValues, int scale) { | ||
super(nullValue, new BigDecimal[numValues]); | ||
this.dataReader = dataReader; | ||
this.scale = scale; | ||
} | ||
|
||
@Override | ||
public void fillValues(int startIndex, int endIndex) { | ||
for (int ii = startIndex; ii < endIndex; ii++) { | ||
data[ii] = BigDecimal.valueOf(dataReader.readInteger(), scale); | ||
} | ||
} | ||
} |
51 changes: 51 additions & 0 deletions
51
...src/main/java/io/deephaven/parquet/base/materializers/BigDecimalFromLongMaterializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
// | ||
// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
// | ||
package io.deephaven.parquet.base.materializers; | ||
|
||
import io.deephaven.parquet.base.PageMaterializer; | ||
import io.deephaven.parquet.base.PageMaterializerFactory; | ||
import org.apache.parquet.column.values.ValuesReader; | ||
|
||
import java.math.BigDecimal; | ||
|
||
public class BigDecimalFromLongMaterializer extends ObjectMaterializerBase<BigDecimal> implements PageMaterializer { | ||
|
||
public static final class Factory implements PageMaterializerFactory { | ||
final int scale; | ||
|
||
public Factory(final int scale) { | ||
this.scale = scale; | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerWithNulls(ValuesReader dataReader, Object nullValue, int numValues) { | ||
return new BigDecimalFromLongMaterializer(dataReader, (BigDecimal) nullValue, numValues, scale); | ||
} | ||
|
||
@Override | ||
public PageMaterializer makeMaterializerNonNull(ValuesReader dataReader, int numValues) { | ||
return new BigDecimalFromLongMaterializer(dataReader, numValues, scale); | ||
} | ||
}; | ||
|
||
private final ValuesReader dataReader; | ||
private final int scale; | ||
|
||
private BigDecimalFromLongMaterializer(ValuesReader dataReader, int numValues, int scale) { | ||
this(dataReader, null, numValues, scale); | ||
} | ||
|
||
private BigDecimalFromLongMaterializer(ValuesReader dataReader, BigDecimal nullValue, int numValues, int scale) { | ||
super(nullValue, new BigDecimal[numValues]); | ||
this.dataReader = dataReader; | ||
this.scale = scale; | ||
} | ||
|
||
@Override | ||
public void fillValues(int startIndex, int endIndex) { | ||
for (int ii = startIndex; ii < endIndex; ii++) { | ||
data[ii] = BigDecimal.valueOf(dataReader.readLong(), scale); | ||
} | ||
} | ||
} |
Oops, something went wrong.