Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dataset maximum dimension sizes #283

Merged
merged 8 commits into from
Jul 9, 2021
Merged

Dataset maximum dimension sizes #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
172d0b5
Change max size type to long
obermeier Jul 7, 2021
aa1b25a
Update API and add test
obermeier Jul 9, 2021
abf9a74
Fix JavaDoc
obermeier Jul 9, 2021
5a759b4
Romove dead variable
obermeier Jul 9, 2021
1fdaa9c
Fix API retrun type
obermeier Jul 9, 2021
4bcb0f0
Fix tests
obermeier Jul 9, 2021
89345cb
Remove broken test file
obermeier Jul 9, 2021
19d47db
Fix hdf5 file
obermeier Jul 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion jhdf/src/main/java/io/jhdf/api/Dataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public interface Dataset extends Node {
*
* @return the max size of this dataset
*/
int[] getMaxSize();
long[] getMaxSize();

/**
* Gets the data layout of this dataset.
Expand Down
5 changes: 3 additions & 2 deletions jhdf/src/main/java/io/jhdf/dataset/DatasetBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;

import static java.nio.ByteOrder.LITTLE_ENDIAN;
import static org.apache.commons.lang3.ClassUtils.primitiveToWrapper;
Expand Down Expand Up @@ -85,11 +86,11 @@ public int[] getDimensions() {
}

@Override
public int[] getMaxSize() {
public long[] getMaxSize() {
if (dataSpace.isMaxSizesPresent()) {
return dataSpace.getMaxSizes();
} else {
return getDimensions();
return Arrays.stream(getDimensions()).asLongStream().toArray();
}
}

Expand Down
10 changes: 5 additions & 5 deletions jhdf/src/main/java/io/jhdf/object/message/DataSpace.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public class DataSpace {
private final byte version;
private final boolean maxSizesPresent;
private final int[] dimensions;
private final int[] maxSizes;
private final long[] maxSizes;
private final byte type;
private final long totalLength;

Expand Down Expand Up @@ -58,12 +58,12 @@ private DataSpace(ByteBuffer bb, Superblock sb) {

// Max dimension sizes
if (maxSizesPresent) {
maxSizes = new int[numberOfDimensions];
maxSizes = new long[numberOfDimensions];
for (int i = 0; i < numberOfDimensions; i++) {
maxSizes[i] = Utils.readBytesAsUnsignedInt(bb, sb.getSizeOfLengths());
maxSizes[i] = Utils.readBytesAsUnsignedLong(bb, sb.getSizeOfLengths());
}
} else {
maxSizes = new int[0];
maxSizes = new long[0];
}

// If type == 2 then it's an empty dataset and totalLength should be 0
Expand Down Expand Up @@ -104,7 +104,7 @@ public int[] getDimensions() {
return ArrayUtils.clone(dimensions);
}

public int[] getMaxSizes() {
public long [] getMaxSizes() {
return ArrayUtils.clone(maxSizes);
}

Expand Down
2 changes: 1 addition & 1 deletion jhdf/src/test/java/io/jhdf/DatasetImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ void testInt32DatasetDimensions() {
@Test
void testInt32DatasetMaxSizes() {
Dataset dataset = (Dataset) hdfFile.getByPath(INT32_PATH);
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{21})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{21})));
}

@Test
Expand Down
13 changes: 13 additions & 0 deletions jhdf/src/test/java/io/jhdf/HdfFileTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,19 @@ void testOpeningValidFile() {
}
}

@Test
void testOpeningFileWithLargeMaxDimensonSize() {
String filePath = "/hdf5/100B_max_dimension_size.hdf5";

testFileUrl = this.getClass().getResource(filePath).getFile();
try (HdfFile hdfFile = new HdfFile(new File(testFileUrl))) {
Dataset dataset = hdfFile.getDatasetByPath("/100B-MaxSize");

assertThat(dataset.getMaxSize().length, is(equalTo(1)));
assertThat(dataset.getMaxSize()[0], is(equalTo(100000000000L)));
}
}

@Test
void testOpeningInvalidFile() {
File file = new File(nonHdfFile);
Expand Down
6 changes: 3 additions & 3 deletions jhdf/src/test/java/io/jhdf/dataset/BitfieldDatasetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ static Stream<Arguments> testBitfieldDataset() {
@MethodSource
void testBitfieldDataset(Dataset dataset) {
assertThat(dataset.getDimensions(), is(equalTo(new int[]{15})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{15})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{15})));
assertThat(dataset.isCompound(), is(false));
assertThat(dataset.isEmpty(), is(false));
assertThat(dataset.isScalar(), is(false));
Expand All @@ -79,7 +79,7 @@ void test2DBitfieldDataset() {

// Should be 2D
assertThat(dataset.getDimensions(), is(equalTo(new int[]{3, 5})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{3, 5})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{3, 5})));
assertThat(dataset.isCompound(), is(false));
assertThat(dataset.isEmpty(), is(false));
assertThat(dataset.isScalar(), is(false));
Expand All @@ -103,7 +103,7 @@ void testScalarBitfieldDataset() {
Dataset dataset = hdfFile.getDatasetByPath("scalar_bitfield");

assertThat(dataset.getDimensions(), is(equalTo(new int[]{})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{})));
assertThat(dataset.isCompound(), is(false));
assertThat(dataset.isEmpty(), is(false));
assertThat(dataset.isScalar(), is(true));
Expand Down
10 changes: 5 additions & 5 deletions jhdf/src/test/java/io/jhdf/dataset/CompoundDatasetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ static Stream<Arguments> test2dCompound() {
void test2dCompound(Dataset dataset) {
// General checks
assertThat(dataset.getDimensions(), is(equalTo(new int[]{3, 3})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{3, 3})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{3, 3})));
assertThat(dataset.getJavaType(), is(Map.class));
assertThat(dataset.getFillValue(), is(nullValue()));
assertThat(dataset.isCompound(), is(true));
Expand Down Expand Up @@ -105,7 +105,7 @@ static Stream<Arguments> testCompound() {
void testCompound(Dataset dataset) {
// General checks
assertThat(dataset.getDimensions(), is(equalTo(new int[]{4})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{4})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{4})));
assertThat(dataset.getJavaType(), is(Map.class));
assertThat(dataset.getFillValue(), is(nullValue()));
assertThat(dataset.isCompound(), is(true));
Expand Down Expand Up @@ -154,7 +154,7 @@ static Stream<Arguments> testArrayVariableLength() {
@MethodSource
void testArrayVariableLength(Dataset dataset) {
assertThat(dataset.getDimensions(), is(equalTo(new int[]{1})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{1})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{1})));
assertThat(dataset.isCompound(), is(true));
assertThat(dataset.getJavaType(), is(Map.class));
assertThat(dataset.getFillValue(), is(nullValue()));
Expand Down Expand Up @@ -182,7 +182,7 @@ static Stream<Arguments> testVariableLengthCompound() {
@MethodSource
void testVariableLengthCompound(Dataset dataset) {
assertThat(dataset.getDimensions(), is(equalTo(new int[]{3})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{3})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{3})));
assertThat(dataset.isCompound(), is(true));
assertThat(dataset.getJavaType(), is(Map.class));
assertThat(dataset.getFillValue(), is(nullValue()));
Expand Down Expand Up @@ -217,7 +217,7 @@ static Stream<Arguments> testNestedCompound() {
@MethodSource
void testNestedCompound(Dataset dataset) {
assertThat(dataset.getDimensions(), is(equalTo(new int[]{3})));
assertThat(dataset.getMaxSize(), is(equalTo(new int[]{3})));
assertThat(dataset.getMaxSize(), is(equalTo(new long[]{3})));
assertThat(dataset.isCompound(), is(true));
assertThat(dataset.getJavaType(), is(Map.class));
assertThat(dataset.getFillValue(), is(nullValue()));
Expand Down
Binary file added jhdf/src/test/resources/hdf5/100B_max_dimension_size.hdf5
View file
Open in desktop
Binary file not shown.