Merge pull request #530 from jamesmudd/writing

#354 Add Writing Support for file structure

jhdf/build.gradle

@@ -63,6 +63,7 @@ dependencies {
     testImplementation group: 'org.powermock', name: 'powermock-core', version: '2.0.9'
     // Matchers
     testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.2'
+    testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.25.1'
     // Alternative bitshuffle impl to check results against
     testImplementation 'org.xerial.snappy:snappy-java:1.1.10.5'
 }

jhdf/src/main/java/io/jhdf/AbstractWritableNode.java

@@ -0,0 +1,74 @@
+/*
+ * This file is part of jHDF. A pure Java library for accessing HDF5 files.
+ *
+ * http://jhdf.io
+ *
+ * Copyright (c) 2023 James Mudd
+ *
+ * MIT License see 'LICENSE' file
+ */
+
+package io.jhdf;
+
+import io.jhdf.api.Attribute;
+import io.jhdf.api.Group;
+import io.jhdf.api.WritableNode;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
+
+public abstract class AbstractWritableNode implements WritableNode {
+	private final Group parent;
+	private final String name;
+
+	AbstractWritableNode(Group parent, String name) {
+		this.parent = parent;
+		this.name = name;
+	}
+
+	@Override
+	public Group getParent() {
+		return parent;
+	}
+
+	@Override
+	public String getName() {
+		return name;
+	}
+
+	@Override
+	public String getPath() {
+		if (parent == null) {
+			return "/" + getName();
+		} else {
+			return parent.getPath() + "/" + getName();
+		}
+	}
+
+	@Override
+	public Map<String, Attribute> getAttributes() {
+		return Collections.emptyMap();
+	}
+
+	@Override
+	public Attribute getAttribute(String name) {
+		return null;
+	}
+
+	@Override
+	public File getFile() {
+		return parent.getFile();
+	}
+
+	@Override
+	public Path getFileAsPath() {
+		return parent.getFileAsPath();
+	}
+
+	@Override
+	public HdfFile getHdfFile() {
+		return parent.getHdfFile();
+	}
+}

jhdf/src/main/java/io/jhdf/BufferBuilder.java

@@ -0,0 +1,165 @@
+/*
+ * This file is part of jHDF. A pure Java library for accessing HDF5 files.
+ *
+ * http://jhdf.io
+ *
+ * Copyright (c) 2023 James Mudd
+ *
+ * MIT License see 'LICENSE' file
+ */
+
+package io.jhdf;
+
+import io.jhdf.checksum.ChecksumUtils;
+import io.jhdf.exceptions.HdfException;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.BitSet;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+public class BufferBuilder {
+
+	public static final long UNSIGNED_BYTE_MAX = Byte.MAX_VALUE * 2L;
+	public static final long UNSIGNED_SHORT_MAX = Short.MAX_VALUE * 2L;
+	private static final ByteOrder BYTE_ORDER = LITTLE_ENDIAN;
+
+	private final ByteArrayOutputStream byteArrayOutputStream;
+	private final DataOutputStream dataOutputStream; // Note always big endian
+
+	public BufferBuilder() {
+		this.byteArrayOutputStream = new ByteArrayOutputStream();
+		this.dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+	}
+
+	public BufferBuilder writeByte(int i) {
+		try {
+			dataOutputStream.writeByte(i);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder writeBytes(byte[] bytes) {
+		try {
+			dataOutputStream.write(bytes);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder writeShort(int i) {
+		try {
+			short s = (short) i;
+			if(BYTE_ORDER == LITTLE_ENDIAN) {
+				s = Short.reverseBytes(s);
+			}
+			dataOutputStream.writeShort(s);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder writeInt(int i) {
+		try {
+			if(BYTE_ORDER == LITTLE_ENDIAN) {
+				i = Integer.reverseBytes(i);
+			}
+			dataOutputStream.writeInt(i);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder writeLong(long l) {
+		try {
+			if(BYTE_ORDER == LITTLE_ENDIAN) {
+				l = Long.reverseBytes(l);
+			}
+			dataOutputStream.writeLong(l);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public ByteBuffer build() {
+		try {
+			ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+			byteBuffer.order(BYTE_ORDER);
+			dataOutputStream.close();
+			byteArrayOutputStream.close();
+			return byteBuffer;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder writeBitSet(BitSet bitSet, int length) {
+		if(bitSet.toByteArray().length > length) {
+			throw new IllegalArgumentException("BitSet is longer than length provided");
+		}
+		try {
+			final byte[] bytes = Arrays.copyOf(bitSet.toByteArray(), length); // Ensure empty Bitset are not shortened
+			dataOutputStream.write(bytes);
+			return this;
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+	}
+
+	public BufferBuilder appendChecksum() {
+		writeInt(ChecksumUtils.checksum(byteArrayOutputStream.toByteArray()));
+		return this;
+	}
+
+	public BufferBuilder writeBuffer(ByteBuffer byteBuffer) {
+		try {
+			dataOutputStream.write(byteBuffer.array());
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+		return this;
+	}
+
+	public BufferBuilder writeShortestRepresentation(int i) {
+		try {
+			if (i <= UNSIGNED_BYTE_MAX) {
+				dataOutputStream.writeByte(i);
+			} else if(i <= UNSIGNED_SHORT_MAX) {
+				dataOutputStream.writeShort(i);
+			} else {
+				dataOutputStream.write(i);
+			}
+		} catch (IOException e) {
+			throw new BufferBuilderException(e);
+		}
+		return this;
+	}
+
+	public static final class BufferBuilderException extends HdfException {
+		private BufferBuilderException(String message, Throwable throwable) {
+			super(message, throwable);
+		}
+
+		private BufferBuilderException(Throwable throwable) {
+			this("Error in BufferBuilder", throwable);
+		}
+	}
+
+	@Override
+	public String toString() {
+		return "BufferBuilder{" +
+			 "bytesWriten=" + dataOutputStream.size() +
+			"}";
+	}
+}

jhdf/src/main/java/io/jhdf/HdfFile.java

@@ -237,6 +237,10 @@ private static long nextOffset(long offset) {
 		return offset * 2;
 	}
 
+	public static WritableHdfFile write(Path path) {
+		return new WritableHdfFile(path);
+	}
+
 	/**
 	 * Gets the size of the user block of this file.
 	 *

jhdf/src/main/java/io/jhdf/ObjectHeader.java

@@ -11,6 +11,7 @@
 
 import io.jhdf.checksum.ChecksumUtils;
 import io.jhdf.exceptions.HdfException;
+import io.jhdf.exceptions.UnsupportedHdfException;
 import io.jhdf.object.message.Message;
 import io.jhdf.object.message.ObjectHeaderContinuationMessage;
 import io.jhdf.storage.HdfBackingStorage;
@@ -221,20 +222,7 @@ private ObjectHeaderV2(HdfBackingStorage hdfBackingStorage, long address) {
 				flags = BitSet.valueOf(new byte[]{bb.get()});
 
 				// Size of chunk 0
-				final byte sizeOfChunk0;
-				if (flags.get(1)) {
-					if (flags.get(0)) {
-						sizeOfChunk0 = 8;
-					} else {
-						sizeOfChunk0 = 4;
-					}
-				} else { // bit 0 = false
-					if (flags.get(0)) {
-						sizeOfChunk0 = 2;
-					} else {
-						sizeOfChunk0 = 1;
-					}
-				}
+				final byte sizeOfChunk0 = getSizeOfChunk0Size();
 
 				// Timestamps
 				if (flags.get(TIMESTAMPS_PRESENT)) {
@@ -291,6 +279,78 @@ private ObjectHeaderV2(HdfBackingStorage hdfBackingStorage, long address) {
 			}
 		}
 
+		private byte getSizeOfChunk0Size() {
+			final byte sizeOfChunk0;
+			if (flags.get(1)) {
+				if (flags.get(0)) {
+					sizeOfChunk0 = 8;
+				} else {
+					sizeOfChunk0 = 4;
+				}
+			} else { // bit 0 = false
+				if (flags.get(0)) {
+					sizeOfChunk0 = 2;
+				} else {
+					sizeOfChunk0 = 1;
+				}
+			}
+			return sizeOfChunk0;
+		}
+
+		public ObjectHeaderV2(long address, List<Message> messages) {
+			super(address);
+			this.messages.addAll(messages);
+			version = 2;
+
+			accessTime = -1;
+			modificationTime = -1;
+			changeTime = -1;
+			birthTime = -1;
+
+			maximumNumberOfCompactAttributes = -1;
+			maximumNumberOfDenseAttributes = -1;
+
+			flags = new BitSet(8); // TODO make consistent with values
+		}
+
+		public ByteBuffer toBuffer() {
+			BufferBuilder bufferBuilder = new BufferBuilder()
+				.writeBytes(OBJECT_HEADER_V2_SIGNATURE)
+				.writeByte(version)
+				.writeBitSet(flags, 1);
+
+			if (flags.get(TIMESTAMPS_PRESENT)) {
+				bufferBuilder.writeInt((int) accessTime);
+				bufferBuilder.writeInt((int) modificationTime);
+				bufferBuilder.writeInt((int) changeTime);
+				bufferBuilder.writeInt((int) birthTime);
+			}
+
+			if(flags.get(NUMBER_OF_ATTRIBUTES_PRESENT)) {
+				// TODO min/max attributes
+				throw new UnsupportedHdfException("Writting number of attributes");
+			}
+
+			// Start messages
+			ByteBuffer messagesBuffer = messagesToBuffer();
+			bufferBuilder.writeByte(messagesBuffer.capacity()) // Size of chunk 0 TODO support variable sizes
+					.writeBuffer(messagesBuffer);
+
+			return bufferBuilder.appendChecksum().build();
+		}
+
+		private ByteBuffer messagesToBuffer() {
+			BufferBuilder bufferBuilder = new BufferBuilder();
+			for (Message message : messages) {
+				final ByteBuffer messageBuffer = message.toBuffer();
+				bufferBuilder.writeByte(message.getMessageType())
+					.writeShort(messageBuffer.capacity())
+					.writeBytes(message.flagsToBytes())
+					.writeBuffer(messageBuffer);
+			}
+			return bufferBuilder.build();
+		}
+
 		private void readMessages(HdfBackingStorage hdfBackingStorage, ByteBuffer bb) {
 			while (bb.remaining() >= 8) {
 				Message m = Message.readObjectHeaderV2Message(bb, hdfBackingStorage, this.isAttributeCreationOrderTracked());
@@ -380,4 +440,5 @@ protected ObjectHeader initialize() {
 
 		};
 	}
+
 }