diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9e4ddf3737bbc..ee0f6601bf2eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -48,6 +48,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FSUtils; @@ -1122,6 +1123,8 @@ public class IoTDBConfig { */ private String RateLimiterType = "FixedIntervalRateLimiter"; + private CompressionType WALCompressionAlgorithm = CompressionType.UNCOMPRESSED; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -3906,11 +3909,11 @@ public void setInnerCompactionTaskSelectionDiskRedundancy( this.innerCompactionTaskSelectionDiskRedundancy = innerCompactionTaskSelectionDiskRedundancy; } - public boolean isEnableWALCompression() { - return enableWALCompression; + public CompressionType getWALCompressionAlgorithm() { + return WALCompressionAlgorithm; } - public void setEnableWALCompression(boolean enableWALCompression) { - this.enableWALCompression = enableWALCompression; + public void setWALCompressionAlgorithm(CompressionType WALCompressionAlgorithm) { + this.WALCompressionAlgorithm = WALCompressionAlgorithm; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index ce14a5e4cfd5d..4256e8bcc6f61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -61,6 +61,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FilePathUtils; @@ -429,10 +430,10 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "io_task_queue_size_for_flushing", Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); - conf.setEnableWALCompression( - Boolean.parseBoolean( + conf.setWALCompressionAlgorithm( + CompressionType.valueOf( properties.getProperty( - "enable_wal_compression", Boolean.toString(conf.isEnableWALCompression())))); + "wal_compression_algorithm", conf.getWALCompressionAlgorithm().toString()))); conf.setCompactionScheduleIntervalInMs( Long.parseLong( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 44ef2f4e1965d..ab7e11e911f45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -49,8 +49,9 @@ public abstract class LogWriter implements ILogWriter { protected long size; protected boolean isEndFile = false; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); - private static final CompressionType compressionType = CompressionType.GZIP; - private final ICompressor compressor = ICompressor.getCompressor(CompressionType.GZIP); + private static final CompressionType compressionAlg = + IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); + private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; private static final long MIN_COMPRESS_SIZE = 1024 * 512; @@ -58,7 +59,7 @@ protected LogWriter(File logFile) throws FileNotFoundException { this.logFile = logFile; this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); - if (IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression()) { + if (compressionAlg != CompressionType.UNCOMPRESSED) { compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( @@ -75,7 +76,7 @@ public void write(ByteBuffer buffer) throws IOException { boolean compressed = false; int uncompressedSize = bufferSize; if (!isEndFile - && IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression() + && compressionAlg != CompressionType.UNCOMPRESSED && bufferSize > MIN_COMPRESS_SIZE /* Do not compress buffer that is less than 512KB */) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); @@ -91,7 +92,7 @@ public void write(ByteBuffer buffer) throws IOException { */ headerBuffer.clear(); headerBuffer.put( - compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); + compressed ? compressionAlg.serialize() : CompressionType.UNCOMPRESSED.serialize()); headerBuffer.putInt(bufferSize); if (compressed) { headerBuffer.putInt(uncompressedSize); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index 400b58b6188e8..9711e5049f5ae 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -773,6 +773,10 @@ data_replication_factor=1 # Datatype: long # iot_consensus_cache_window_time_in_ms=-1 +# WAL compression algorithm +# options: UNCOMPRESSED, SNAPPY, LZ4, GZIP, ZSTD +# wal_compress_algorithm=UNCOMPRESSED + #################### ### IoTConsensus Configuration ####################