-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add another checksum to OpenZFS that has similar performance to Edon-R, but without the caveats around the latter. Add micro benchmarking for all advanced hashing functions like it's done for fletcher-4. The statistics are located here: '/proc/spl/kstat/zfs/chksum_bench' ''' 20 0 0x01 -1 0 42606477364 55268564180 implementation bit 1k 2k 4k 8k 16k 32k 64k 128k 256k 512k 1m edonr-generic 256 892 982 1103 1167 1135 1178 1193 1226 1192 1165 1173 skein-generic 256 402 416 431 443 453 455 454 449 437 444 441 sha256-generic 256 202 215 227 230 237 228 222 190 229 226 224 sha512-generic 512 107 122 129 131 134 129 132 150 224 278 343 blake3-generic 256 175 183 171 179 182 181 180 176 176 175 174 blake3-sse2 256 323 676 1011 1151 1169 1086 1148 1187 1138 1197 1182 blake3-sse41 256 106 291 678 1223 1273 1326 1458 1441 1401 1431 1402 ''' The units are MiB/s for each method. TODO Split up the patch into into an patchset of 3 or 4 parts. Signed-off-by: Rich Ercolani <rincebrain@gmail.com> Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
- Loading branch information
Showing
40 changed files
with
12,053 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
|
|
||
| /* | ||
| * CDDL HEADER START | ||
| * | ||
| * The contents of this file are subject to the terms of the | ||
| * Common Development and Distribution License, Version 1.0 only | ||
| * (the "License"). You may not use this file except in compliance | ||
| * with the License. | ||
| * | ||
| * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | ||
| * or http://www.opensolaris.org/os/licensing. | ||
| * See the License for the specific language governing permissions | ||
| * and limitations under the License. | ||
| * | ||
| * When distributing Covered Code, include this CDDL HEADER in each | ||
| * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | ||
| * If applicable, add the following below this CDDL HEADER, with the | ||
| * fields enclosed by brackets "[]" replaced with your own identifying | ||
| * information: Portions Copyright [yyyy] [name of copyright owner] | ||
| * | ||
| * CDDL HEADER END | ||
| */ | ||
|
|
||
| /* | ||
| * Based on BLAKE3 v1.2.0, https://github.com/BLAKE3-team/BLAKE3 | ||
| * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor | ||
| * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de> | ||
| */ | ||
|
|
||
| #ifndef BLAKE3_H | ||
| #define BLAKE3_H | ||
|
|
||
| #ifdef _KERNEL | ||
| #include <sys/types.h> | ||
| #else | ||
| #include <stdint.h> | ||
| #include <stdlib.h> | ||
| #endif | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| #define BLAKE3_KEY_LEN 32 | ||
| #define BLAKE3_OUT_LEN 32 | ||
| #define BLAKE3_MAX_DEPTH 54 | ||
| #define BLAKE3_BLOCK_LEN 64 | ||
| #define BLAKE3_CHUNK_LEN 1024 | ||
|
|
||
| /* | ||
| * This struct is a private implementation detail. | ||
| * It has to be here because it's part of BLAKE3_CTX below. | ||
| */ | ||
| typedef struct { | ||
| uint32_t cv[8]; | ||
| uint64_t chunk_counter; | ||
| uint8_t buf[BLAKE3_BLOCK_LEN]; | ||
| uint8_t buf_len; | ||
| uint8_t blocks_compressed; | ||
| uint8_t flags; | ||
| } blake3_chunk_state_t; | ||
|
|
||
| typedef struct { | ||
| uint32_t key[8]; | ||
| blake3_chunk_state_t chunk; | ||
| uint8_t cv_stack_len; | ||
|
|
||
| /* | ||
| * The stack size is MAX_DEPTH + 1 because we do lazy merging. For | ||
| * example, with 7 chunks, we have 3 entries in the stack. Adding an | ||
| * 8th chunk requires a 4th entry, rather than merging everything down | ||
| * to 1, because we don't know whether more input is coming. This is | ||
| * different from how the reference implementation does things. | ||
| */ | ||
| uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; | ||
| } BLAKE3_CTX; | ||
|
|
||
| /* init the context for hash operation */ | ||
| void Blake3_Init(BLAKE3_CTX *ctx); | ||
|
|
||
| /* init the context for a MAC and/or tree hash operation */ | ||
| void Blake3_InitKeyed(BLAKE3_CTX *ctx, const uint8_t key[BLAKE3_KEY_LEN]); | ||
|
|
||
| /* process the input bytes */ | ||
| void Blake3_Update(BLAKE3_CTX *ctx, const void *input, size_t input_len); | ||
|
|
||
| /* finalize the hash computation and output the result */ | ||
| void Blake3_Final(const BLAKE3_CTX *ctx, uint8_t *out); | ||
|
|
||
| /* finalize the hash computation and output the result */ | ||
| void Blake3_FinalSeek(const BLAKE3_CTX *ctx, uint64_t seek, uint8_t *out, | ||
| size_t out_len); | ||
|
|
||
| /* | ||
| * Returns the number of supported BLAKE3 implementations | ||
| */ | ||
| extern int blake3_get_impl_count(void); | ||
|
|
||
| /* | ||
| * Returns the name of selected BLAKE3 implementation | ||
| */ | ||
| extern const char *blake3_get_impl_name(void); | ||
|
|
||
| /* | ||
| * Set BLAKE3 implementation via id or name | ||
| */ | ||
| extern void blake3_set_impl_id(int id); | ||
| extern void blake3_set_impl_name(char *name); | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* BLAKE3_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
|
|
||
| /* | ||
| * CDDL HEADER START | ||
| * | ||
| * The contents of this file are subject to the terms of the | ||
| * Common Development and Distribution License (the "License"). | ||
| * You may not use this file except in compliance with the License. | ||
| * | ||
| * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | ||
| * or http://www.opensolaris.org/os/licensing. | ||
| * See the License for the specific language governing permissions | ||
| * and limitations under the License. | ||
| * | ||
| * When distributing Covered Code, include this CDDL HEADER in each | ||
| * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | ||
| * If applicable, add the following below this CDDL HEADER, with the | ||
| * fields enclosed by brackets "[]" replaced with your own identifying | ||
| * information: Portions Copyright [yyyy] [name of copyright owner] | ||
| * | ||
| * CDDL HEADER END | ||
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2021 Tino Reichardt <milky-zfs@mcmilk.de> | ||
| */ | ||
|
|
||
| #ifndef _ZFS_CHKSUM_H | ||
| #define _ZFS_CHKSUM_H | ||
|
|
||
| #ifdef _KERNEL | ||
| #include <sys/types.h> | ||
| #else | ||
| #include <stdint.h> | ||
| #include <stdlib.h> | ||
| #endif | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| /* Benchmark the chksums of ZFS when the module is loading */ | ||
| void chksum_init(void); | ||
| void chksum_fini(void); | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif /* _ZFS_CHKSUM_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.