From a5f3440e07b5d77a05d2fe1335cfd4adc98931c2 Mon Sep 17 00:00:00 2001 From: Lorenz Pullwitt <68168097+lopul@users.noreply.github.com> Date: Fri, 25 Sep 2020 02:45:18 +0200 Subject: [PATCH] Add files via upload --- imf/indexedmemoryfile.c | 864 ++++++++++++++++++++++++++++++++++++++++ imf/indexedmemoryfile.h | 77 ++++ imf/sha1.c | 317 +++++++++++++++ imf/sha1.h | 54 +++ 4 files changed, 1312 insertions(+) create mode 100644 imf/indexedmemoryfile.c create mode 100644 imf/indexedmemoryfile.h create mode 100644 imf/sha1.h diff --git a/imf/indexedmemoryfile.c b/imf/indexedmemoryfile.c new file mode 100644 index 0000000..fac6d7c --- /dev/null +++ b/imf/indexedmemoryfile.c @@ -0,0 +1,864 @@ + +// +// Author: Lorenz Pullwitt +// Copyright 2016-2020 +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, you can find it here: +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +// + +#include "indexedmemoryfile.h" +#include "sha1.h" +#include +#include +#include +#include +#include +#include +#include // abs + +const int32_t INITIAL_CHUNKS = 8; + +void imf_init (struct IndexedMemoryFile *imf) +{ + imf->filedesc = -1; + imf->chunks = NULL; + imf->chunk_count = 0; + imf->chunk_order = NULL; + imf->delete_mark = NULL; + imf->delete_end = 0; + sw_init (&imf->sw); + imf->stat_swap = -1; + imf->stats_gaps = -1; + imf->stats_gaps_space = -1; + imf->stats_gaps_str = NULL; +} + +struct GapStats +{ + int32_t gs_size; + int32_t gs_n; +}; + +char imf_is_open (struct IndexedMemoryFile *imf) +{ + return (imf->chunk_count > 0) && (imf->filedesc != -1); +} + +static int +imf_read (struct IndexedMemoryFile *imf, void *data, int64_t position, int32_t data_size) +{ + int e; + off_t off; + off_t cur_pos; + ssize_t ssize; + struct Sha1Context sha1; + uint8_t message_digest_data[SHA1_HASH_SIZE]; + uint8_t message_digest[SHA1_HASH_SIZE]; + off = position; + cur_pos = lseek (imf->filedesc, off, SEEK_SET); + e = cur_pos == -1; + if (e == 0) + { + assert (sizeof (data_size) <= sizeof (size_t)); + ssize = read (imf->filedesc, data, data_size); + e = ssize == -1; + if (e == 0) + { + e = sha1_reset (&sha1); + if (e == 0) + { + e = sha1_input (&sha1, data, data_size); + if (e == 0) + { + e = sha1_result (&sha1, message_digest_data); + if (e == 0) + { + ssize = read (imf->filedesc, message_digest, SHA1_HASH_SIZE); + e = ssize == -1; + if (e == 0) + { + e = memcmp (message_digest_data, message_digest, SHA1_HASH_SIZE); + } + } + } + } + } + } + return e; +} + +static int +imf_write (struct IndexedMemoryFile *imf, const void *data, int64_t position, int32_t data_size) +{ + int e; + off_t off; + off_t cur_pos; + ssize_t ssize; + struct Sha1Context sha1; + uint8_t message_digest[SHA1_HASH_SIZE]; + off = position; + cur_pos = lseek (imf->filedesc, off, SEEK_SET); + e = cur_pos == -1; + if (e == 0) + { + assert (sizeof (data_size) <= sizeof (size_t)); + ssize = write (imf->filedesc, data, data_size); + e = ssize == -1; + if (e == 0) + { + e = sha1_reset (&sha1); + if (e == 0) + { + e = sha1_input (&sha1, data, data_size); + if (e == 0) + { + e = sha1_result (&sha1, message_digest); + if (e == 0) + { + ssize = write (imf->filedesc, message_digest, SHA1_HASH_SIZE); + e = ssize == -1; + } + } + } + } + } + return e; +} + +static int +imf_alloc_chunks (struct IndexedMemoryFile *imf) +{ + int e; + int i; + int32_t data_size; + int32_t increase; + int32_t chunk_count; + struct Chunk *chunks; + int32_t *chunk_order; + size_t size; + increase = imf->chunk_count; + if (increase > 128) + { + increase = 128; + } + chunk_count = imf->chunk_count + increase; + assert (chunk_count < ((0x7FFFFFFF - SHA1_HASH_SIZE) / sizeof (struct Chunk) - 2)); + data_size = sizeof (struct Chunk) * chunk_count; + chunks = realloc (imf->chunks, data_size); + e = chunks == NULL; + if (e == 0) + { + size = sizeof (int32_t) * chunk_count; + chunk_order = realloc (imf->chunk_order, size); + e = chunk_order == NULL; + if (e == 0) + { + imf->chunks = chunks; + imf->chunk_order = chunk_order; + for (i = imf->chunk_count; i < chunk_count; i++) + { + chunks[i].position = INT64_MAX; + chunks[i].chunk_size = 0; + chunk_order[i] = i; + } + imf->chunk_count = chunk_count; + } + } + return e; +} + +static int +imf_find_space (struct IndexedMemoryFile *imf, int64_t *position, int32_t chunk_size) +{ + int e; + int i; + int64_t l_offset; + int64_t r_offset; + int64_t space; + int64_t lg_pos; // large gap + int32_t lg_chunk_size; + lg_pos = -1; + e = -1; + l_offset = 0; + for (i = 0; i < imf->chunk_count && e != 0; i++) + { + r_offset = imf->chunks[imf->chunk_order[i]].position; + space = r_offset - l_offset; + assert (space >= 0); // make sure no overlapping has happend + if (space == chunk_size) + { + *position = l_offset; + e = 0; + } + else if (space > chunk_size * 2) + { + if ((lg_pos == -1) || (lg_pos >= 0 && lg_chunk_size > space)) + { + lg_pos = l_offset; + lg_chunk_size = space; + } + } + l_offset = r_offset + imf->chunks[imf->chunk_order[i]].chunk_size; + } + if ((e != 0) && (lg_pos != -1)) + { + assert (lg_pos >= 0); + *position = lg_pos; + e = 0; + } + return e; +} + +static void +imf_sort_order (struct IndexedMemoryFile *imf) +{ + int e; + int32_t tmp; + int i; + int j; + int64_t i_pos; + int64_t j_pos; + int sw_i; + sw_i = sw_start ("sort order", &imf->sw); + e = sw_i < 0; + if (e == 0) + { + if (imf->chunk_count > 1) + { + i = 1; + while (i < imf->chunk_count) + { + tmp = imf->chunk_order[i]; + j = i - 1; + i_pos = imf->chunks[tmp].position; + do + { + j_pos = imf->chunks[imf->chunk_order[j]].position; + if (j_pos > i_pos) + { + imf->chunk_order[j + 1] = imf->chunk_order[j]; + } + else + break; + j--; + } while (j >= 0); + imf->chunk_order[j + 1] = tmp; + i++; + } + } + e = sw_stop (sw_i, &imf->sw); + } + assert (e == 0); +} + +int +imf_create (struct IndexedMemoryFile *imf, const char *filename, int flags_mask) +{ + int e; + int filedesc; + int chunk_count; + int32_t data_size; + struct Chunk *chunks; + size_t size; + int32_t *chunk_order; + int64_t position; + int i; + int32_t chunk_size; + struct flock fl; + assert (imf->filedesc == -1 && imf->chunk_count == 0 && imf->chunks == NULL); + filedesc = open (filename, O_CREAT | flags_mask | O_RDWR, S_IRUSR | S_IWUSR); + e = filedesc == -1; + if (e == 0) + { + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; // Lock entire file + e = fcntl (filedesc, F_SETLK, &fl); + if (e == 0) + { + chunk_count = INITIAL_CHUNKS; + data_size = sizeof (struct Chunk) * chunk_count; + chunks = malloc (data_size); + e = chunks == NULL; + if (e == 0) + { + size = sizeof (int32_t) * chunk_count; + chunk_order = malloc (size); + e = chunk_order == NULL; + if (e == 0) + { + imf->filedesc = filedesc; + imf->chunks = chunks; + imf->chunk_count = chunk_count; + imf->chunk_order = chunk_order; + for (i = 0; i < imf->chunk_count; i++) + { + imf->chunks[i].position = INT64_MAX; + imf->chunks[i].chunk_size = 0; + imf->chunk_order[i] = i; + } + chunk_size = 2 * sizeof (struct Chunk) + SHA1_HASH_SIZE; + e = imf_find_space (imf, &position, chunk_size); + if (e == 0) + { + imf->chunks[0].position = position; + imf->chunks[0].chunk_size = chunk_size; + e = imf_sync (imf); + } + } + } + } + } + return e; +} + +int imf_open(struct IndexedMemoryFile *imf, const char *filename) +{ + int e; + int filedesc; + struct flock fl; + int32_t data_size; + struct Chunk *chunks; + int32_t chunk_count; + int32_t *chunk_order; + int i; + e = imf->filedesc != -1 || imf->chunk_count != 0; + if (e == 0) { + filedesc = open(filename, O_RDWR, 0); + e = filedesc == -1; + if (e == 0) { + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; // Lock entire file + e = fcntl(filedesc, F_SETLK, &fl); + if (e == 0) { + data_size = sizeof(struct Chunk) * 2; + chunks = malloc(data_size); + e = chunks == NULL; + if (e == 0) { + imf->filedesc = filedesc; + e = imf_read(imf, chunks, 0, data_size); + if (e == 0) { + chunk_count = (chunks[1].chunk_size - SHA1_HASH_SIZE) / sizeof(struct Chunk) + 2; + data_size = sizeof(struct Chunk) * chunk_count; + chunks = realloc(chunks, data_size); + e = chunks == NULL; + if (e == 0) { + data_size = chunks[1].chunk_size - SHA1_HASH_SIZE; + e = imf_read(imf, chunks + 2, chunks[1].position, data_size); + if (e == 0) { + data_size = sizeof(int32_t) * chunk_count; + chunk_order = malloc(data_size); + e = chunk_order == NULL; + if (e == 0) { + imf->chunks = chunks; + imf->chunk_count = chunk_count; + imf->chunk_order = chunk_order; + for (i = 0; i < imf->chunk_count; i++) + imf->chunk_order[i] = i; + imf_sort_order(imf); + } + } + } + } + } + } + else + e = 0x0556e9f3; // IMFOFF - imf_open fcntl (failed) + } + else + e = 0x059fe56c; // IMFOOF - imf_open open (failed) + } + else + e = 0x052e5e3e; // IMFOAF - imf_open assert (failed) + return e; +} + +int imf_seek_unused (struct IndexedMemoryFile *imf, int32_t *index) +{ + int e; + int32_t i; + int32_t chunk_size; + assert (imf->chunk_count > 0); + for (i = 0; i < imf->chunk_count; i++) + { + chunk_size = imf->chunks[i].chunk_size; + e = chunk_size != 0; + if (e == 0) + { + *index = i; + break; + } + } + assert (e == 0); + // alloc at least one "ending chunk" + if (i == (imf->chunk_count - 1)) + { + e = imf_alloc_chunks (imf); + } + return e; +} + +int32_t imf_get_size (struct IndexedMemoryFile *imf, int32_t index) +{ + int32_t data_size; + assert (index >= 0 && index < imf->chunk_count && imf->chunks[index].chunk_size > 0); + data_size = imf->chunks[index].chunk_size - SHA1_HASH_SIZE; + return data_size; +} + +int imf_get (struct IndexedMemoryFile *imf, int32_t index, void *data) +{ + int e; + int32_t data_size; + int64_t position; + data_size = imf_get_size (imf, index); + position = imf->chunks[index].position; + e = imf_read (imf, data, position, data_size); + return e; +} + +static int +imf_prepare_free(struct IndexedMemoryFile *imf) +{ + int e; + size_t size; + int32_t *delete_mark; + size = sizeof(int32_t) * (imf->delete_end + 1); + delete_mark = realloc(imf->delete_mark, size); + e = delete_mark == NULL; + if (e == 0) + imf->delete_mark = delete_mark; + return e; +} + +static void +imf_free(struct IndexedMemoryFile *imf, int32_t index) +{ + assert(imf->chunks[index].chunk_size != 0); + imf->delete_mark[imf->delete_end++] = index; +} + +int imf_delete(struct IndexedMemoryFile *imf, int32_t index) +{ + int e; + e = imf_prepare_free(imf); + if (e == 0) + imf_free(imf, index); + return e; +} + +int imf_put (struct IndexedMemoryFile *imf, int32_t index, void *data, int32_t data_size) +{ + int e; + int64_t position; + int32_t chunk_size; + int32_t free_index; + assert (index > 1); + chunk_size = data_size + SHA1_HASH_SIZE; + e = imf_find_space (imf, &position, chunk_size); + if (e == 0) + { + e = imf_write (imf, data, position, data_size); + if (e == 0) + { + e = imf->chunks[index].chunk_size > 0; + if (e != 0) + { + e = imf_seek_unused (imf, &free_index); + if (e == 0) + { + e = imf_prepare_free (imf); + if (e == 0) + { + imf->chunks[free_index].position = imf->chunks[index].position; + imf->chunks[free_index].chunk_size = imf->chunks[index].chunk_size; + imf_free (imf, free_index); + } + } + } + if (e == 0) + { + imf->chunks[index].position = position; + imf->chunks[index].chunk_size = chunk_size; + imf_sort_order (imf); + } + } + } + return e; +} + +int imf_sync(struct IndexedMemoryFile *imf) +{ + int e; + int i; + int32_t del_index; + int64_t position; + int32_t data_size; + int64_t space; + void *data; + int32_t chunk_size; + int32_t index; + int32_t r_index; // right + e = 0; + for (i = 1; i < (imf->chunk_count - 1) && e == 0; i++) { + index = imf->chunk_order[i]; + if (index != 1) { + r_index = imf->chunk_order[i+1]; + assert(r_index < imf->chunk_count); + chunk_size = imf->chunks[index].chunk_size; + space = imf->chunks[r_index].position - imf->chunks[index].position - chunk_size; + assert(space >= 0); + if (space > 0) { + if (space < chunk_size) { + data_size = imf_get_size(imf, index); + data = malloc(data_size); + e = data == NULL; + if (e == 0) { + e = imf_get(imf, index, data); + if (e == 0) { + e = imf_put(imf, index, data, data_size); + i = INT32_MAX - 1; + } + free(data); + } + } + } + } + } + if (e == 0) { + data_size = sizeof (struct Chunk) * (imf->chunk_count - 2); + e = imf_find_space (imf, &position, data_size + SHA1_HASH_SIZE); + if (e == 0) + { + imf->chunks[1].position = position; + imf->chunks[1].chunk_size = data_size + SHA1_HASH_SIZE; + // free the deleted chunks + if (imf->delete_end > 0) + { + for (i = 0; i < imf->delete_end; i++) + { + del_index = imf->delete_mark[i]; + imf->chunks[del_index].position = INT64_MAX; + imf->chunks[del_index].chunk_size = 0; + } + imf->delete_end = 0; + } + imf_sort_order (imf); + e = imf_write (imf, imf->chunks + 2, position, data_size); + if (e == 0) + { + data_size = sizeof (struct Chunk) * 2; + assert (imf->chunks[0].position == 0); + e = imf_write (imf, imf->chunks, 0, data_size); + if (e == 0) + { + e = fsync (imf->filedesc); + } + } + } + } + return e; +} + +int imf_close (struct IndexedMemoryFile *imf) +{ + int e; + assert (imf->filedesc != -1); + e = close (imf->filedesc); + assert (e == 0); + if (e == 0) + { + imf->filedesc = -1; + free (imf->chunks); + imf->chunks = NULL; + imf->chunk_count = 0; + free (imf->chunk_order); + imf->chunk_order = NULL; + free (imf->delete_mark); + imf->delete_mark = NULL; + imf->delete_end = 0; + free (imf->stats_gaps_str); + imf->stats_gaps_str = NULL; + } + return e; +} + +// file_length returns the lenght of the used part in the file. +int +imf_get_length (struct IndexedMemoryFile *imf, int64_t *file_length) +{ + int status; + int i; + int32_t chunk_size; + assert (file_length != NULL); + status = (imf->chunk_count <= 0); + if (status == 0) + { + imf_sort_order(imf); + i = imf->chunk_count; + while (i > 0) + { + i--; + chunk_size = imf->chunks[imf->chunk_order[i]].chunk_size; + status = (chunk_size == 0); + if (status == 0) + { + *file_length = imf->chunks[imf->chunk_order[i]].position + chunk_size; + break; + } + } + } + return status; +} +/* +// truncates the file +int +imf_truncate (IndexedMemoryFile *imf) +{ + int status; + int64_t file_length; + status = imf_get_length(imf, &file_length); + if (status == 0) + { + status = ftruncate (imf->filedesc, file_length); + } + return status; +} +*/ +void imf_info_swaps (struct IndexedMemoryFile *imf) +{ + int i; + int dist; + int abs_dist; + int tot_abs_dist; + tot_abs_dist = 0; + for (i = 0; i < imf->chunk_count; i++) + { + dist = i - imf->chunk_order[i]; + abs_dist = abs (dist); + tot_abs_dist += abs_dist; + } + assert ((tot_abs_dist & 1) == 0); + imf->stat_swap = tot_abs_dist / 2; +} + +int imf_info_gaps (struct IndexedMemoryFile *imf) +{ + int e; + int rv; // return value + int s; // stop + struct GapStats* gstats; + int gs_a; // allocated + int gs_i; + int i; + int64_t l_offset; + int64_t r_offset; + int64_t space; + size_t size; + char gap_str[15]; // 9999x999999999/0 + e = 0; + gstats = NULL; + gs_a = 0; + l_offset = 0; + imf->stats_gaps = 0; + imf->stats_gaps_space = 0; + s = 0; + for (i = 0; i < imf->chunk_count && e == 0 && s == 0; i++) + { + r_offset = imf->chunks[imf->chunk_order[i]].position; + s = r_offset == INT64_MAX; + if (s == 0) + { + space = r_offset - l_offset; + assert (space >= 0); + if (space > 0) + { + imf->stats_gaps++; + imf->stats_gaps_space += space; + e = -1; + gs_i = 0; + while (gs_i < gs_a && e != 0) + { + if (gstats[gs_i].gs_size == space) + { + e = 0; + } + else + { + gs_i++; + } + } + if (e == 0) + { + gstats[gs_i].gs_n++; + } + else + { + gs_a++; + size = sizeof (struct GapStats) * gs_a; + gstats = realloc (gstats, size); + e = gstats == NULL; + if (e == 0) + { + gstats[gs_i].gs_size = space; + gstats[gs_i].gs_n = 1; + } + } + } + l_offset = r_offset + imf->chunks[imf->chunk_order[i]].chunk_size; + } + } + for (gs_i = 0; gs_i < gs_a && e == 0; gs_i++) + { + size = sizeof (gap_str); + rv = snprintf (gap_str, size, "%dx%d", gstats[gs_i].gs_n, gstats[gs_i].gs_size); + e = rv < 0 || rv >= size; + if (e == 0) + { + size = 0; + if (imf->stats_gaps_str != NULL) + { + size = strlen (imf->stats_gaps_str) + 2; + } + size += strlen (gap_str) + 1; + imf->stats_gaps_str = realloc (imf->stats_gaps_str, size); + e = imf->stats_gaps_str == NULL; + if (e == 0) + { + if (gs_i == 0) + { + strcpy (imf->stats_gaps_str, gap_str); + } + else + { + strcat (imf->stats_gaps_str, ", "); + strcat (imf->stats_gaps_str, gap_str); + } + } + } + } + free(gstats); + return e; +} + +void sw_init (struct Stopwatch *sw) +{ + sw->sw_time = NULL; + sw->sw_end = NULL; + sw->sw_text = NULL; + sw->sw_c = 0; +} + +int sw_start (char *sw_text, struct Stopwatch *sw) +{ + int e; + int ret; + int sw_i; + size_t size; + assert(sw_text != NULL && sw->sw_c >= 0); + sw_i = sw->sw_c++; + size = sizeof(char*) * sw->sw_c; + sw->sw_text = realloc(sw->sw_text, size); + e = sw->sw_text == NULL; + if (e == 0) { + size = strlen(sw_text) + 1; + sw->sw_text[sw_i] = malloc(size); + e = sw->sw_text[sw_i] == NULL; + if (e == 0) { + strcpy(sw->sw_text[sw_i], sw_text); + size = sizeof(struct timeval) * sw->sw_c; + sw->sw_time = realloc(sw->sw_time, size); + e = sw->sw_time == NULL; + if (e == 0) { + sw->sw_end = realloc(sw->sw_end, size); + e = sw->sw_end == NULL; + if (e == 0) { + e = gettimeofday(&sw->sw_time[sw_i], NULL); + } + } + } + } + ret = e == 0 ? sw_i : -1; + return ret; +} + +int sw_stop (int sw_i, struct Stopwatch *sw) +{ + int e; + assert(sw->sw_c > 0 && sw_i < sw->sw_c); + e = gettimeofday(&sw->sw_end[sw_i], NULL); + return e; +} + +int sw_info (char **sw_info_str, struct Stopwatch *sw) +{ + int e; + int i; + size_t size_a; // allocated + size_t size_p; // printed + int off; + time_t tv_sec; + suseconds_t tv_usec; + char *comma_str; + assert (sw_info_str != NULL); + *sw_info_str = NULL; + e = 0; + size_a = 0; + size_p = 0; + off = 0; + for (i = 0; i < sw->sw_c && e == 0; i++) + { + tv_sec = sw->sw_end[i].tv_sec - sw->sw_time[i].tv_sec; + tv_usec = sw->sw_end[i].tv_usec - sw->sw_time[i].tv_usec; + if (tv_usec < 0) { + tv_sec--; + tv_usec += 1000000; + } + comma_str = i + 1 < sw->sw_c ? ", " : ""; + do + { + if (size_a < off + size_p + 1) + { + size_a = off + size_p + 1; + *sw_info_str = realloc (*sw_info_str, size_a); + e = *sw_info_str == NULL; + } + if (e == 0) + { + size_p = snprintf (*sw_info_str + off, size_a - off, "%s: %d.%06d%s", sw->sw_text[i], (int) tv_sec, (int) tv_usec, comma_str); + } + } while (((size_a - off) < size_p) && (e == 0)); + off += size_p; + size_p = 0; + } + return e; +} + +void sw_free(struct Stopwatch *sw) +{ + int i; + free(sw->sw_time); + sw->sw_time = NULL; + free(sw->sw_end); + sw->sw_end = NULL; + for (i = 0; i < sw->sw_c; i++) + free(sw->sw_text[i]); + sw->sw_c = 0; + free(sw->sw_text); + sw->sw_text = NULL; +} diff --git a/imf/indexedmemoryfile.h b/imf/indexedmemoryfile.h new file mode 100644 index 0000000..8e5f63a --- /dev/null +++ b/imf/indexedmemoryfile.h @@ -0,0 +1,77 @@ + +// +// Author: Lorenz Pullwitt +// Copyright 2016-2020 +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, you can find it here: +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +// + +#include +#include // stopwatch + +#pragma pack(push) +#pragma pack(4) +struct Chunk +{ + int64_t position; + int32_t chunk_size; +}; +#pragma pack(pop) + +struct Stopwatch +{ + struct timeval *sw_time; + struct timeval *sw_end; + char **sw_text; + int sw_c; +}; + +struct IndexedMemoryFile +{ + int filedesc; + struct Chunk *chunks; + int32_t chunk_count; + int32_t *chunk_order; + int32_t *delete_mark; + int delete_end; + struct Stopwatch sw; + int stat_swap; + int stats_gaps; + int stats_gaps_space; + char *stats_gaps_str; +}; + +void imf_init (struct IndexedMemoryFile *imf); +char imf_is_open (struct IndexedMemoryFile *imf); +int imf_create (struct IndexedMemoryFile *imf, const char *filename, int flags_mask); +int imf_open (struct IndexedMemoryFile *imf, const char *filename); +int imf_seek_unused (struct IndexedMemoryFile *imf, int32_t *index); +int32_t imf_get_size (struct IndexedMemoryFile *imf, int32_t index); +int imf_get (struct IndexedMemoryFile *imf, int32_t index, void *data); +int imf_delete (struct IndexedMemoryFile *imf, int32_t index); +int imf_put (struct IndexedMemoryFile *imf, int32_t index, void *data, int32_t data_size); +int imf_sync (struct IndexedMemoryFile *imf); +int imf_close (struct IndexedMemoryFile *imf); +int imf_get_length (struct IndexedMemoryFile *imf, int64_t *file_length); +/* +int imf_truncate (IndexedMemoryFile *imf); +*/ +void imf_info_swaps (struct IndexedMemoryFile *imf); +int imf_info_gaps (struct IndexedMemoryFile *imf); +void sw_init (struct Stopwatch *sw); +int sw_start (char *sw_text, struct Stopwatch *sw); +int sw_stop (int sw_i, struct Stopwatch *sw); +int sw_info (char **sw_info_str, struct Stopwatch *sw); +void sw_free(struct Stopwatch *sw); diff --git a/imf/sha1.c b/imf/sha1.c index 8b13789..15e615e 100644 --- a/imf/sha1.c +++ b/imf/sha1.c @@ -1 +1,318 @@ +// +// Author: Lorenz Pullwitt +// Copyright 2017-2020 +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, you can find it here: +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +// + +// +// Description: +// This file implements the Secure Hashing Algorithm 1 as +// defined in FIPS PUB 180-1 published April 17, 1995. +// +// The SHA-1, produces a 160-bit message digest for a given +// data stream. It should take about 2**n steps to find a +// message with the same digest as a given message and +// 2**(n/2) to find any two messages with the same digest, +// when n is the digest size in bits. Therefore, this +// algorithm can serve as a means of providing a +// "fingerprint" for a message. +// +// Portability Issues: +// SHA-1 is defined in terms of 32-bit "words". This code +// uses (included via "sha1.h" to define 32 and 8 +// bit unsigned integer types. If your C compiler does not +// support 32 bit unsigned integers, this code is not +// appropriate. +// +// Caveats: +// SHA-1 is designed to work with messages less than 2^64 bits +// long. Although SHA-1 allows a message digest to be generated +// for messages of any number of bits less than 2^64, this +// implementation only works with messages with a length that is +// a multiple of the size of an 8-bit character. +// + +#include "sha1.h" +#include +#include + +// Define the SHA1 circular left shift macro +#define SHA1CircularShift(bits,word) (((word) << (bits)) | ((word) >> (32-(bits)))) + +// +// Description: +// This function will initialize the Sha1Context in preparation +// for computing a new SHA1 message digest. +// +// Parameters: +// context: [in/out] +// The context to reset. +// +int sha1_reset (struct Sha1Context *context) +{ + int err; + err = context == NULL; + if (err == 0) + { + context->Length_Low = 0; + context->Length_High = 0; + context->Message_Block_Index = 0; + + context->Intermediate_Hash[0] = 0x67452301; + context->Intermediate_Hash[1] = 0xEFCDAB89; + context->Intermediate_Hash[2] = 0x98BADCFE; + context->Intermediate_Hash[3] = 0x10325476; + context->Intermediate_Hash[4] = 0xC3D2E1F0; + } + return err; +} + +// +// Description: +// This function will process the next 512 bits of the message +// stored in the message_block array. +// +// Comments: +// Many of the variable names in this code, especially the +// single character names, were used because those were the +// names used in the publication. +// +void SHA1ProcessMessageBlock (struct Sha1Context *context) +{ + const uint32_t K[] = { 0x5A827999, 0x6ED9EBA1, 0x8F1BBCDC, 0xCA62C1D6 }; // Constants defined in SHA-1 + int t; // Loop counter + uint32_t temp; // Temporary word value + uint32_t W[80]; // Word sequence + uint32_t A, B, C, D, E; // Word buffers + + uint32_t W_t; // temp + + // + // Initialize the first 16 words in the array W + // + for (t = 0; t < 16; t++) + { + W[t] = context->message_block[t * 4] << 24; + W[t] |= context->message_block[t * 4 + 1] << 16; + W[t] |= context->message_block[t * 4 + 2] << 8; + W[t] |= context->message_block[t * 4 + 3]; + } + + for (t = 16; t < 80; t++) + { + W_t = W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]; + W[t] = SHA1CircularShift(1,W_t); + } + + A = context->Intermediate_Hash[0]; + B = context->Intermediate_Hash[1]; + C = context->Intermediate_Hash[2]; + D = context->Intermediate_Hash[3]; + E = context->Intermediate_Hash[4]; + + for (t = 0; t < 20; t++) + { + temp = SHA1CircularShift(5,A) + ((B & C) | ((~B) & D)) + E + W[t] + K[0]; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for (t = 20; t < 40; t++) + { + temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1]; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for (t = 40; t < 60; t++) + { + temp = SHA1CircularShift(5,A) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + for (t = 60; t < 80; t++) + { + temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3]; + E = D; + D = C; + C = SHA1CircularShift(30,B); + B = A; + A = temp; + } + + context->Intermediate_Hash[0] += A; + context->Intermediate_Hash[1] += B; + context->Intermediate_Hash[2] += C; + context->Intermediate_Hash[3] += D; + context->Intermediate_Hash[4] += E; + + context->Message_Block_Index = 0; +} + +// +// Description: +// This function accepts an array of octets as the next portion +// of the message. +// +// Parameters: +// context: [in/out] +// The SHA context to update +// message_array: [in] +// An array of characters representing the next portion of +// the message. +// length: [in] +// The length of the message in message_array +// +int sha1_input (struct Sha1Context *context, const uint8_t *message_array, unsigned int length) +{ + int err; + err = 0; + if (length > 0) + { + err = (context == NULL) || (message_array == NULL); + while (length-- && err == 0) + { + context->message_block[context->Message_Block_Index++] = *message_array; + assert ((*message_array) == (*message_array & 0xFF)); + context->Length_Low += 8; + if (context->Length_Low == 0) + { + context->Length_High++; + err = context->Length_High == 0; // Message is too long + } + if (context->Message_Block_Index == 64) + { + SHA1ProcessMessageBlock (context); + } + message_array++; + } + } + return err; +} + +// +// Description: +// According to the standard, the message must be padded to an even +// 512 bits. The first padding bit must be a '1'. The last 64 +// bits represent the length of the original message. All bits in +// between should be 0. This function will pad the message +// according to those rules by filling the message_block array +// accordingly. It will also call the ProcessMessageBlock function +// provided appropriately. When it returns, it can be assumed that +// the message digest has been computed. +// +// Parameters: +// context: [in/out] +// The context to pad +// ProcessMessageBlock: [in] +// The appropriate SHA*ProcessMessageBlock function +// +void SHA1PadMessage (struct Sha1Context *context) +{ + // + // Check to see if the current message block is too small to hold + // the initial padding bits and length. If so, we will pad the + // block, process it, and then continue padding into a second + // block. + // + if (context->Message_Block_Index > 55) + { + context->message_block[context->Message_Block_Index++] = 0x80; + while(context->Message_Block_Index < 64) + { + context->message_block[context->Message_Block_Index++] = 0; + } + + SHA1ProcessMessageBlock(context); + + while(context->Message_Block_Index < 56) + { + context->message_block[context->Message_Block_Index++] = 0; + } + } + else + { + context->message_block[context->Message_Block_Index++] = 0x80; + while(context->Message_Block_Index < 56) + { + context->message_block[context->Message_Block_Index++] = 0; + } + } + + // + // Store the message length as the last 8 octets + // + context->message_block[56] = context->Length_High >> 24; + context->message_block[57] = context->Length_High >> 16; + context->message_block[58] = context->Length_High >> 8; + context->message_block[59] = context->Length_High; + context->message_block[60] = context->Length_Low >> 24; + context->message_block[61] = context->Length_Low >> 16; + context->message_block[62] = context->Length_Low >> 8; + context->message_block[63] = context->Length_Low; + + SHA1ProcessMessageBlock(context); +} + +// +// Description: +// This function will return the 160-bit message digest into the +// message_digest array provided by the caller. +// NOTE: The first octet of hash is stored in the 0th element, +// the last octet of hash in the 19th element. +// +// Parameters: +// context: [in/out] +// The context to use to calculate the SHA-1 hash. +// message_digest: [out] +// Where the digest is returned. +// +int sha1_result (struct Sha1Context *context, uint8_t message_digest[SHA1_HASH_SIZE]) +{ + int err; + int i; + err = (context == NULL) || (message_digest == NULL); + if (err == 0) + { + SHA1PadMessage (context); + } + + for (i=0; i<64; ++i) + { + // message may be sensitive, clear it out + context->message_block[i] = 0; + } + context->Length_Low = 0; // and clear length + context->Length_High = 0; + + for (i = 0; i < SHA1_HASH_SIZE; ++i) + { + message_digest[i] = context->Intermediate_Hash[i>>2] >> 8 * ( 3 - ( i & 0x03 ) ); + } + + return err; +} diff --git a/imf/sha1.h b/imf/sha1.h new file mode 100644 index 0000000..a5af5a6 --- /dev/null +++ b/imf/sha1.h @@ -0,0 +1,54 @@ + +// +// Author: Lorenz Pullwitt +// Copyright 2017-2020 +// +// This program is free software; you can redistribute it and/or +// modify it under the terms of the GNU General Public License +// as published by the Free Software Foundation; either version 2 +// of the License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, you can find it here: +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +// + +// +// Description: +// This is the header file for code which implements the Secure +// Hashing Algorithm 1 as defined in FIPS PUB 180-1 published +// April 17, 1995. +// +// Many of the variable names in this code, especially the +// single character names, were used because those were the names +// used in the publication. +// +// Please read the file sha1.c for more information. +// + +#include + +enum { SHA1_HASH_SIZE = 20 }; + +// +// This structure will hold context information for the SHA-1 hashing operation +// +struct Sha1Context +{ + uint32_t Intermediate_Hash[SHA1_HASH_SIZE/4]; // Message Digest + + uint32_t Length_Low; // Message length in bits + uint32_t Length_High; // Message length in bits + + int_least16_t Message_Block_Index; // Index into message block array + uint8_t message_block[64]; // 512-bit message blocks +}; + +int sha1_reset (struct Sha1Context *context); +int sha1_input (struct Sha1Context *context, const uint8_t *message_array, unsigned int length); +int sha1_result (struct Sha1Context *context, uint8_t message_digest[SHA1_HASH_SIZE]);