diff --git a/README.md b/README.md index 4184b95..be27061 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # tellydb -A key-value database project for educational purposes. +An in-memory key-value database project for educational purposes. ## Features + Follows [RESP2/RESP3](https://redis.io/docs/latest/develop/reference/protocol-spec/) specification from redis, so all redis clients are compatible @@ -9,7 +9,8 @@ A key-value database project for educational purposes. + Supports integer, string, null, boolean, list and hash table types + Provides atomicity when saving to the database file + Provides saving to the database file using a background thread -+ Provides authorization system with permissions ++ Provides authorization system with permissions using passwords ++ Uses Direct I/O for logging and database files > Look at: > [docs/SPECS.md](./docs/SPECS.md) for more technical information, diff --git a/docs/AUTH.md b/docs/AUTH.md index 236b00d..0a3cc4a 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -5,7 +5,11 @@ This file defines authorization system concepts. If a client use a password defined in the server with `AUTH`, this client have all permissions of the password. ## Limitations -Server can have up to `2^6-1` passwords. +* Server can have up to `2^6-1` passwords. + +## Notes +* Password deriving algorithm is HKDF SHA384. +* The length of derived passwords is 48. ## Permissions * `P_READ`, read a value from database, not included data type and data existence diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 0d513c8..5f64c1d 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -517,6 +517,19 @@ APPEND user_name " Black" DECR user_age ``` +#### DEL +**Syntax**: `DEL key [key ...]` +**Description**: Deletes the specified keys. +**Since**: `0.1.7` +**Time complexity**: `O(N) where N is key count` +**Permissions**: `P_WRITE` +**Returns**: Deleted key count + +**Example**: +```shell +DEL user_name user_id +``` + --- ### EXISTS @@ -572,6 +585,23 @@ INCR user_age --- +#### RENAME +**Syntax**: `RENAME old new` +**Description**: Renames existing key to new key. +**Since**: `0.1.7` +**Time complexity**: `O(1)` +**Permissions**: `P_WRITE` +**Returns**: `OK` or null reply if key is ntot exist +**Behavior**: +* If new key already exists, throws an error. + +**Example**: +```shell +RENAME name user_name +``` + +--- + ### SET **Syntax**: `SET key value [NX|XX] [GET]` **Description**: Sets value. diff --git a/docs/FILE.md b/docs/FILE.md index 2945ec3..a0ec604 100644 --- a/docs/FILE.md +++ b/docs/FILE.md @@ -28,7 +28,7 @@ Authorization part consists of passwords and their permissions and as follows: * `password count byte count (1 byte, n) + password count (n byte) + passwords` A password is as follows: -* `string length specifier + hashed password + password salt (2 bytes) + password permissions (1 byte)` +* `derived password (48 byte) + password permissions (1 byte)` For permissions, look at [AUTH.md](./AUTH.md). diff --git a/headers/commands.h b/headers/commands.h index ba5bd0e..125bdfe 100644 --- a/headers/commands.h +++ b/headers/commands.h @@ -62,9 +62,11 @@ extern const struct Command cmd_htype; /* KV COMMANDS */ extern const struct Command cmd_append; extern const struct Command cmd_decr; +extern const struct Command cmd_del; extern const struct Command cmd_get; extern const struct Command cmd_exists; extern const struct Command cmd_incr; +extern const struct Command cmd_rename; extern const struct Command cmd_set; extern const struct Command cmd_type; /* /KV COMMANDS */ diff --git a/headers/database.h b/headers/database.h index d9f2540..e596b61 100644 --- a/headers/database.h +++ b/headers/database.h @@ -15,33 +15,29 @@ struct KVPair { string_t key; void *value; enum TellyTypes type; - - struct { - off_t start_at; - off_t end_at; - } pos; }; struct BTree *create_cache(); struct BTree *get_cache(); -struct KVPair *get_kv_from_cache(const char *key); -struct BTreeValue **get_sorted_kvs_by_pos_as_values(uint32_t *size); -bool delete_kv_from_cache(const char *key); +struct KVPair *get_kv_from_cache(const char *key, const size_t length); +bool delete_kv_from_cache(const char *key, const size_t length); void free_cache(); -void get_all_keys(off_t from); -struct KVPair *get_data(const char *key); +void get_all_data_from_file(const int fd, const off64_t file_size, char *block, const uint16_t block_size, const uint16_t filled_block_size); +struct KVPair *get_data(const string_t key); struct KVPair *set_data(struct KVPair *data, const string_t key, void *value, const enum TellyTypes type); +bool delete_data(const string_t key); void save_data(const uint64_t server_age); bool bg_save(uint64_t server_age); -void set_kv(struct KVPair *kv, const string_t key, void *value, const enum TellyTypes type, const off_t start_at, const off_t end_at); +void set_kv(struct KVPair *kv, const string_t key, void *value, const enum TellyTypes type); void free_kv(struct KVPair *kv); /* /DATABASE */ /* DATABASE FILE */ bool open_database_fd(const char *filename, uint64_t *server_age); +uint16_t get_block_size(); int get_database_fd(); void close_database_fd(); /* /DATABASE FILE */ diff --git a/headers/hashtable.h b/headers/hashtable.h index 175eb8f..76dff11 100644 --- a/headers/hashtable.h +++ b/headers/hashtable.h @@ -28,7 +28,7 @@ struct HashTable { struct HashTable *create_hashtable(uint32_t default_size); void resize_hashtable(struct HashTable *table, const uint32_t size); -struct FVPair *get_fv_from_hashtable(struct HashTable *table, char *name); +struct FVPair *get_fv_from_hashtable(struct HashTable *table, const string_t name); void free_hashtable(struct HashTable *table); void free_fv(struct FVPair *fv); diff --git a/headers/server.h b/headers/server.h index 38c37d1..2152a2b 100644 --- a/headers/server.h +++ b/headers/server.h @@ -63,9 +63,8 @@ enum Permissions { }; struct Password { - string_t data; + unsigned char data[48]; uint8_t permissions; - char salt[3]; }; void create_constant_passwords(); @@ -73,16 +72,19 @@ void free_constant_passwords(); struct Password *get_full_password(); struct Password *get_empty_password(); +bool initialize_kdf(); +void free_kdf(); + struct Password **get_passwords(); uint32_t get_password_count(); -off_t get_authorization_from_file(const int fd); +uint16_t get_authorization_from_file(const int fd, char *block, const uint16_t block_size); void free_passwords(); void add_password(struct Client *client, const string_t data, const uint8_t permissions); -bool remove_password(struct Client *executor, const char *value); -int32_t where_password(const char *value); -struct Password *get_password(const char *value); -bool edit_password(const char *value, const uint32_t permissions); +bool remove_password(struct Client *executor, char *value, const size_t value_len); +int32_t where_password(char *value, const size_t value_len); +struct Password *get_password(char *value, const size_t value_len); +bool edit_password(char *value, const size_t value_len, const uint32_t permissions); /* /AUTH */ diff --git a/headers/utils.h b/headers/utils.h index 28f5c55..68df56a 100644 --- a/headers/utils.h +++ b/headers/utils.h @@ -45,4 +45,4 @@ void generate_date_string(char *text, const time_t value); int open_file(const char *file, int flags); -uint64_t hash(char *key); +uint64_t hash(char *key, uint32_t length); diff --git a/src/commands/generic/auth.c b/src/commands/generic/auth.c index 6e1edc2..17ea32e 100644 --- a/src/commands/generic/auth.c +++ b/src/commands/generic/auth.c @@ -13,10 +13,10 @@ static void run(struct Client *client, commanddata_t *command, struct Password * return; } - const char *value = command->args[0].value; - const int32_t at = where_password(value); + const string_t input = command->args[0]; + struct Password *found = get_password(input.value, input.len); - if (at == -1) { + if (!found) { _write(client, "-This password does not exist\r\n", 31); } else { if (password && password != get_empty_password()) { @@ -33,9 +33,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } } - struct Password **passwords = get_passwords(); - client->password = passwords[at]; - + client->password = found; WRITE_OK(client); } } diff --git a/src/commands/generic/pwd.c b/src/commands/generic/pwd.c index 6d40671..2f0309d 100644 --- a/src/commands/generic/pwd.c +++ b/src/commands/generic/pwd.c @@ -78,7 +78,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * return; } - if (where_password(data.value) == -1) { + if (where_password(data.value, data.len) == -1) { add_password(client, data, permissions); WRITE_OK(client); } else { @@ -90,10 +90,10 @@ static void run(struct Client *client, commanddata_t *command, struct Password * return; } - const char *value = command->args[1].value; + const string_t input = command->args[1]; char *permissions_value = command->args[2].value; - struct Password *target = get_password(value); + struct Password *target = get_password(input.value, input.len); if (target) { const uint8_t permissions = read_permissions_value(client, permissions_value); @@ -115,9 +115,9 @@ static void run(struct Client *client, commanddata_t *command, struct Password * return; } - const char *value = command->args[1].value; + const string_t input = command->args[1]; - if (remove_password(client, value)) WRITE_OK(client); + if (remove_password(client, input.value, input.len)) WRITE_OK(client); else { _write(client, "-This password cannot be found\r\n", 32); } diff --git a/src/commands/hashtable/hdel.c b/src/commands/hashtable/hdel.c index b669f35..de4a124 100644 --- a/src/commands/hashtable/hdel.c +++ b/src/commands/hashtable/hdel.c @@ -16,7 +16,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & P_WRITE) { const string_t key = command->args[0]; - struct KVPair *kv = get_data(key.value); + struct KVPair *kv = get_data(key); struct HashTable *table; if (kv) { @@ -39,6 +39,8 @@ static void run(struct Client *client, commanddata_t *command, struct Password * del_fv_to_hashtable(table, command->args[i]); } + if (table->size.all == 0) delete_data(key); + char buf[14]; const size_t nbytes = sprintf(buf, ":%d\r\n", old_size - table->size.all); _write(client, buf, nbytes); @@ -49,6 +51,8 @@ static void run(struct Client *client, commanddata_t *command, struct Password * for (uint32_t i = 1; i < command->arg_count; ++i) { del_fv_to_hashtable(table, command->args[i]); } + + if (table->size.all == 0) delete_data(key); } } else if (client) { _write(client, "-Not allowed to use this command, need P_WRITE\r\n", 48); diff --git a/src/commands/hashtable/hget.c b/src/commands/hashtable/hget.c index 9301c3a..c20dc53 100644 --- a/src/commands/hashtable/hget.c +++ b/src/commands/hashtable/hget.c @@ -14,12 +14,11 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const struct KVPair *kv = get_data(command->args[0].value); + const struct KVPair *kv = get_data(command->args[0]); if (kv) { if (kv->type == TELLY_HASHTABLE) { - char *name = command->args[1].value; - const struct FVPair *field = get_fv_from_hashtable(kv->value, name); + const struct FVPair *field = get_fv_from_hashtable(kv->value, command->args[1]); if (field) { write_value(client, field->value, field->type); diff --git a/src/commands/hashtable/hlen.c b/src/commands/hashtable/hlen.c index ba002f1..d86ffc2 100644 --- a/src/commands/hashtable/hlen.c +++ b/src/commands/hashtable/hlen.c @@ -15,7 +15,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const struct KVPair *kv = get_data(command->args[0].value); + const struct KVPair *kv = get_data(command->args[0]); if (kv) { if (kv->type == TELLY_HASHTABLE) { diff --git a/src/commands/hashtable/hset.c b/src/commands/hashtable/hset.c index c63a8dc..a4b73fd 100644 --- a/src/commands/hashtable/hset.c +++ b/src/commands/hashtable/hset.c @@ -18,7 +18,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } const string_t key = command->args[0]; - struct KVPair *kv = get_data(key.value); + struct KVPair *kv = get_data(key); struct HashTable *table; if (kv) { diff --git a/src/commands/hashtable/htype.c b/src/commands/hashtable/htype.c index 0dee930..9d04ca2 100644 --- a/src/commands/hashtable/htype.c +++ b/src/commands/hashtable/htype.c @@ -14,13 +14,11 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const char *key = command->args[0].value; - const struct KVPair *kv = get_data(key); + const struct KVPair *kv = get_data(command->args[0]); if (kv) { if (kv->type == TELLY_HASHTABLE) { - char *name = command->args[1].value; - struct FVPair *fv = get_fv_from_hashtable(kv->value, name); + struct FVPair *fv = get_fv_from_hashtable(kv->value, command->args[1]); switch (fv->type) { case TELLY_NULL: diff --git a/src/commands/kv/append.c b/src/commands/kv/append.c index e3484ec..3bc4d7e 100644 --- a/src/commands/kv/append.c +++ b/src/commands/kv/append.c @@ -12,7 +12,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & (P_READ | P_WRITE)) { const string_t key = command->args[0]; - const struct KVPair *kv = get_data(key.value); + const struct KVPair *kv = get_data(key); if (kv) { if (kv->type == TELLY_STR) { diff --git a/src/commands/kv/decr.c b/src/commands/kv/decr.c index d8f8199..8cc1add 100644 --- a/src/commands/kv/decr.c +++ b/src/commands/kv/decr.c @@ -15,7 +15,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & (P_READ | P_WRITE)) { string_t key = command->args[0]; - struct KVPair *result = get_data(key.value); + struct KVPair *result = get_data(key); if (!result) { long *number = calloc(1, sizeof(long)); diff --git a/src/commands/kv/del.c b/src/commands/kv/del.c new file mode 100644 index 0000000..965b50d --- /dev/null +++ b/src/commands/kv/del.c @@ -0,0 +1,40 @@ +#include "../../../headers/server.h" +#include "../../../headers/database.h" +#include "../../../headers/commands.h" + +#include +#include + +static void run(struct Client *client, commanddata_t *command, struct Password *password) { + if (command->arg_count == 0) { + if (client) WRONG_ARGUMENT_ERROR(client, "DEL", 3); + return; + } + + if (password->permissions & P_WRITE) { + uint32_t deleted = 0; + + for (uint32_t i = 0; i < command->arg_count; ++i) { + deleted += delete_data(command->args[i]); + } + + if (client) { + char res[13]; + const size_t res_len = sprintf(res, ":%d\r\n", deleted); + + _write(client, res, res_len); + } + } else if (client) { + _write(client, "-Not allowed to use this command, need P_WRITE\r\n", 48); + } +} + +const struct Command cmd_del = { + .name = "DEL", + .summary = "Deletes the specified keys.", + .since = "0.1.7", + .complexity = "O(N) where N is key count", + .subcommands = NULL, + .subcommand_count = 0, + .run = run +}; diff --git a/src/commands/kv/exists.c b/src/commands/kv/exists.c index 4e0b7d6..3585121 100644 --- a/src/commands/kv/exists.c +++ b/src/commands/kv/exists.c @@ -20,7 +20,7 @@ static void run(struct Client *client, commanddata_t *command, __attribute__((un buf[0] = '\0'; for (uint32_t i = 0; i < command->arg_count; ++i) { - const char *key = command->args[i].value; + const string_t key = command->args[i]; if (get_data(key)) { existed += 1; diff --git a/src/commands/kv/get.c b/src/commands/kv/get.c index f69423a..3f5eaaa 100644 --- a/src/commands/kv/get.c +++ b/src/commands/kv/get.c @@ -12,8 +12,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const char *key = command->args[0].value; - const struct KVPair *kv = get_data(key); + const struct KVPair *kv = get_data(command->args[0]); if (kv) { write_value(client, kv->value, kv->type); diff --git a/src/commands/kv/incr.c b/src/commands/kv/incr.c index 6e0b459..3cb6f6f 100644 --- a/src/commands/kv/incr.c +++ b/src/commands/kv/incr.c @@ -15,7 +15,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & (P_READ | P_WRITE)) { const string_t key = command->args[0]; - struct KVPair *result = get_data(key.value); + struct KVPair *result = get_data(key); if (!result) { long *number = calloc(1, sizeof(long)); diff --git a/src/commands/kv/rename.c b/src/commands/kv/rename.c new file mode 100644 index 0000000..862600a --- /dev/null +++ b/src/commands/kv/rename.c @@ -0,0 +1,57 @@ +#include "../../../headers/server.h" +#include "../../../headers/database.h" +#include "../../../headers/commands.h" + +#include +#include + +static void run(struct Client *client, commanddata_t *command, struct Password *password) { + if (command->arg_count != 2) { + if (client) WRONG_ARGUMENT_ERROR(client, "RENAME", 6); + return; + } + + // TODO: conflict kv names + if (password->permissions & P_WRITE) { + const string_t search = command->args[0]; + const uint64_t index = hash(search.value, search.len); + + struct BTreeValue *value = find_value_from_btree(get_cache(), index); + + if (value) { + struct KVPair *kv = value->data; + + if ((search.len == kv->key.len) && (strncmp(search.value, kv->key.value, search.len) == 0)) { + string_t *old = &kv->key; + const string_t new = command->args[1]; + + if (get_kv_from_cache(new.value, new.len)) { + _write(client, "-The new key already exists\r\n", 26); + return; + } + + value->index = hash(new.value, new.len); + old->len = new.len; + old->value = realloc(old->value, new.len); + memcpy(old->value, new.value, new.len); + + if (client) WRITE_OK(client); + return; + } + } + + if (client) WRITE_NULL_REPLY(client); + } else if (client) { + _write(client, "-Not allowed to use this command, need P_WRITE\r\n", 48); + } +} + +const struct Command cmd_rename = { + .name = "RENAME", + .summary = "Renames existing key to new key.", + .since = "0.1.7", + .complexity = "O(1)", + .subcommands = NULL, + .subcommand_count = 0, + .run = run +}; diff --git a/src/commands/kv/set.c b/src/commands/kv/set.c index 52c523a..29378a4 100644 --- a/src/commands/kv/set.c +++ b/src/commands/kv/set.c @@ -39,7 +39,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * const string_t key = command->args[0]; void *value; enum TellyTypes type; - struct KVPair *res = get_data(key.value); + struct KVPair *res = get_data(key); if (nx && res) { if (client) WRITE_NULL_REPLY(client); diff --git a/src/commands/kv/type.c b/src/commands/kv/type.c index 1654ff5..448b845 100644 --- a/src/commands/kv/type.c +++ b/src/commands/kv/type.c @@ -11,8 +11,7 @@ static void run(struct Client *client, commanddata_t *command, __attribute__((un return; } - const char *key = command->args[0].value; - struct KVPair *res = get_data(key); + struct KVPair *res = get_data(command->args[0]); if (res) { switch (res->type) { diff --git a/src/commands/list/lindex.c b/src/commands/list/lindex.c index 2c29547..9d00231 100644 --- a/src/commands/list/lindex.c +++ b/src/commands/list/lindex.c @@ -14,8 +14,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const char *key = command->args[0].value; - const struct KVPair *kv = get_data(key); + const struct KVPair *kv = get_data(command->args[0]); if (!kv || kv->type != TELLY_LIST) { _write(client, "-Value stored at the key is not a list\r\n", 40); diff --git a/src/commands/list/llen.c b/src/commands/list/llen.c index d965122..9b6ae2f 100644 --- a/src/commands/list/llen.c +++ b/src/commands/list/llen.c @@ -15,8 +15,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & P_READ) { - const char *key = command->args[0].value; - const struct KVPair *kv = get_data(key); + const struct KVPair *kv = get_data(command->args[0]); if (!kv) { _write(client, ":0\r\n", 4); diff --git a/src/commands/list/lpop.c b/src/commands/list/lpop.c index 09c9c5d..b0ab98a 100644 --- a/src/commands/list/lpop.c +++ b/src/commands/list/lpop.c @@ -12,7 +12,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if(password->permissions & (P_READ | P_WRITE)) { - const char *key = command->args[0].value; + const string_t key = command->args[0]; const struct KVPair *kv = get_data(key); if (kv) { @@ -27,8 +27,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (client) write_value(client, node->value, node->type); if (list->size == 1) { - // TODO: complete deletion of the list - delete_kv_from_cache(key); + delete_data(key); } else { list->begin = list->begin->next; list->begin->prev = NULL; diff --git a/src/commands/list/lpush.c b/src/commands/list/lpush.c index a5df8ba..30a7605 100644 --- a/src/commands/list/lpush.c +++ b/src/commands/list/lpush.c @@ -29,7 +29,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & P_WRITE) { const string_t key = command->args[0]; - struct KVPair *kv = get_data(key.value); + struct KVPair *kv = get_data(key); struct List *list; if (kv) { diff --git a/src/commands/list/rpop.c b/src/commands/list/rpop.c index 0257c70..f9516ed 100644 --- a/src/commands/list/rpop.c +++ b/src/commands/list/rpop.c @@ -12,7 +12,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * } if (password->permissions & (P_READ & P_WRITE)) { - const char *key = command->args[0].value; + const string_t key = command->args[0]; struct KVPair *kv = get_data(key); if (kv) { @@ -27,8 +27,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (client) write_value(client, list->end->value, list->end->type); if (list->size == 1) { - // TODO: complete deletion of the list - delete_kv_from_cache(key); + delete_data(key); } else { list->end = list->end->prev; list->end->next = NULL; diff --git a/src/commands/list/rpush.c b/src/commands/list/rpush.c index 161b8be..1100efa 100644 --- a/src/commands/list/rpush.c +++ b/src/commands/list/rpush.c @@ -29,7 +29,7 @@ static void run(struct Client *client, commanddata_t *command, struct Password * if (password->permissions & (P_READ | P_WRITE)) { const string_t key = command->args[0]; - struct KVPair *kv = get_data(key.value); + struct KVPair *kv = get_data(key); struct List *list; if (kv) { diff --git a/src/database/cache.c b/src/database/cache.c index 04a4e19..bbd0968 100644 --- a/src/database/cache.c +++ b/src/database/cache.c @@ -2,6 +2,7 @@ #include "../../headers/btree.h" #include +#include #include static struct BTree *cache = NULL; @@ -14,8 +15,8 @@ struct BTree *get_cache() { return cache; } -struct KVPair *get_kv_from_cache(const char *key) { - uint64_t index = hash((char *) key); +struct KVPair *get_kv_from_cache(const char *key, const size_t length) { + uint64_t index = hash((char *) key, length); while (true) { struct BTreeValue *value = find_value_from_btree(cache, index); @@ -23,39 +24,17 @@ struct KVPair *get_kv_from_cache(const char *key) { if (value) { struct KVPair *kv = value->data; - if (streq(kv->key.value, key)) return kv; + if (strncmp(kv->key.value, key, length) == 0) return kv; else index += 1; } else return NULL; } } -bool delete_kv_from_cache(const char *key) { - const uint64_t index = hash((char *) key); +bool delete_kv_from_cache(const char *key, const size_t length) { + const uint64_t index = hash((char *) key, length); return delete_value_from_btree(cache, index, (void (*)(void *)) free_kv); } void free_cache() { free_btree(cache, (void (*)(void *)) free_kv); } - -struct BTreeValue **get_sorted_kvs_by_pos_as_values(uint32_t *size) { - struct BTreeValue **values = get_values_from_btree(cache, size); - if (*size <= 1) return values; - - const uint32_t bound_parent = *size - 1; - - for (uint32_t i = 0; i < bound_parent; ++i) { - const uint32_t bound = *size - 1 - i; - - for (uint32_t j = 0; j < bound; ++j) { - struct BTreeValue *a = values[j]; - struct BTreeValue *b = values[j + 1]; - - if (((struct KVPair *) a->data)->pos.start_at <= ((struct KVPair *) b->data)->pos.start_at) continue; - values[j + 1] = a; - values[j] = b; - } - } - - return values; -} diff --git a/src/database/delete.c b/src/database/delete.c new file mode 100644 index 0000000..383dc6f --- /dev/null +++ b/src/database/delete.c @@ -0,0 +1,8 @@ +#include "../../headers/database.h" +#include "../../headers/utils.h" + +#include + +bool delete_data(const string_t key) { + return delete_kv_from_cache(key.value, key.len); +} diff --git a/src/database/file.c b/src/database/file.c index 9a22be8..d22a54f 100644 --- a/src/database/file.c +++ b/src/database/file.c @@ -1,6 +1,4 @@ #include "../../headers/database.h" -#include "../../headers/btree.h" -#include "../../headers/hashtable.h" #include "../../headers/utils.h" #include @@ -11,30 +9,57 @@ #include #include +#include #include #include static int fd = -1; static bool saving = false; +static uint16_t block_size; +static off64_t file_size; bool open_database_fd(const char *filename, uint64_t *server_age) { - fd = open(filename, (O_RDWR | O_CREAT), (S_IRUSR | S_IWUSR)); + if ((fd = open_file(filename, O_LARGEFILE)) != -1) { + struct stat sostat; + stat(filename, &sostat); - if (fd == -1) { - write_log(LOG_ERR, "Database file cannot be opened or created."); - return false; - } + block_size = sostat.st_blksize; + file_size = sostat.st_size; + + if (file_size != 0) { + char *block; + + if (posix_memalign((void **) &block, block_size, block_size) == 0) { + read(fd, block, block_size); + + if (block[0] != 0x18 || block[1] != 0x10) { + close(fd); + free(block); + write_log(LOG_ERR, "Invalid headers for database file, file is closed."); + return false; + } - if (lseek(fd, 0, SEEK_END) != 0) { - lseek(fd, 2, SEEK_SET); - read(fd, server_age, sizeof(long)); - } else { - *server_age = 0; + memcpy(server_age, block + 2, 8); + + const uint64_t filled_block_size = get_authorization_from_file(fd, block, block_size); + get_all_data_from_file(fd, file_size, block, block_size, filled_block_size); + write_log(LOG_INFO, "Read authorization part of database file. Loaded password count: %d", get_password_count()); + + free(block); + } + } else { + write_log(LOG_INFO, "Database file is empty, loaded password and data count: 0"); + *server_age = 0; + } } return true; } +uint16_t get_block_size() { + return block_size; +} + int get_database_fd() { return fd; } @@ -44,15 +69,31 @@ void close_database_fd() { close(fd); } -static off_t generate_value(char **line, struct KVPair *kv) { - off_t len; +static off64_t generate_value(char **data, struct KVPair *kv) { + off64_t len; + + { + string_t key = kv->key; + + const uint8_t bit_count = log2(key.len) + 1; + const uint8_t byte_count = ceil((float) (bit_count - 6) / 8); + const uint8_t first = (byte_count << 6) | (key.len & 0b111111); + const uint32_t length_in_bytes = key.len >> 6; + + len = key.len + byte_count + 1; + *data = realloc(*data, len); + + (*data)[0] = first; + memcpy(*data + 1, &length_in_bytes, byte_count); + memcpy(*data + byte_count + 1, key.value, key.len); + } switch (kv->type) { case TELLY_NULL: - len = 1; - *line = malloc(len); + *data = realloc(*data, len + 1); + (*data)[len] = TELLY_NULL; - (*line)[0] = TELLY_NULL; + len += 1; break; case TELLY_NUM: { @@ -60,12 +101,12 @@ static off_t generate_value(char **line, struct KVPair *kv) { const uint32_t bit_count = log2(*number) + 1; const uint32_t byte_count = (bit_count / 8) + 1; - len = byte_count + 2; - *line = malloc(len); + *data = realloc(*data, len + byte_count + 2); + (*data)[len] = TELLY_NUM; + (*data)[len += 1] = byte_count; + memcpy(*data + (len += 1), number, byte_count); - (*line)[0] = TELLY_NUM; - (*line)[1] = byte_count; - memcpy(*line + 2, number, byte_count); + len += byte_count; break; } @@ -77,195 +118,23 @@ static off_t generate_value(char **line, struct KVPair *kv) { const uint8_t first = (byte_count << 6) | (string->len & 0b111111); const uint32_t length_in_bytes = string->len >> 6; - len = string->len + byte_count + 2; - *line = malloc(len); + *data = realloc(*data, len + string->len + byte_count + 2); + (*data)[len] = TELLY_STR; + (*data)[len += 1] = first; + memcpy(*data + (len += 1), &length_in_bytes, byte_count); + memcpy(*data + (len += byte_count), string->value, string->len); - (*line)[0] = TELLY_STR; - (*line)[1] = first; - memcpy(*line + 2, &length_in_bytes, byte_count); - - memcpy(*line + byte_count + 2, string->value, string->len); + len += string->len; break; } case TELLY_BOOL: - len = 2; - *line = malloc(len); - - (*line)[0] = TELLY_BOOL; - (*line)[1] = *((bool *) kv->value); - break; - - case TELLY_HASHTABLE: { - struct HashTable *table = kv->value; - - len = 5; - *line = malloc(len); - - (*line)[0] = TELLY_HASHTABLE; - memcpy(*line + 1, &table->size.allocated, sizeof(uint32_t)); - - off_t at = len; - - for (uint32_t i = 0; i < table->size.allocated; ++i) { - struct FVPair *fv = table->fvs[i]; - - while (fv) { - const string_t name = fv->name; - const uint8_t n_bit_count = log2(name.len) + 1; - const uint8_t n_byte_count = ceil((float) (n_bit_count - 6) / 8); - const uint8_t n_first = (n_byte_count << 6) | (name.len & 0b111111); - const uint32_t n_length_in_bytes = name.len >> 6; - - switch (fv->type) { - case TELLY_NULL: - len += 2 + n_byte_count + name.len; - *line = realloc(*line, len); - - (*line)[at] = TELLY_NULL; - (*line)[at += 1] = n_first; - memcpy(*line + (at += 1), &n_length_in_bytes, n_byte_count); - memcpy(*line + (at += n_byte_count), name.value, name.len); - at += name.len; - break; - - case TELLY_NUM: { - const long *number = fv->value; - const uint32_t bit_count = log2(*number) + 1; - const uint32_t byte_count = (bit_count / 8) + 1; - - len += 3 + n_byte_count + name.len + byte_count; - *line = realloc(*line, len); - - (*line)[at] = TELLY_NUM; - (*line)[at += 1] = n_first; - memcpy(*line + (at += 1), &n_length_in_bytes, n_byte_count); - memcpy(*line + (at += n_byte_count), name.value, name.len); - - (*line)[at += name.len] = byte_count; - memcpy(*line + (at += 1), number, byte_count); - at += byte_count; - break; - } - - case TELLY_STR: { - string_t *string = fv->value; - - const uint8_t bit_count = log2(string->len) + 1; - const uint8_t byte_count = ceil((float) (bit_count - 6) / 8); - const uint8_t first = (byte_count << 6) | (string->len & 0b111111); - const uint32_t length_in_bytes = string->len >> 6; - - len += 3 + n_byte_count + name.len + byte_count + string->len; - *line = realloc(*line, len); - - (*line)[at] = TELLY_STR; - (*line)[at += 1] = n_first; - memcpy(*line + (at += 1), &n_length_in_bytes, n_byte_count); - memcpy(*line + (at += n_byte_count), name.value, name.len); - - (*line)[at += name.len] = first; - memcpy(*line + (at += 1), &length_in_bytes, byte_count); - memcpy(*line + (at += byte_count), string->value, string->len); - at += string->len; - break; - } - - case TELLY_BOOL: - len += 3 + n_byte_count + name.len; - *line = realloc(*line, len); - - (*line)[at] = TELLY_BOOL; - (*line)[at += 1] = n_first; - memcpy(*line + (at += 1), &n_length_in_bytes, n_byte_count); - memcpy(*line + (at += n_byte_count), name.value, name.len); - - (*line)[at += name.len] = *((bool *) fv->value); - at += 1; - break; - - default: - break; - } - - fv = fv->next; - } - } + *data = realloc(*data, len + 2); + (*data)[len] = TELLY_BOOL; + (*data)[len += 1] = *((bool *) kv->value); len += 1; - *line = realloc(*line, len); - (*line)[at] = 0x17; - break; - } - - case TELLY_LIST: { - struct List *list = kv->value; - struct ListNode *node = list->begin; - - len = 5; - *line = malloc(len); - - (*line)[0] = TELLY_LIST; - memcpy(*line + 1, &list->size, sizeof(uint32_t)); - - while (node) { - switch (node->type) { - case TELLY_NULL: - len += 1; - *line = realloc(*line, len + 1); - - (*line)[len - 1] = TELLY_NULL; - break; - - case TELLY_NUM: { - const long *number = node->value; - const uint32_t bit_count = log2(*number) + 1; - const uint32_t byte_count = (bit_count / 8) + 1; - - len += byte_count + 2; - *line = realloc(*line, len + 1); - - (*line)[len - byte_count - 2] = TELLY_NUM; - (*line)[len - byte_count - 1] = byte_count; - memcpy(*line + len - byte_count, number, byte_count); - break; - } - - case TELLY_STR: { - string_t *string = node->value; - - const uint8_t bit_count = log2(string->len) + 1; - const uint8_t byte_count = ceil((float) (bit_count - 6) / 8); - const uint8_t first = (byte_count << 6) | (string->len & 0b111111); - const uint32_t length_in_bytes = string->len >> 6; - - len += string->len + byte_count + 2; - *line = realloc(*line, len + 1); - - (*line)[len - string->len - byte_count - 2] = TELLY_STR; - (*line)[len - string->len - byte_count - 1] = first; - memcpy(*line + len - string->len - byte_count, &length_in_bytes, byte_count); - memcpy(*line + len - string->len, string->value, string->len); - break; - } - - case TELLY_BOOL: - len += 2; - *line = realloc(*line, len + 1); - - (*line)[len - 2] = TELLY_BOOL; - (*line)[len - 1] = *((bool *) node->value); - break; - - default: - break; - } - - node = node->next; - } - break; - } default: len = 0; @@ -274,170 +143,104 @@ static off_t generate_value(char **line, struct KVPair *kv) { return len; } -static off_t generate_authorization_part(char **part) { - struct Password **passwords = get_passwords(); - const uint32_t password_count = get_password_count(); - const uint8_t password_count_byte_count = (password_count != 0) ? (log2(password_count) + 1) : 0; - off_t part_length = 1 + password_count_byte_count; - *part = malloc(part_length); - - (*part)[0] = password_count_byte_count; - memcpy(*part + 1, &password_count, password_count_byte_count); - - char *buf = malloc(1); - - for (uint32_t i = 0; i < password_count; ++i) { - struct Password *password = passwords[i]; - string_t data = password->data; - - const uint8_t bit_count = log2(data.len) + 1; - const uint8_t byte_count = ceil((float) (bit_count - 6) / 8); - const uint8_t first = (byte_count << 6) | (data.len & 0b111111); - const uint32_t length_in_bytes = data.len >> 6; - - const uint32_t buf_len = 4 + byte_count + data.len; - buf = realloc(buf, buf_len); - buf[0] = first; - memcpy(buf + 1, &length_in_bytes, byte_count); - memcpy(buf + 1 + byte_count, data.value, data.len); - memcpy(buf + 1 + byte_count + data.len, password->salt, 2); - buf[3 + byte_count + data.len] = password->permissions; - - *part = realloc(*part, part_length + buf_len); - memcpy(*part + part_length, buf, buf_len); - part_length += buf_len; - } - - free(buf); - return part_length; +static void generate_headers(char *headers, const uint64_t server_age) { + headers[0] = 0x18; + headers[1] = 0x10; + memcpy(headers + 2, &server_age, sizeof(uint64_t)); } void save_data(const uint64_t server_age) { if (saving) return; saving = true; - uint32_t size; - struct BTreeValue **values = get_sorted_kvs_by_pos_as_values(&size); - - uint32_t file_size = lseek(fd, 0, SEEK_END); - int32_t diff = 0; - - if (file_size != 0) { - // File headers - { - uint8_t constants[2]; - lseek(fd, 0, SEEK_SET); + lseek(fd, 0, SEEK_SET); - if (read(fd, constants, 2) != 2 || constants[0] != 0x18 || constants[1] != 0x10 || file_size < 11) { - write_log(LOG_ERR, "Cannot save data, invalid file headers"); - free(values); - return; - } + char *block; + + if (posix_memalign((void **) &block, block_size, block_size) == 0) { + memset(block, 0, block_size); - write(fd, &server_age, sizeof(uint64_t)); - } + uint32_t length, total = 0; + generate_headers(block, server_age); - // Authorization part { - char *part; - const off_t length = generate_authorization_part(&part); - off_t *end_at = get_authorization_end_at(); - - char *buf = malloc(file_size - *end_at); - lseek(fd, *end_at, SEEK_SET); - read(fd, buf, file_size - *end_at); - - lseek(fd, 10, SEEK_SET); - write(fd, part, length); - write(fd, buf, file_size - *end_at); - - file_size += length - (*end_at - 10); - *end_at += (length + 10) - *end_at; - free(part); - free(buf); - } - } else { - // File headers - { - uint8_t data[10] = {0x18, 0x10}; - memcpy(data + 2, &server_age, 8); - - lseek(fd, 0, SEEK_SET); - write(fd, data, 10); + struct Password **passwords = get_passwords(); + const uint32_t password_count = get_password_count(); + const uint8_t password_count_byte_count = (password_count != 0) ? (log2(password_count) + 1) : 0; + length = 11 + password_count_byte_count; + + block[10] = password_count_byte_count; + memcpy(block + 11, &password_count, password_count_byte_count); + + for (uint32_t i = 0; i < password_count; ++i) { + struct Password *password = passwords[i]; + const uint32_t new_length = (length + 49); + + if (new_length > block_size) { + const uint32_t allowed = (new_length - block_size); + memcpy(block + length, password->data, allowed); + write(fd, block, block_size); + + const uint32_t remaining = (48 - allowed); // remaining byte count except permissions + memcpy(block, password->data, remaining); + block[remaining] = password->permissions; + total += block_size; + length = (remaining + 1); + } else { + memcpy(block + length, password->data, 48); + block[length + 48] = password->permissions; + length = new_length; + } + } } - // Authorization part { - char *part; - const off_t length = generate_authorization_part(&part); - off_t *end_at = get_authorization_end_at(); + uint32_t size; + struct BTree *cache = get_cache(); + struct BTreeValue **values = get_values_from_btree(cache, &size); - write(fd, part, length); - *end_at = (file_size = 10 + length); - free(part); - } - } + char *data = malloc(1); - for (uint32_t i = 0; i < size; ++i) { - struct KVPair *kv = values[i]->data; + for (uint32_t i = 0; i < size; ++i) { + struct KVPair *kv = values[i]->data; + const off64_t data_length = generate_value(&data, kv); - char *line; - const off_t line_len = generate_value(&line, kv); + const uint32_t block_count = ((length + data_length + block_size - 1) / block_size); - if (line_len != 0) { - if (kv->pos.start_at != -1) { - const off_t line_len_in_file = kv->pos.end_at - (kv->pos.start_at - 1); + if (block_count != 1) { + off64_t remaining = data_length; + const uint16_t complete = (block_size - length); - if (line_len_in_file != line_len) { - const off_t buf_len = file_size - kv->pos.end_at; - const off_t total_len = line_len + buf_len; - line = realloc(line, total_len); + memcpy(block + length, data, complete); + write(fd, data, complete); + remaining -= complete; - lseek(fd, kv->pos.end_at, SEEK_SET); - read(fd, line + line_len, buf_len); - lseek(fd, kv->pos.start_at + diff - 1, SEEK_SET); - write(fd, line, total_len); + if (remaining > block_size) { + do { + memcpy(block, data + (data_length - remaining), block_size); + write(fd, block, block_size); + remaining -= block_size; + } while (remaining > block_size); - diff += line_len - line_len_in_file; - kv->pos.end_at += diff; + length = (data_length - remaining); + } else { + length += (data_length - remaining); + } } else { - lseek(fd, kv->pos.start_at + diff - 1, SEEK_SET); - write(fd, line, line_len); + memcpy(block + length, data, data_length); + length += data_length; } - } else { - kv->pos.start_at = lseek(fd, 0, SEEK_END); - - const string_t key = kv->key; - - const uint8_t bit_count = log2(key.len) + 1; - const uint8_t byte_count = ceil((float) (bit_count - 6) / 8); - const uint8_t first = (byte_count << 6) | (key.len & 0b111111); - const uint32_t length_in_bytes = key.len >> 6; - - const uint32_t length_specifier_len = byte_count + 1; - const uint32_t key_part_len = key.len + length_specifier_len; - - kv->pos.end_at = kv->pos.start_at + key_part_len + line_len; - kv->pos.start_at += key_part_len + 1; - - const uint32_t buf_len = key_part_len + line_len; - char buf[buf_len]; - - buf[0] = first; - memcpy(buf + 1, &length_in_bytes, byte_count); - memcpy(buf + length_specifier_len, key.value, key.len); - memcpy(buf + length_specifier_len + key.len, line, line_len); - - write(fd, buf, buf_len); - file_size += buf_len; } - free(line); + total += length; + if (values) free(values); + free(data); } - } - ftruncate(fd, file_size + diff); - free(values); + if (length != block_size) write(fd, block, block_size); + ftruncate(fd, total); + free(block); + } saving = false; } diff --git a/src/database/get.c b/src/database/get.c index 3331476..74f852d 100644 --- a/src/database/get.c +++ b/src/database/get.c @@ -1,6 +1,5 @@ #include "../../headers/database.h" #include "../../headers/btree.h" -#include "../../headers/hashtable.h" #include "../../headers/utils.h" #include @@ -10,350 +9,119 @@ #include -void get_all_keys(const off_t from) { - const int fd = get_database_fd(); - struct BTree *cache = get_cache(); - - uint8_t eof; - lseek(fd, from, SEEK_SET); - - while (read(fd, &eof, 1)) { - lseek(fd, -1, SEEK_CUR); - - string_t key; - uint8_t type; - off_t start_at, end_at; - - // Key reading - read_string_from_file(fd, &key, true, true); - - // Value reading - { - read(fd, &type, 1); - - switch (type) { - case TELLY_NULL: - start_at = lseek(fd, 0, SEEK_CUR); - end_at = start_at; - break; - - case TELLY_NUM: { - uint8_t count; - read(fd, &count, 1); - end_at = lseek(fd, count, SEEK_CUR); - start_at = end_at - count; - break; - } - - case TELLY_BOOL: - end_at = lseek(fd, 1, SEEK_CUR); - start_at = end_at - 1; - break; - - case TELLY_STR: { - uint8_t first; - read(fd, &first, 1); - - const uint8_t byte_count = first >> 6; - uint32_t length = 0; - read(fd, &length, byte_count); - length = (length << 6) | (first & 0b111111); - - end_at = lseek(fd, length, SEEK_CUR); - start_at = end_at - length - byte_count - 1; - break; - } - - case TELLY_HASHTABLE: { - uint32_t size = 0; - off_t length = 5; - - read(fd, &size, sizeof(uint32_t)); - - while (true) { - uint8_t node_type; - read(fd, &node_type, 1); - - if (node_type == 0x17) { - end_at = lseek(fd, 0, SEEK_CUR); - start_at = end_at - length; - break; - } else { - { - uint8_t first; - read(fd, &first, 1); - - const uint8_t byte_count = first >> 6; - uint32_t key_length = 0; - - read(fd, &key_length, byte_count); - key_length = (key_length << 6) | (first & 0b111111); - - lseek(fd, key_length, SEEK_CUR); - length += 2 + byte_count + key_length; - } - - switch (node_type) { - case TELLY_NULL: - break; - - case TELLY_NUM: { - uint8_t count; - read(fd, &count, 1); - lseek(fd, count, SEEK_CUR); - length += 1 + count; - break; - } - - case TELLY_BOOL: - lseek(fd, 1, SEEK_CUR); - length += 1; - break; - - case TELLY_STR: { - uint32_t string_length = 0; - - uint8_t first; - read(fd, &first, 1); - - const uint8_t byte_count = first >> 6; - lseek(fd, byte_count, SEEK_CUR); - - read(fd, &string_length, byte_count); - string_length = (string_length << 6) | (first & 0b111111); - - lseek(fd, string_length, SEEK_CUR); - length += 1 + byte_count + string_length; - break; - } - } - } - } - - break; - } - - case TELLY_LIST: { - uint32_t size = 0; - off_t length = 4; - - read(fd, &size, sizeof(uint32_t)); - - while (size--) { - uint8_t node_type; - read(fd, &node_type, 1); - - switch (node_type) { - case TELLY_NULL: - length += 1; - break; - - case TELLY_NUM: { - uint8_t count; - read(fd, &count, 1); - lseek(fd, count, SEEK_CUR); - length += 2 + count; - break; - } - - case TELLY_BOOL: - lseek(fd, 1, SEEK_CUR); - length += 2; - break; - - case TELLY_STR: { - uint8_t first; - read(fd, &first, 1); +static void collect_bytes(const int fd, char *block, const uint16_t block_size, uint16_t *at, const uint32_t count, void *data) { + uint32_t remaining = count; - const uint8_t byte_count = first >> 6; - uint32_t string_length = 0; + if ((*at + remaining) >= block_size) { + const uint16_t available = (block_size - *at); + memcpy(data + (count - remaining), block + *at, available); + read(fd, block, block_size); + remaining -= available; - read(fd, &string_length, byte_count); - string_length = (string_length << 6) | (first & 0b111111); - - lseek(fd, string_length, SEEK_CUR); - length += 2 + byte_count + string_length; - break; - } - } - } - - end_at = lseek(fd, 0, SEEK_CUR); - start_at = end_at - length; - break; - } - - default: - start_at = 0; - end_at = 0; - } + while ((*at + remaining) >= block_size) { + memcpy(data + (count - remaining), block, block_size); + read(fd, block, block_size); } - struct KVPair *kv = malloc(sizeof(struct KVPair)); - set_kv(kv, key, NULL, type, start_at, end_at); - - insert_value_to_btree(cache, hash(key.value), kv); - free(key.value); + memcpy(data, block, remaining); + } else { + memcpy(data, block + *at, remaining); } -} - -struct KVPair *get_data(const char *key) { - struct KVPair *data = get_kv_from_cache(key); - - if (data && !data->value) { - const int fd = get_database_fd(); - - const off_t start_at = data->pos.start_at; - const off_t end_at = data->pos.end_at; - lseek(fd, start_at, SEEK_SET); - - switch (data->type) { - case TELLY_NUM: { - const uint8_t count = end_at - start_at; - data->value = malloc(sizeof(long)); - memset(data->value, 0, sizeof(long)); // to initialize all bytes of long number - read(fd, data->value, count); - - break; - } - - case TELLY_STR: { - string_t *string = (data->value = malloc(sizeof(string_t))); + *at += remaining; +} - uint8_t first; - read(fd, &first, 1); +static void collect_string(string_t *string, const int fd, char *block, const uint16_t block_size, uint16_t *at) { + string->len = 0; - const uint8_t byte_count = first >> 6; - lseek(fd, byte_count, SEEK_CUR); + uint8_t first; + collect_bytes(fd, block, block_size, at, 1, &first); - string->len = data->pos.end_at - data->pos.start_at - byte_count - 1; - string->value = malloc(string->len); - read(fd, string->value, string->len); + const uint8_t byte_count = (first >> 6); + collect_bytes(fd, block, block_size, at, byte_count, &string->len); - break; - } + string->len = ((string->len << 6) | (first & 0b111111)); + string->value = malloc(string->len); + collect_bytes(fd, block, block_size, at, string->len, string->value); +} - case TELLY_BOOL: - data->value = malloc(sizeof(bool)); - read(fd, data->value, 1); - break; +static void collect_number(long *number, const int fd, char *block, const uint16_t block_size, uint16_t *at) { + uint8_t byte_count; + collect_bytes(fd, block, block_size, at, 1, &byte_count); + collect_bytes(fd, block, block_size, at, byte_count, number); +} - case TELLY_HASHTABLE: { - uint32_t size; - read(fd, &size, sizeof(uint32_t)); +static void collect_kv(struct KVPair *kv, const int fd, char *block, const uint16_t block_size, uint16_t *at) { + string_t key; + void *value = NULL; + uint8_t type; - struct HashTable *table = (data->value = create_hashtable(size)); - string_t name = { - .value = malloc(1) - }; + collect_string(&key, fd, block, block_size, at); + collect_bytes(fd, block, block_size, at, 1, &type); - while (true) { - uint8_t type; - read(fd, &type, 1); - if (type == 0x17) break; + switch (type) { + case TELLY_NULL: + break; - // Element key reading - read_string_from_file(fd, &name, false, true); + case TELLY_NUM: { + value = malloc(sizeof(long)); + collect_number(value, fd, block, block_size, at); + break; + } - // Element value reading - switch (type) { - case TELLY_NULL: - add_fv_to_hashtable(table, name, NULL, TELLY_NULL); - break; + case TELLY_STR: { + value = malloc(sizeof(string_t)); + collect_string(value, fd, block, block_size, at); + break; + } - case TELLY_NUM: { - long *number = malloc(sizeof(long)); - memset(number, 0, sizeof(long)); + case TELLY_BOOL: { + value = malloc(sizeof(bool)); + collect_bytes(fd, block, block_size, at, 1, value); + break; + } - uint8_t count; - read(fd, &count, 1); - read(fd, number, count); + default: + break; + } - add_fv_to_hashtable(table, name, number, TELLY_NUM); - break; - } + set_kv(kv, key, value, type); + free(key.value); +} - case TELLY_STR: { - string_t *string = malloc(sizeof(string_t)); - read_string_from_file(fd, string, true, false); - add_fv_to_hashtable(table, name, string, TELLY_STR); - break; - } +void get_all_data_from_file(const int fd, off64_t file_size, char *block, const uint16_t block_size, const uint16_t filled_block_size) { + struct BTree *cache = get_cache(); + uint16_t at = filled_block_size; - case TELLY_BOOL: { - bool *value = malloc(sizeof(bool)); - read(fd, value, 1); + if (at != file_size) { + if (at != 0) read(fd, block, block_size); - add_fv_to_hashtable(table, name, value, TELLY_NUM); - break; - } - } - } + if (file_size <= block_size) { + while (at != file_size) { + struct KVPair *kv = malloc(sizeof(struct KVPair)); + collect_kv(kv, fd, block, block_size, &at); - free(name.value); - break; + const uint64_t index = hash(kv->key.value, kv->key.len); + insert_value_to_btree(cache, index, kv); } - - case TELLY_LIST: { - struct List *list = (data->value = create_list()); - read(fd, &list->size, sizeof(uint32_t)); - - for (uint32_t i = 0; i < list->size; ++i) { - struct ListNode *node = NULL; - uint8_t type; - read(fd, &type, 1); - - switch (type) { - case TELLY_NULL: - node = create_listnode(NULL, TELLY_NULL); - break; - - case TELLY_NUM: { - long *number = malloc(sizeof(long)); - memset(number, 0, sizeof(long)); - - uint8_t count; - read(fd, &count, 1); - read(fd, number, count); - - node = create_listnode(number, TELLY_NUM); - break; - } - - case TELLY_STR: { - string_t *string = malloc(sizeof(string_t)); - read_string_from_file(fd, string, true, false); - node = create_listnode(string, TELLY_STR); - break; - } - - case TELLY_BOOL: { - bool *value = malloc(sizeof(bool)); - read(fd, value, 1); - - node = create_listnode(value, TELLY_BOOL); - break; - } - } - - if (node) { - node->prev = list->end; - list->end = node; - - if (i == 0) { - list->begin = node; - } else { - node->prev->next = node; - } - } + } else { + do { + while (at <= block_size) { + struct KVPair *kv = malloc(sizeof(struct KVPair)); + collect_kv(kv, fd, block, block_size, &at); + + const uint64_t index = hash(kv->key.value, kv->key.len); + insert_value_to_btree(cache, index, kv); } - } - default: - break; + if (lseek(fd, 0, SEEK_CUR) != file_size) { + read(fd, block, block_size); + } else break; + } while (true); } } +} - return data; +struct KVPair *get_data(const string_t key) { + return get_kv_from_cache(key.value, key.len); } diff --git a/src/database/kv.c b/src/database/kv.c index 7059ba7..da3b263 100644 --- a/src/database/kv.c +++ b/src/database/kv.c @@ -7,17 +7,12 @@ #include -void set_kv(struct KVPair *kv, const string_t key, void *value, const enum TellyTypes type, const off_t start_at, const off_t end_at) { - kv->pos.start_at = start_at; - kv->pos.end_at = end_at; - - const uint32_t key_size = key.len + 1; - +void set_kv(struct KVPair *kv, const string_t key, void *value, const enum TellyTypes type) { kv->type = type; kv->value = value; kv->key.len = key.len; - kv->key.value = malloc(key_size); - memcpy(kv->key.value, key.value, key_size); + kv->key.value = malloc(key.len); + memcpy(kv->key.value, key.value, key.len); } void free_kv(struct KVPair *kv) { diff --git a/src/database/set.c b/src/database/set.c index 57e00d6..a12251f 100644 --- a/src/database/set.c +++ b/src/database/set.c @@ -37,8 +37,8 @@ struct KVPair *set_data(struct KVPair *data, const string_t key, void *value, co } else { struct BTree *cache = get_cache(); struct KVPair *kv = malloc(sizeof(struct KVPair)); - set_kv(kv, key, value, type, -1, -1); + set_kv(kv, key, value, type); - return insert_value_to_btree(cache, hash(key.value), kv)->data; + return insert_value_to_btree(cache, hash(key.value, key.len), kv)->data; } } diff --git a/src/hashtable/hashtable.c b/src/hashtable/hashtable.c index ebf30ce..caffeca 100644 --- a/src/hashtable/hashtable.c +++ b/src/hashtable/hashtable.c @@ -40,11 +40,11 @@ void resize_hashtable(struct HashTable *table, const uint32_t size) { table->fvs = fvs; } -struct FVPair *get_fv_from_hashtable(struct HashTable *table, char *name) { - const uint32_t index = hash(name) % table->size.allocated; +struct FVPair *get_fv_from_hashtable(struct HashTable *table, const string_t name) { + const uint32_t index = hash(name.value, name.len) % table->size.allocated; struct FVPair *fv = table->fvs[index]; - while (fv && !streq(fv->name.value, name)) fv = fv->next; + while (fv && !streq(fv->name.value, name.value)) fv = fv->next; return fv; } diff --git a/src/hashtable/memory.c b/src/hashtable/memory.c index 0e6fc03..d1222ef 100644 --- a/src/hashtable/memory.c +++ b/src/hashtable/memory.c @@ -7,7 +7,7 @@ #include void add_fv_to_hashtable(struct HashTable *table, const string_t name, void *value, const enum TellyTypes type) { - const uint64_t hashed = hash(name.value); + const uint64_t hashed = hash(name.value, name.len); uint32_t index = hashed % table->size.allocated; struct FVPair *fv; @@ -78,7 +78,7 @@ void add_fv_to_hashtable(struct HashTable *table, const string_t name, void *val } bool del_fv_to_hashtable(struct HashTable *table, const string_t name) { - const uint64_t hashed = hash(name.value); + const uint64_t hashed = hash(name.value, name.len); uint32_t index = hashed % table->size.allocated; struct FVPair *fv; diff --git a/src/server/auth.c b/src/server/auth.c index 3492034..f125892 100644 --- a/src/server/auth.c +++ b/src/server/auth.c @@ -2,9 +2,53 @@ #include "../../headers/utils.h" #include +#include +#include + +#include +#include +#include +#include #include +static OSSL_LIB_CTX *libctx; +static OSSL_PROVIDER *prov; +static EVP_KDF_CTX *ctx; +static OSSL_PARAM params[5]; + +bool initialize_kdf() { + libctx = OSSL_LIB_CTX_new(); + prov = OSSL_PROVIDER_load(libctx, "default"); + + EVP_KDF *kdf = EVP_KDF_fetch(libctx, "HKDF", NULL); + ctx = EVP_KDF_CTX_new(kdf); + EVP_KDF_free(kdf); + + params[0] = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST, "SHA384", 0); + params[1] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_INFO, "label", 5); + params[2] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, "salt", 4); + params[4] = OSSL_PARAM_construct_end(); + + return true; +} + +void free_kdf() { + EVP_KDF_CTX_free(ctx); + OSSL_PROVIDER_unload(prov); + OSSL_LIB_CTX_free(libctx); +} + +static bool password_derive(char *value, const size_t value_len, unsigned char *out) { + params[3] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_KEY, value, value_len); + + if (EVP_KDF_derive(ctx, out, 48, params) <= 0) { + write_log(LOG_ERR, "Cannot derive the password."); + free_kdf(); + return false; + } else return true; +} + void remove_password_from_clients(struct Password *password) { struct Client **clients = get_clients(); const uint32_t client_count = get_client_count(); @@ -12,7 +56,7 @@ void remove_password_from_clients(struct Password *password) { for (uint32_t i = 0; i < client_count; ++i) { struct Client *client = clients[i]; - if (client->password == (password)) { + if (client->password == password) { client->password = get_empty_password(); } } @@ -29,70 +73,120 @@ uint32_t get_password_count() { return password_count; } -static void get_password_from_file(const int fd, const uint32_t i) { - struct Password *password = (passwords[i] = malloc(sizeof(struct Password))); - - read_string_from_file(fd, &password->data, true, true); - - read(fd, password->salt, 2); - password->salt[2] = '\0'; - - read(fd, &password->permissions, 1); -} - -off_t get_authorization_from_file(const int fd) { - lseek(fd, 10, SEEK_SET); +uint16_t get_authorization_from_file(const int fd, char *block, const uint16_t block_size) { + const uint8_t password_count_byte_count = block[10]; + memcpy(&password_count, block + 11, password_count_byte_count); - uint8_t password_count_byte_count; - read(fd, &password_count_byte_count, 1); - read(fd, &password_count, password_count_byte_count); + off64_t at = 11 + password_count_byte_count; + off_t total = at; if (password_count != 0) { + struct Password *password; + uint32_t password_at = 0; passwords = malloc(password_count * sizeof(struct Password *)); - get_password_from_file(fd, 0); - for (uint32_t i = 1; i < password_count; ++i) { - get_password_from_file(fd, i); - } + /* + Interrupted operations + + 0: No interrupted operation + 1: Retrieving value + 2: Retrieving permissions + */ + uint8_t op = 0; + uint32_t overflow_len = 0; + + do { + switch (op) { + case 1: { + password = passwords[password_at]; + memcpy(password->data + (48 - overflow_len), block, overflow_len); + + at = overflow_len; + break; + } + + case 2: { + password = passwords[password_at]; + password->permissions = block[0]; + + op = 0; + at += 1; + password_at += 1; + break; + } + } + + while (password_at != password_count) { + { + if (op < 1) { + password = (passwords[password_at] = malloc(sizeof(struct Password))); + + if ((at + 48) > block_size) { + const uint32_t remaining = (block_size - at); + memcpy(password->data, block + at, remaining); + overflow_len = 48 - remaining; + op = 1; + break; + } else { + memcpy(password->data, block + at, 48); + at += 48; + } + } + } + + if (op < 2) { + if ((at + 1) > block_size) { + op = 2; + break; + } else { + password->permissions = block[at]; + at += 1; + op = 0; + } + } + + password_at += 1; + } + + total += at; + at = 0; + } while (read(fd, block, block_size)); } - return lseek(fd, 0, SEEK_CUR); + return (total % block_size); } -int32_t where_password(const char *value) { +int32_t where_password(char *value, const size_t value_len) { + unsigned char derived[48]; + if (!password_derive(value, value_len, derived)) return -1; + for (uint32_t i = 0; i < password_count; ++i) { struct Password *password = passwords[i]; - const char *hashed = crypt(value, password->salt); - - if (streq(hashed, password->data.value)) return i; + if (memcmp(derived, password->data, 48) == 0) return i; } return -1; } -struct Password *get_password(const char *value) { +struct Password *get_password(char *value, const size_t value_len) { + unsigned char derived[48]; + if (!password_derive(value, value_len, derived)) return NULL; + for (uint32_t i = 0; i < password_count; ++i) { struct Password *password = passwords[i]; - const char *hashed = crypt(value, password->salt); - - if (streq(hashed, password->data.value)) return password; + if (memcmp(derived, password->data, 48) == 0) return password; } return NULL; } -bool edit_password(const char *value, const uint32_t permissions) { - for (uint32_t i = 0; i < password_count; ++i) { - struct Password *password = passwords[i]; - const char *hashed = crypt(value, password->salt); +bool edit_password(char *value, const size_t value_len, const uint32_t permissions) { + struct Password *password = get_password(value, value_len); - if (streq(hashed, password->data.value)) { - password->permissions = permissions; - return true; - } - } - - return false; + if (password) { + password->permissions = permissions; + return true; + } else return false; } void add_password(struct Client *client, const string_t data, const uint8_t permissions) { @@ -110,17 +204,11 @@ void add_password(struct Client *client, const string_t data, const uint8_t perm passwords[password_count - 1] = password; } - generate_random_string(password->salt, 2); - const char *hashed = crypt(data.value, password->salt); + password_derive(data.value, data.len, password->data); password->permissions = permissions; - - const uint32_t size = (password->data.len = strlen(hashed)) + 1; - password->data.value = malloc(size); - memcpy(password->data.value, hashed, size); } void free_password(struct Password *password) { - free(password->data.value); free(password); } @@ -136,9 +224,9 @@ void free_passwords() { } } -bool remove_password(struct Client *executor, const char *value) { +bool remove_password(struct Client *executor, char *value, const size_t value_len) { if (password_count == 1) { - if (where_password(value) == 0) { + if (where_password(value, value_len) == 0) { struct Password *password = passwords[0]; remove_password_from_clients(password); @@ -151,7 +239,7 @@ bool remove_password(struct Client *executor, const char *value) { return true; } else return false; } else { - const int32_t at = where_password(value); + const int32_t at = where_password(value, value_len); if (at == -1) return false; else { diff --git a/src/server/commands.c b/src/server/commands.c index 6b733eb..2cbb7f3 100644 --- a/src/server/commands.c +++ b/src/server/commands.c @@ -8,7 +8,7 @@ #include static struct Command *commands = NULL; -static uint32_t command_count = 31; +static uint32_t command_count = 33; void load_commands() { const struct Command _commands[] = { @@ -39,9 +39,11 @@ void load_commands() { // KV commands cmd_append, cmd_decr, + cmd_del, cmd_exists, cmd_get, cmd_incr, + cmd_rename, cmd_set, cmd_type, diff --git a/src/server/server.c b/src/server/server.c index 6db8683..37b1fb3 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -1,7 +1,6 @@ #include "../../headers/server.h" #include "../../headers/database.h" #include "../../headers/commands.h" -#include "../../headers/btree.h" #include "../../headers/utils.h" #include @@ -25,16 +24,11 @@ static uint32_t nfds; static struct Configuration *conf; static time_t start_at; static uint64_t age; -static off_t authorization_end_at; struct Configuration *get_server_configuration() { return conf; } -off_t *get_authorization_end_at() { - return &authorization_end_at; -} - void get_server_time(time_t *server_start_at, uint64_t *server_age) { *server_start_at = start_at; *server_age = age; @@ -86,6 +80,7 @@ static void close_server() { usleep(15); write_log(LOG_INFO, "Exited transaction thread."); + free_kdf(); free_constant_passwords(); free_passwords(); free_transactions(); @@ -221,7 +216,14 @@ void start_server(struct Configuration *config) { return; } - struct BTree *cache = create_cache(); + write_log(LOG_INFO, "tellydb server age: %ld seconds", age); + + create_constant_passwords(); + initialize_kdf(); + write_log(LOG_INFO, "Created constant passwords and key deriving algorithm."); + + create_cache(); + write_log(LOG_INFO, "Created cache."); if (!open_database_fd(conf->data_file, &age)) { deactive_transaction_thread(); @@ -235,25 +237,6 @@ void start_server(struct Configuration *config) { return; } - write_log(LOG_INFO, "Created cache and opened database file."); - write_log(LOG_INFO, "tellydb server age: %ld seconds", age); - - create_constant_passwords(); - - { - const int fd = get_database_fd(); - - if (lseek(fd, 0, SEEK_END) != 0) { - authorization_end_at = get_authorization_from_file(fd); - write_log(LOG_INFO, "Read authorization part of database file. Loaded password count: %d", get_password_count()); - - get_all_keys(authorization_end_at); - write_log(LOG_INFO, "Read all database file to create keyspace. Loaded key count: %d", cache->size); - } else { - write_log(LOG_INFO, "Database file is empty. Loaded key and password count: 0"); - } - } - nfds = 1; fds = malloc(sizeof(struct pollfd)); fds[0].fd = sockfd; diff --git a/src/utils/hash.c b/src/utils/hash.c index e3c38d1..f86d415 100644 --- a/src/utils/hash.c +++ b/src/utils/hash.c @@ -2,10 +2,9 @@ #include -uint64_t hash(char *key) { +uint64_t hash(char *key, uint32_t length) { uint64_t hash = 5381; - char c; - while ((c = *key++)) hash = ((hash << 5) + hash) + c; + while (length--) hash = ((hash << 5) + hash) + (*key++); return hash; }