From afe0e0f3b57d57d93215f1c9ec9d5a2b1beafbb7 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 29 May 2024 14:04:07 -0700 Subject: [PATCH 1/5] Disable replication when writing to the pending bootstrap store Nothing observes this table for notifications, so we don't need CT history. This is a pretty minor optimization unless we get a very large number of very small changests from the server. --- .../sync/noinst/pending_bootstrap_store.cpp | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/src/realm/sync/noinst/pending_bootstrap_store.cpp b/src/realm/sync/noinst/pending_bootstrap_store.cpp index d576762cf30..03667c58c00 100644 --- a/src/realm/sync/noinst/pending_bootstrap_store.cpp +++ b/src/realm/sync/noinst/pending_bootstrap_store.cpp @@ -141,39 +141,44 @@ void PendingBootstrapStore::add_batch(int64_t query_version, util::Optionalstart_write(); - auto bootstrap_table = tr->get_table(m_table); - auto incomplete_bootstraps = Query(bootstrap_table).not_equal(m_query_version, query_version).find_all(); - incomplete_bootstraps.for_each([&](Obj obj) { - m_logger.debug(util::LogCategory::changeset, "Clearing incomplete bootstrap for query version %1", - obj.get(m_query_version)); - return IteratorControl::AdvanceToNext; - }); - incomplete_bootstraps.clear(); - bool did_create = false; - auto bootstrap_obj = bootstrap_table->create_object_with_primary_key(Mixed{query_version}, &did_create); - if (progress) { - auto progress_obj = bootstrap_obj.create_and_set_linked_object(m_progress); - progress_obj.set(m_progress_latest_server_version, int64_t(progress->latest_server_version.version)); - progress_obj.set(m_progress_latest_server_version_salt, int64_t(progress->latest_server_version.salt)); - progress_obj.set(m_progress_download_server_version, int64_t(progress->download.server_version)); - progress_obj.set(m_progress_download_client_version, - int64_t(progress->download.last_integrated_client_version)); - progress_obj.set(m_progress_upload_server_version, int64_t(progress->upload.last_integrated_server_version)); - progress_obj.set(m_progress_upload_client_version, int64_t(progress->upload.client_version)); - } - auto changesets_list = bootstrap_obj.get_linklist(m_changesets); - for (size_t idx = 0; idx < changesets.size(); ++idx) { - auto cur_changeset = changesets_list.create_and_insert_linked_object(changesets_list.size()); - cur_changeset.set(m_changeset_remote_version, int64_t(changesets[idx].remote_version)); - cur_changeset.set(m_changeset_last_integrated_client_version, - int64_t(changesets[idx].last_integrated_local_version)); - cur_changeset.set(m_changeset_origin_file_ident, int64_t(changesets[idx].origin_file_ident)); - cur_changeset.set(m_changeset_origin_timestamp, int64_t(changesets[idx].origin_timestamp)); - cur_changeset.set(m_changeset_original_changeset_size, int64_t(changesets[idx].original_changeset_size)); - BinaryData compressed_data(compressed_changesets[idx].data(), compressed_changesets[idx].size()); - cur_changeset.set(m_changeset_data, compressed_data); + { + DisableReplication disable_replication(*tr); + auto bootstrap_table = tr->get_table(m_table); + auto incomplete_bootstraps = Query(bootstrap_table).not_equal(m_query_version, query_version).find_all(); + incomplete_bootstraps.for_each([&](Obj obj) { + m_logger.debug(util::LogCategory::changeset, "Clearing incomplete bootstrap for query version %1", + obj.get(m_query_version)); + return IteratorControl::AdvanceToNext; + }); + incomplete_bootstraps.clear(); + + auto bootstrap_obj = bootstrap_table->create_object_with_primary_key(Mixed{query_version}, &did_create); + if (progress) { + auto progress_obj = bootstrap_obj.create_and_set_linked_object(m_progress); + progress_obj.set(m_progress_latest_server_version, int64_t(progress->latest_server_version.version)); + progress_obj.set(m_progress_latest_server_version_salt, int64_t(progress->latest_server_version.salt)); + progress_obj.set(m_progress_download_server_version, int64_t(progress->download.server_version)); + progress_obj.set(m_progress_download_client_version, + int64_t(progress->download.last_integrated_client_version)); + progress_obj.set(m_progress_upload_server_version, + int64_t(progress->upload.last_integrated_server_version)); + progress_obj.set(m_progress_upload_client_version, int64_t(progress->upload.client_version)); + } + + auto changesets_list = bootstrap_obj.get_linklist(m_changesets); + for (size_t idx = 0; idx < changesets.size(); ++idx) { + auto cur_changeset = changesets_list.create_and_insert_linked_object(changesets_list.size()); + cur_changeset.set(m_changeset_remote_version, int64_t(changesets[idx].remote_version)); + cur_changeset.set(m_changeset_last_integrated_client_version, + int64_t(changesets[idx].last_integrated_local_version)); + cur_changeset.set(m_changeset_origin_file_ident, int64_t(changesets[idx].origin_file_ident)); + cur_changeset.set(m_changeset_origin_timestamp, int64_t(changesets[idx].origin_timestamp)); + cur_changeset.set(m_changeset_original_changeset_size, int64_t(changesets[idx].original_changeset_size)); + BinaryData compressed_data(compressed_changesets[idx].data(), compressed_changesets[idx].size()); + cur_changeset.set(m_changeset_data, compressed_data); + } } tr->commit(); @@ -309,6 +314,7 @@ void PendingBootstrapStore::pop_front_pending(const TransactionRef& tr, size_t c if (bootstrap_table->is_empty()) { return; } + DisableReplication disable_replication(*tr); // We should only have one pending bootstrap at a time. REALM_ASSERT(bootstrap_table->size() == 1); @@ -336,7 +342,7 @@ void PendingBootstrapStore::pop_front_pending(const TransactionRef& tr, size_t c bootstrap_obj.get(m_query_version), changeset_list.size()); } - m_has_pending = (bootstrap_table->is_empty() == false); + m_has_pending = !bootstrap_table->is_empty(); } } // namespace realm::sync From 553d05929a259c838aaf8c0fe0681d6b7508ad41 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Mon, 3 Jun 2024 11:17:11 -0700 Subject: [PATCH 2/5] Rewrite a stale doc comment to reflect that abort_transaction() no longer exists --- src/realm/replication.hpp | 92 +++++++++++---------------------------- 1 file changed, 26 insertions(+), 66 deletions(-) diff --git a/src/realm/replication.hpp b/src/realm/replication.hpp index 7e646525b35..29c45f48436 100644 --- a/src/realm/replication.hpp +++ b/src/realm/replication.hpp @@ -112,70 +112,34 @@ class Replication { /// \defgroup replication_transactions //@{ - /// From the point of view of the Replication class, a transaction is - /// initiated when, and only when the associated Transaction object calls - /// initiate_transact() and the call is successful. The associated - /// Transaction object must terminate every initiated transaction either by - /// calling finalize_commit() or by calling abort_transact(). It may only - /// call finalize_commit(), however, after calling prepare_commit(), and - /// only when prepare_commit() succeeds. If prepare_commit() fails (i.e., - /// throws) abort_transact() must still be called. + /// From the point of view of the Replication class, a write transaction + /// has the following steps: /// - /// The associated Transaction object is supposed to terminate a transaction - /// as soon as possible, and is required to terminate it before attempting - /// to initiate a new one. + /// 1. The parent Transaction acquires exclusive write access to the local Realm. + /// 2. initiate_transact() is called and succeeds. + /// 3. Mutations in the Realm occur, each of which is reported to + /// Replication via one of the member functions at the top of the class + /// (`set()` and friends). + /// 4. prepare_commit() is called as the first phase of two-phase commit. + /// This writes the produced replication log to whatever form of persisted + /// storage the specific Replication subclass uses. As this may be the + /// Realm file itself, this must be called while the write transaction is + /// still active. After this function is called, no more modifications + /// which require replication may be performed until the next transaction + /// is initiated. If this step fails (by throwing an exception), the + /// transaction cannot be committed and must be rolled back. + /// 5. The parent Transaction object performs the commit operation on the local Realm. + /// 6. finalize_commit() is called by the Transaction object. With + /// out-of-Realm replication logs this was used to mark the logs written in + /// step 4 as being valid. With modern in-Realm storage it is merely used + /// to clean up temporary state. /// - /// initiate_transact() is called by the associated Transaction object as - /// part of the initiation of a transaction, and at a time where the caller - /// has acquired exclusive write access to the local Realm. The Replication - /// implementation is allowed to perform "precursor transactions" on the - /// local Realm at this time. During the initiated transaction, the - /// associated DB object must inform the Replication object of all - /// modifying operations by calling set_value() and friends. - /// - /// FIXME: There is currently no way for implementations to perform - /// precursor transactions, since a regular transaction would cause a dead - /// lock when it tries to acquire a write lock. Consider giving access to - /// special non-locking precursor transactions via an extra argument to this - /// function. - /// - /// prepare_commit() serves as the first phase of a two-phase commit. This - /// function is called by the associated Transaction object immediately - /// before the commit operation on the local Realm. The associated - /// Transaction object will then, as the second phase, either call - /// finalize_commit() or abort_transact() depending on whether the commit - /// operation succeeded or not. The Replication implementation is allowed to - /// modify the Realm via the associated Transaction object at this time - /// (important to in-Realm histories). - /// - /// initiate_transact() and prepare_commit() are allowed to block the - /// calling thread if, for example, they need to communicate over the - /// network. If a calling thread is blocked in one of these functions, it - /// must be possible to interrupt the blocking operation by having another - /// thread call interrupt(). The contract is as follows: When interrupt() is - /// called, then any execution of initiate_transact() or prepare_commit(), - /// initiated before the interruption, must complete without blocking, or - /// the execution must be aborted by throwing an Interrupted exception. If - /// initiate_transact() or prepare_commit() throws Interrupted, it counts as - /// a failed operation. - /// - /// finalize_commit() is called by the associated Transaction object - /// immediately after a successful commit operation on the local Realm. This - /// happens at a time where modification of the Realm is no longer possible - /// via the associated Transaction object. In the case of in-Realm - /// histories, the changes are automatically finalized as part of the commit - /// operation performed by the caller prior to the invocation of - /// finalize_commit(), so in that case, finalize_commit() might not need to - /// do anything. - /// - /// abort_transact() is called by the associated Transaction object to - /// terminate a transaction without committing. That is, any transaction - /// that is not terminated by finalize_commit() is terminated by - /// abort_transact(). This could be due to an explicit rollback, or due to a - /// failed commit attempt. - /// - /// Note that finalize_commit() and abort_transact() are not allowed to - /// throw. + /// In previous versions every call to initiate_transact() had to be + /// paired with either a call to finalize_commit() or abort_transaction(). + /// This is no longer the case, and aborted write transactions are no + /// longer reported to Replication. This means that initiate_transact() + /// must discard any pending state and begin a fresh transaction if it is + /// called twice without an intervening finalize_commit(). /// /// \param current_version The version of the snapshot that the current /// transaction is based on. @@ -184,10 +148,6 @@ class Replication { /// updated to reflect the currently bound snapshot, such as when /// _impl::History::update_early_from_top_ref() was called during the /// transition from a read transaction to the current write transaction. - /// - /// \throw Interrupted Thrown by initiate_transact() and prepare_commit() if - /// a blocking operation was interrupted. - void initiate_transact(Group& group, version_type current_version, bool history_updated); /// \param current_version The version of the snapshot that the current /// transaction is based on. From 50e48e0f65095d7a48a63c0e5e99bc7702fc5c46 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Fri, 24 May 2024 10:45:06 -0700 Subject: [PATCH 3/5] Don't emit transaction log instructions for mutations on newly-created objects The non-sync transaction logs are only used to drive notifications and notifications don't care about mutations on objects in the same commit as the objects were created it, so we don't need to emit the instructions at all. This significantly cuts the size of the transaction log for commits which are primarily inserting objects. This does a very basic check for "newly-created" which tracks the most recently created object for each table and skips mutation instructions for that object. This handles recursively creating an object and all of its embedded objects without the overhead of tracking every single object created within a transaction, and insertion workflows will typically not return to an object after creating another object in the same table. This requires adding an additional small amount of tracking for embedded objects, as Replication previously didn't know when new embedded objects were created. --- CHANGELOG.md | 1 + src/realm/replication.cpp | 165 +++++---- src/realm/replication.hpp | 43 ++- src/realm/table.cpp | 4 +- test/object-store/transaction_log_parsing.cpp | 5 +- test/test_replication.cpp | 333 ++++++++++++++++++ test/test_table_helper.hpp | 121 ------- 7 files changed, 468 insertions(+), 204 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08005fb69f0..888bfdcce0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Enhancements * (PR [#????](https://github.com/realm/realm-core/pull/????)) * Include the originating client reset error in AutoClientResetFailure errors. ([#7761](https://github.com/realm/realm-core/pull/7761)) +* Reduce the size of the local transaction log produced by creating objects, improving the performance of insertion-heavy transactions ([PR #7734](https://github.com/realm/realm-core/pull/7734)). ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) diff --git a/src/realm/replication.cpp b/src/realm/replication.cpp index 8b0d48b56eb..350eb61fd05 100644 --- a/src/realm/replication.cpp +++ b/src/realm/replication.cpp @@ -54,6 +54,7 @@ void Replication::do_initiate_transact(Group&, version_type, bool) char* data = m_stream.get_data(); size_t size = m_stream.get_size(); m_encoder.set_buffer(data, data + size); + m_most_recently_created_object.clear(); } Replication::version_type Replication::prepare_commit(version_type orig_version) @@ -100,7 +101,6 @@ void Replication::erase_class(TableKey tk, StringData table_name, size_t) m_encoder.erase_class(tk); // Throws } - void Replication::insert_column(const Table* t, ColKey col_key, DataType type, StringData col_name, Table* target_table) { @@ -140,6 +140,21 @@ void Replication::erase_column(const Table* t, ColKey col_key) m_encoder.erase_column(col_key); // Throws } +void Replication::track_new_object(ObjKey key) +{ + m_selected_obj = key; + m_selected_collection = CollectionId(); + m_newly_created_object = true; + + auto table_index = m_selected_table->get_index_in_group(); + if (table_index >= m_most_recently_created_object.size()) { + if (table_index >= m_most_recently_created_object.capacity()) + m_most_recently_created_object.reserve(table_index * 2); + m_most_recently_created_object.resize(table_index + 1); + } + m_most_recently_created_object[table_index] = m_selected_obj; +} + void Replication::create_object(const Table* t, GlobalKey id) { if (auto logger = would_log(LogLevel::debug)) { @@ -147,6 +162,7 @@ void Replication::create_object(const Table* t, GlobalKey id) } select_table(t); // Throws m_encoder.create_object(id.get_local_key(0)); // Throws + track_new_object(id.get_local_key(0)); // Throws } void Replication::create_object_with_primary_key(const Table* t, ObjKey key, Mixed pk) @@ -157,6 +173,14 @@ void Replication::create_object_with_primary_key(const Table* t, ObjKey key, Mix } select_table(t); // Throws m_encoder.create_object(key); // Throws + track_new_object(key); +} + +void Replication::create_linked_object(const Table* t, ObjKey key) +{ + select_table(t); // Throws + track_new_object(key); // Throws + // Does not need to encode anything as embedded tables can't be observed } void Replication::remove_object(const Table* t, ObjKey key) @@ -177,11 +201,27 @@ void Replication::remove_object(const Table* t, ObjKey key) m_encoder.remove_object(key); // Throws } -void Replication::select_obj(ObjKey key) +void Replication::do_select_table(const Table* table) { - if (key == m_selected_obj) { - return; + m_encoder.select_table(table->get_key()); // Throws + m_selected_table = table; + m_selected_collection = CollectionId(); + m_selected_obj = ObjKey(); +} + +void Replication::do_select_obj(ObjKey key) +{ + m_selected_obj = key; + m_selected_collection = CollectionId(); + + auto table_index = m_selected_table->get_index_in_group(); + if (table_index < m_most_recently_created_object.size()) { + m_newly_created_object = m_most_recently_created_object[table_index] == key; } + else { + m_newly_created_object = false; + } + if (auto logger = would_log(LogLevel::debug)) { auto class_name = m_selected_table->get_class_name(); if (m_selected_table->get_primary_key_column()) { @@ -198,16 +238,28 @@ void Replication::select_obj(ObjKey key) logger->log(LogCategory::object, LogLevel::debug, "Mutating anonymous object '%1'[%2]", class_name, key); } } - m_selected_obj = key; - m_selected_list = CollectionId(); +} + +void Replication::do_select_collection(const CollectionBase& coll) +{ + select_table(coll.get_table().unchecked_ptr()); + ColKey col_key = coll.get_col_key(); + ObjKey key = coll.get_owner_key(); + auto path = coll.get_stable_path(); + + if (select_obj(key)) { + m_encoder.select_collection(col_key, key, path); // Throws + } + m_selected_collection = CollectionId(coll.get_table()->get_key(), key, std::move(path)); } void Replication::do_set(const Table* t, ColKey col_key, ObjKey key, _impl::Instruction variant) { if (variant != _impl::Instruction::instr_SetDefault) { select_table(t); // Throws - select_obj(key); - m_encoder.modify_object(col_key, key); // Throws + if (select_obj(key)) { + m_encoder.modify_object(col_key, key); // Throws + } } } @@ -243,8 +295,9 @@ void Replication::set(const Table* t, ColKey col_key, ObjKey key, Mixed value, _ void Replication::nullify_link(const Table* t, ColKey col_key, ObjKey key) { select_table(t); // Throws - select_obj(key); - m_encoder.modify_object(col_key, key); // Throws + if (select_obj(key)) { + m_encoder.modify_object(col_key, key); // Throws + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Nullify '%1'", t->get_column_name(col_key)); } @@ -258,7 +311,6 @@ void Replication::add_int(const Table* t, ColKey col_key, ObjKey key, int_fast64 } } - Path Replication::get_prop_name(Path&& path) const { auto col_key = path[0].get_col_key(); @@ -308,24 +360,26 @@ void Replication::log_collection_operation(const char* operation, const Collecti void Replication::list_insert(const CollectionBase& list, size_t list_ndx, Mixed value, size_t) { - select_collection(list); // Throws - m_encoder.collection_insert(list.translate_index(list_ndx)); // Throws + if (select_collection(list)) { // Throws + m_encoder.collection_insert(list.translate_index(list_ndx)); // Throws + } log_collection_operation("Insert", list, value, int64_t(list_ndx)); } void Replication::list_set(const CollectionBase& list, size_t list_ndx, Mixed value) { - select_collection(list); // Throws - m_encoder.collection_set(list.translate_index(list_ndx)); // Throws + if (select_collection(list)) { // Throws + m_encoder.collection_set(list.translate_index(list_ndx)); // Throws + } log_collection_operation("Set", list, value, int64_t(list_ndx)); } void Replication::list_erase(const CollectionBase& list, size_t link_ndx) { - select_collection(list); // Throws - m_encoder.collection_erase(list.translate_index(link_ndx)); // Throws + if (select_collection(list)) { // Throws + m_encoder.collection_erase(list.translate_index(link_ndx)); // Throws + } if (auto logger = would_log(LogLevel::trace)) { - logger->log(LogCategory::object, LogLevel::trace, " Erase '%1' at position %2", get_prop_name(list.get_short_path()), link_ndx); } @@ -333,8 +387,9 @@ void Replication::list_erase(const CollectionBase& list, size_t link_ndx) void Replication::list_move(const CollectionBase& list, size_t from_link_ndx, size_t to_link_ndx) { - select_collection(list); // Throws - m_encoder.collection_move(list.translate_index(from_link_ndx), list.translate_index(to_link_ndx)); // Throws + if (select_collection(list)) { // Throws + m_encoder.collection_move(list.translate_index(from_link_ndx), list.translate_index(to_link_ndx)); // Throws + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Move %1 to %2 in '%3'", from_link_ndx, to_link_ndx, get_prop_name(list.get_short_path())); @@ -343,55 +398,24 @@ void Replication::list_move(const CollectionBase& list, size_t from_link_ndx, si void Replication::set_insert(const CollectionBase& set, size_t set_ndx, Mixed value) { - select_collection(set); // Throws - m_encoder.collection_insert(set_ndx); // Throws - log_collection_operation("Insert", set, value, Mixed()); + Replication::list_insert(set, set_ndx, value, 0); // Throws } -void Replication::set_erase(const CollectionBase& set, size_t set_ndx, Mixed value) +void Replication::set_erase(const CollectionBase& set, size_t set_ndx, Mixed) { - select_collection(set); // Throws - m_encoder.collection_erase(set_ndx); // Throws - if (auto logger = would_log(LogLevel::trace)) { - logger->log(LogCategory::object, LogLevel::trace, " Erase %1 from '%2'", value, - get_prop_name(set.get_short_path())); - } + Replication::list_erase(set, set_ndx); // Throws } void Replication::set_clear(const CollectionBase& set) { - select_collection(set); // Throws - m_encoder.collection_clear(set.size()); // Throws - if (auto logger = would_log(LogLevel::trace)) { - logger->log(LogCategory::object, LogLevel::trace, " Clear '%1'", get_prop_name(set.get_short_path())); - } -} - -void Replication::do_select_table(const Table* table) -{ - m_encoder.select_table(table->get_key()); // Throws - m_selected_table = table; - m_selected_list = CollectionId(); - m_selected_obj = ObjKey(); -} - -void Replication::do_select_collection(const CollectionBase& list) -{ - select_table(list.get_table().unchecked_ptr()); - ColKey col_key = list.get_col_key(); - ObjKey key = list.get_owner_key(); - auto path = list.get_stable_path(); - - select_obj(key); - - m_encoder.select_collection(col_key, key, path); // Throws - m_selected_list = CollectionId(list.get_table()->get_key(), key, std::move(path)); + Replication::list_clear(set); // Throws } void Replication::list_clear(const CollectionBase& list) { - select_collection(list); // Throws - m_encoder.collection_clear(list.size()); // Throws + if (select_collection(list)) { // Throws + m_encoder.collection_clear(list.size()); // Throws + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Clear '%1'", get_prop_name(list.get_short_path())); } @@ -399,8 +423,9 @@ void Replication::list_clear(const CollectionBase& list) void Replication::link_list_nullify(const Lst& list, size_t link_ndx) { - select_collection(list); - m_encoder.collection_erase(link_ndx); + if (select_collection(list)) { // Throws + m_encoder.collection_erase(link_ndx); + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Nullify '%1' position %2", m_selected_table->get_column_name(list.get_col_key()), link_ndx); @@ -409,22 +434,25 @@ void Replication::link_list_nullify(const Lst& list, size_t link_ndx) void Replication::dictionary_insert(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { - select_collection(dict); - m_encoder.collection_insert(ndx); + if (select_collection(dict)) { // Throws + m_encoder.collection_insert(ndx); + } log_collection_operation("Insert", dict, value, key); } void Replication::dictionary_set(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { - select_collection(dict); - m_encoder.collection_set(ndx); + if (select_collection(dict)) { // Throws + m_encoder.collection_set(ndx); + } log_collection_operation("Set", dict, value, key); } void Replication::dictionary_erase(const CollectionBase& dict, size_t ndx, Mixed key) { - select_collection(dict); - m_encoder.collection_erase(ndx); + if (select_collection(dict)) { // Throws + m_encoder.collection_erase(ndx); + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Erase %1 from '%2'", key, get_prop_name(dict.get_short_path())); @@ -433,8 +461,9 @@ void Replication::dictionary_erase(const CollectionBase& dict, size_t ndx, Mixed void Replication::dictionary_clear(const CollectionBase& dict) { - select_collection(dict); - m_encoder.collection_clear(dict.size()); + if (select_collection(dict)) { // Throws + m_encoder.collection_clear(dict.size()); + } if (auto logger = would_log(LogLevel::trace)) { logger->log(LogCategory::object, LogLevel::trace, " Clear '%1'", get_prop_name(dict.get_short_path())); } diff --git a/src/realm/replication.hpp b/src/realm/replication.hpp index 29c45f48436..5bae6e7c6ee 100644 --- a/src/realm/replication.hpp +++ b/src/realm/replication.hpp @@ -77,6 +77,7 @@ class Replication { virtual void create_object(const Table*, GlobalKey); virtual void create_object_with_primary_key(const Table*, ObjKey, Mixed); + void create_linked_object(const Table*, ObjKey); virtual void remove_object(const Table*, ObjKey); virtual void typed_link_change(const Table*, ColKey, TableKey); @@ -389,18 +390,31 @@ class Replication { _impl::TransactLogBufferStream m_stream; _impl::TransactLogEncoder m_encoder{m_stream}; util::Logger* m_logger = nullptr; - mutable const Table* m_selected_table = nullptr; - mutable ObjKey m_selected_obj; - mutable CollectionId m_selected_list; + const Table* m_selected_table = nullptr; + ObjKey m_selected_obj; + CollectionId m_selected_collection; + // The ObjKey of the most recently created object for each table (indexed + // by the Table's index in the group). Most insertion patterns will only + // ever update the most recently created object, so this is almost as + // effective as tracking all newly created objects but much cheaper. + std::vector m_most_recently_created_object; + // When true, the currently selected object was created in this transaction + // and we don't need to emit instructions for mutations on it + bool m_newly_created_object = false; void unselect_all() noexcept; void select_table(const Table*); // unselects link list and obj - void select_obj(ObjKey key); - void select_collection(const CollectionBase&); + bool select_obj(ObjKey key); + bool select_collection(const CollectionBase&); void do_select_table(const Table*); + void do_select_obj(ObjKey key); void do_select_collection(const CollectionBase&); + // Mark this ObjKey as being a newly created object that should not emit + // mutation instructions + void track_new_object(ObjKey); + void do_set(const Table*, ColKey col_key, ObjKey key, _impl::Instruction variant = _impl::instr_Set); void log_collection_operation(const char* operation, const CollectionBase& collection, Mixed value, Mixed index) const; @@ -445,11 +459,11 @@ inline size_t Replication::transact_log_size() return m_encoder.write_position() - m_stream.get_data(); } - inline void Replication::unselect_all() noexcept { m_selected_table = nullptr; - m_selected_list = CollectionId(); + m_selected_collection = CollectionId(); + m_newly_created_object = false; } inline void Replication::select_table(const Table* table) @@ -458,11 +472,20 @@ inline void Replication::select_table(const Table* table) do_select_table(table); // Throws } -inline void Replication::select_collection(const CollectionBase& list) +inline bool Replication::select_collection(const CollectionBase& coll) +{ + if (CollectionId(coll) != m_selected_collection) { + do_select_collection(coll); // Throws + } + return !m_newly_created_object; +} + +inline bool Replication::select_obj(ObjKey key) { - if (CollectionId(list) != m_selected_list) { - do_select_collection(list); // Throws + if (key != m_selected_obj) { + do_select_obj(key); } + return !m_newly_created_object; } inline void Replication::rename_class(TableKey table_key, StringData) diff --git a/src/realm/table.cpp b/src/realm/table.cpp index 1bb895eea2e..306a14707b0 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -2253,9 +2253,11 @@ Obj Table::create_linked_object() GlobalKey object_id = allocate_object_id_squeezed(); ObjKey key = object_id.get_local_key(get_sync_file_id()); - REALM_ASSERT(key.value >= 0); + if (auto repl = get_repl()) + repl->create_linked_object(this, key); + Obj obj = m_clusters.insert(key, {}); return obj; diff --git a/test/object-store/transaction_log_parsing.cpp b/test/object-store/transaction_log_parsing.cpp index 316edc50c60..a4f1bc16e9f 100644 --- a/test/object-store/transaction_log_parsing.cpp +++ b/test/object-store/transaction_log_parsing.cpp @@ -118,9 +118,6 @@ class CaptureHelper { REQUIRE(m_list.is_attached()); // and make sure we end up with the same end result - if (m_initial.size() != m_list.size()) { - std::cout << "Error " << m_list.size() << std::endl; - } REQUIRE(m_initial.size() == m_list.size()); for (size_t i = 0; i < m_initial.size(); ++i) CHECK(m_initial[i] == m_list.get_key(i)); @@ -392,7 +389,7 @@ TEST_CASE("Transaction log parsing: changeset calcuation") { } } - SECTION("LinkView change information") { + SECTION("List change information") { auto r = Realm::get_shared_realm(config); r->update_schema({ {"origin", {{"array", PropertyType::Array | PropertyType::Object, "target"}}}, diff --git a/test/test_replication.cpp b/test/test_replication.cpp index e46589d954d..5e0fe845dbf 100644 --- a/test/test_replication.cpp +++ b/test/test_replication.cpp @@ -65,6 +65,72 @@ using unit_test::TestContext; // `experiments/testcase.cpp` and then run `sh build.sh // check-testcase` (or one of its friends) from the command line. +namespace { +class ReplSyncClient : public Replication { +public: + ReplSyncClient(int history_schema_version, uint64_t file_ident = 0) + : m_file_ident(file_ident) + , m_history_schema_version(history_schema_version) + { + } + + version_type prepare_changeset(const char*, size_t, version_type version) override + { + if (!m_arr) { + using gf = _impl::GroupFriend; + Allocator& alloc = gf::get_alloc(*m_group); + m_arr = std::make_unique(alloc); + gf::prepare_history_parent(*m_group, *m_arr, hist_SyncClient, m_history_schema_version, 0); + m_arr->create(); + } + return version + 1; + } + + bool is_upgraded() const + { + return m_upgraded; + } + + bool is_upgradable_history_schema(int) const noexcept override + { + return true; + } + + void upgrade_history_schema(int) override + { + m_group->set_sync_file_id(m_file_ident); + m_upgraded = true; + } + + HistoryType get_history_type() const noexcept override + { + return hist_SyncClient; + } + + int get_history_schema_version() const noexcept override + { + return m_history_schema_version; + } + + std::unique_ptr<_impl::History> _create_history_read() override + { + return {}; + } + +private: + Group* m_group; + std::unique_ptr m_arr; + uint64_t m_file_ident; + int m_history_schema_version; + bool m_upgraded = false; + + void do_initiate_transact(Group& group, version_type version, bool hist_updated) override + { + Replication::do_initiate_transact(group, version, hist_updated); + m_group = &group; + } +}; + TEST(Replication_HistorySchemaVersionNormal) { SHARED_GROUP_TEST_PATH(path); @@ -198,4 +264,271 @@ TEST(Replication_WriteWithoutHistory) } } +struct ObjectMutationObserver : _impl::NoOpTransactionLogParser { + unit_test::TestContext& test_context; + std::set> expected_creations; + std::set> expected_modifications; + + ObjectMutationObserver(unit_test::TestContext& test_context, + std::initializer_list> creations, + std::initializer_list> modifications) + : test_context(test_context) + { + for (auto [tk, ok] : creations) { + expected_creations.emplace(tk, ObjKey(ok)); + } + for (auto [tk, ok, ck] : modifications) { + expected_modifications.emplace(tk, ObjKey(ok), ck); + } + } + + ObjectMutationObserver& operator=(ObjectMutationObserver&& obs) noexcept + { + expected_creations.swap(obs.expected_creations); + expected_modifications.swap(obs.expected_modifications); + return *this; + } + + bool create_object(ObjKey obj_key) + { + CHECK(expected_creations.erase(std::pair(get_current_table(), obj_key))); + return true; + } + bool modify_object(ColKey col, ObjKey obj) + { + CHECK(expected_modifications.erase(std::tuple(get_current_table(), obj, col))); + return true; + } + bool remove_object(ObjKey) + { + return true; + } + + void check() + { + CHECK(expected_creations.empty()); + CHECK(expected_modifications.empty()); + } +}; + +template +void expect(DBRef db, ObjectMutationObserver& observer, Fn&& write) +{ + auto read = db->start_read(); + { + auto tr = db->start_write(); + write(*tr); + tr->commit(); + } + read->advance_read(&observer); + observer.check(); +} + +TEST(Replication_MutationsOnNewlyCreatedObject) +{ + SHARED_GROUP_TEST_PATH(path); + auto db = DB::create(make_in_realm_history(), path); + + TableKey tk; + ColKey col; + { + auto tr = db->start_write(); + auto table = tr->add_table("table"); + tk = table->get_key(); + col = table->add_column(type_Int, "value"); + tr->commit(); + } + + // Object creations with immediate mutations should report creations only + auto obs = ObjectMutationObserver(test_context, {{tk, 0}, {tk, 1}}, {}); + expect(db, obs, [](auto& tr) { + auto table = tr.get_table("table"); + table->create_object().set_all(1); + table->create_object().set_all(1); + }); + + // Mutating existing objects should report modifications + obs = ObjectMutationObserver(test_context, {}, {{tk, 0, col}, {tk, 1, col}}); + expect(db, obs, [](auto& tr) { + auto table = tr.get_table("table"); + table->get_object(0).set_all(1); + table->get_object(1).set_all(1); + }); + + // Create two objects and then mutate them. We only track the most recently + // created object, so this emits a mutation for the first object but not + // the second. + obs = ObjectMutationObserver(test_context, {{tk, 2}, {tk, 3}}, {{tk, 2, col}}); + expect(db, obs, [](auto& tr) { + auto table = tr.get_table("table"); + auto obj1 = table->create_object(); + auto obj2 = table->create_object(); + obj1.set_all(1); + obj2.set_all(1); + }); + + TableKey tk2; + ColKey col2; + { + auto tr = db->start_write(); + auto table = tr->add_table("table 2"); + tk2 = table->get_key(); + col2 = table->add_column(type_Int, "value"); + tr->commit(); + } + + // Creating an object in one table and then modifying the object with the + // same ObjKey in a different table + obs = ObjectMutationObserver(test_context, {{tk2, 0}}, {{tk, 0, col}}); + expect(db, obs, [&](auto& tr) { + auto table1 = tr.get_table(tk); + auto table2 = tr.get_table(tk2); + auto obj1 = table1->get_object(0); + auto obj2 = table2->create_object(); + CHECK_EQUAL(obj1.get_key(), obj2.get_key()); + obj1.set_all(1); + obj2.set_all(1); + }); + + // Mutating an object whose Table has an index in group greater than the + // higest of any created object after creating an object, which has to clear + // the is-new-object flag + obs = ObjectMutationObserver(test_context, {{tk, 4}}, {{tk2, 0, col2}}); + expect(db, obs, [&](auto& tr) { + auto table1 = tr.get_table(tk); + auto table2 = tr.get_table(tk2); + auto obj1 = table1->create_object(); + auto obj2 = table2->get_object(0); + obj1.set_all(1); + obj2.set_all(1); + }); + + // Splitting object creation and mutation over two different writes with the + // same transaction object should produce mutation instructions + obs = ObjectMutationObserver(test_context, {{tk, 5}}, {{tk, 5, col}}); + { + auto read = db->start_read(); + auto tr = db->start_write(); + auto table = tr->get_table(tk); + auto obj = table->create_object(); + tr->commit_and_continue_as_read(); + tr->promote_to_write(); + obj.set_all(1); + tr->commit_and_continue_as_read(); + read->advance_read(&obs); + obs.check(); + } +} + +TEST(Replication_MutationsOnNewlyCreatedObject_Link) +{ + SHARED_GROUP_TEST_PATH(path); + auto db = DB::create(make_in_realm_history(), path); + auto tr = db->start_write(); + + auto target_table = tr->add_table("target table"); + auto tk_target = target_table->get_key(); + auto ck_target_value = target_table->add_column(type_Int, "value"); + auto embedded_table = tr->add_table("embedded table", Table::Type::Embedded); + embedded_table->add_column(type_Int, "value"); + + auto table = tr->add_table("table"); + auto tk = table->get_key(); + ColKey ck_link_1 = table->add_column(*target_table, "link 1"); + ColKey ck_link_2 = table->add_column(*target_table, "link 2"); + ColKey ck_embedded_1 = table->add_column(*embedded_table, "embedded 1"); + ColKey ck_embedded_2 = table->add_column(*embedded_table, "embedded 2"); + tr->commit(); + + // Each top-level object creation is reported along with the mutation on + // target_1 due to that both target objects are created before the mutations. + // Nothing is reported for embedded objects + auto obs = ObjectMutationObserver(test_context, {{tk, 0}, {tk_target, 0}, {tk_target, 1}}, + {{tk_target, 0, ck_target_value}}); + expect(db, obs, [&](auto& tr) { + auto table = tr.get_table(tk); + auto target_table = tr.get_table(tk_target); + Obj obj = table->create_object(); + Obj target_1 = target_table->create_object(); + Obj target_2 = target_table->create_object(); + + obj.set(ck_link_1, target_1.get_key()); + obj.set(ck_link_2, target_2.get_key()); + target_1.set_all(1); + target_2.set_all(1); + + obj.create_and_set_linked_object(ck_embedded_1).set_all(1); + obj.create_and_set_linked_object(ck_embedded_2).set_all(1); + }); + + // Nullifying links via object deletions in both new and pre-existing objects + // only reports the mutation in the pre-existing object + obs = ObjectMutationObserver(test_context, {{tk, 1}}, {{tk, 0, ck_link_1}}); + expect(db, obs, [&](auto& tr) { + auto table = tr.get_table(tk); + auto target_table = tr.get_table(tk_target); + Obj obj = table->create_object(); + obj.set(ck_link_1, target_table->get_object(0).get_key()); + obj.set(ck_link_2, target_table->get_object(1).get_key()); + + target_table->get_object(0).remove(); + }); +} + +TEST(Replication_MutationsOnNewlyCreatedObject_Collections) +{ + SHARED_GROUP_TEST_PATH(path); + auto db = DB::create(make_in_realm_history(), path); + auto tr = db->start_write(); + + auto table = tr->add_table("table"); + auto tk = table->get_key(); + ColKey ck_value = table->add_column(type_Int, "value"); + ColKey ck_value_set = table->add_column_set(type_Int, "value set"); + ColKey ck_value_list = table->add_column_list(type_Int, "value list"); + ColKey ck_value_dictionary = table->add_column_dictionary(type_Int, "value dictionary"); + + auto target_table = tr->add_table("target table"); + auto tk_target = target_table->get_key(); + auto ck_target_value = target_table->add_column(type_Int, "value"); + ColKey ck_obj_set = table->add_column_set(*target_table, "obj set"); + ColKey ck_obj_list = table->add_column_list(*target_table, "obj list"); + ColKey ck_obj_dictionary = table->add_column_dictionary(*target_table, "obj dictionary"); + + auto embedded_table = tr->add_table("embedded table", Table::Type::Embedded); + auto ck_embedded_value = embedded_table->add_column(type_Int, "value"); + ColKey ck_embedded_list = table->add_column_list(*embedded_table, "embedded list"); + ColKey ck_embedded_dictionary = table->add_column_dictionary(*embedded_table, "embedded dictionary"); + + tr->commit(); + + auto obs = ObjectMutationObserver(test_context, {{tk, 0}, {tk_target, 0}}, {}); + expect(db, obs, [&](auto& tr) { + // Should report object creation but none of these mutations + auto table = tr.get_table(tk); + Obj obj = table->create_object(); + obj.set(ck_value, 1); + obj.get_set(ck_value_set).insert(1); + obj.get_list(ck_value_list).add(1); + obj.get_dictionary(ck_value_dictionary).insert("a", 1); + + // Should report the object creation but not the mutations on either object, + // as they're both the most recently created object in each table + auto target_table = tr.get_table(tk_target); + Obj target_obj = target_table->create_object(); + target_obj.set(ck_target_value, 1); + obj.get_linkset(ck_obj_set).insert(target_obj.get_key()); + obj.get_linklist(ck_obj_list).add(target_obj.get_key()); + obj.get_dictionary(ck_obj_dictionary).insert("a", target_obj.get_key()); + + // Should not produce any instructions: embedded object creations aren't + // replicated (as you can't observe embedded tables directly), and the + // mutations are on the newest object for each table + obj.get_linklist(ck_embedded_list).create_and_insert_linked_object(0).set(ck_embedded_value, 1); + obj.get_dictionary(ck_embedded_dictionary).create_and_insert_linked_object("a").set(ck_embedded_value, 1); + }); +} + +} // anonymous namespace + #endif // TEST_REPLICATION diff --git a/test/test_table_helper.hpp b/test/test_table_helper.hpp index 0b866d20d5d..32f4a5fa88f 100644 --- a/test/test_table_helper.hpp +++ b/test/test_table_helper.hpp @@ -32,127 +32,6 @@ class ObjKeyVector : public std::vector { } }; -class MyTrivialReplication : public Replication { -public: - HistoryType get_history_type() const noexcept override - { - return hist_None; - } - - int get_history_schema_version() const noexcept override - { - return 0; - } - - bool is_upgradable_history_schema(int) const noexcept override - { - REALM_ASSERT(false); - return false; - } - - void upgrade_history_schema(int) override - { - REALM_ASSERT(false); - } - - _impl::History* _get_history_write() override - { - return nullptr; - } - - std::unique_ptr<_impl::History> _create_history_read() override - { - return {}; - } - - void do_initiate_transact(Group& group, version_type version, bool hist_updated) override - { - Replication::do_initiate_transact(group, version, hist_updated); - m_group = &group; - } - -protected: - version_type prepare_changeset(const char* data, size_t size, version_type orig_version) override - { - m_incoming_changeset = util::Buffer(size); // Throws - std::copy(data, data + size, m_incoming_changeset.data()); - // Make space for the new changeset in m_changesets such that we can be - // sure no exception will be thrown whan adding the changeset in - // finalize_changeset(). - m_changesets.reserve(m_changesets.size() + 1); // Throws - return orig_version + 1; - } - - void finalize_changeset() noexcept override - { - // The following operation will not throw due to the space reservation - // carried out in prepare_new_changeset(). - m_changesets.push_back(std::move(m_incoming_changeset)); - } - - util::Buffer m_incoming_changeset; - std::vector> m_changesets; - Group* m_group; -}; - -class ReplSyncClient : public MyTrivialReplication { -public: - ReplSyncClient(int history_schema_version, uint64_t file_ident = 0) - : m_history_schema_version(history_schema_version) - , m_file_ident(file_ident) - { - } - - void initialize(DB& sg) override - { - Replication::initialize(sg); - } - - version_type prepare_changeset(const char*, size_t, version_type version) override - { - if (!m_arr) { - using gf = _impl::GroupFriend; - Allocator& alloc = gf::get_alloc(*m_group); - m_arr = std::make_unique(alloc); - gf::prepare_history_parent(*m_group, *m_arr, hist_SyncClient, m_history_schema_version, 0); - m_arr->create(); - m_arr->add(BinaryData("Changeset")); - } - return version + 1; - } - - bool is_upgraded() const - { - return m_upgraded; - } - - bool is_upgradable_history_schema(int) const noexcept override - { - return true; - } - - void upgrade_history_schema(int) override - { - m_group->set_sync_file_id(m_file_ident); - m_upgraded = true; - } - - HistoryType get_history_type() const noexcept override - { - return hist_SyncClient; - } - - int get_history_schema_version() const noexcept override - { - return m_history_schema_version; - } - -private: - int m_history_schema_version; - uint64_t m_file_ident; - bool m_upgraded = false; - std::unique_ptr m_arr; -}; } // namespace realm enum Days { Mon, Tue, Wed, Thu, Fri, Sat, Sun }; From 9affcbc178961cd293d22d7f6fa0bb644e5d2f12 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 29 May 2024 14:49:32 -0700 Subject: [PATCH 4/5] Delete some now-unused functions These were used for converting to the file format where ObjKeys were derived from primary keys. --- src/realm/table.cpp | 10 ---------- src/realm/table.hpp | 3 --- 2 files changed, 13 deletions(-) diff --git a/src/realm/table.cpp b/src/realm/table.cpp index 306a14707b0..f41ec412ad5 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -1577,16 +1577,6 @@ uint64_t Table::allocate_sequence_number() return sn; } -void Table::set_sequence_number(uint64_t seq) -{ - m_top.set(top_position_for_sequence_number, RefOrTagged::make_tagged(seq)); -} - -void Table::set_collision_map(ref_type ref) -{ - m_top.set(top_position_for_collision_map, RefOrTagged::make_ref(ref)); -} - void Table::set_col_key_sequence_number(uint64_t seq) { m_top.set(top_position_for_column_key, RefOrTagged::make_tagged(seq)); diff --git a/src/realm/table.hpp b/src/realm/table.hpp index e175c6548f4..80496b12fb9 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -415,9 +415,6 @@ class Table { TableKey get_key() const noexcept; uint64_t allocate_sequence_number(); - // Used by upgrade - void set_sequence_number(uint64_t seq); - void set_collision_map(ref_type ref); // Used for testing purposes. void set_col_key_sequence_number(uint64_t seq); From 5d5e26d6849e1232bd6198cbbbd929d32b5d4f85 Mon Sep 17 00:00:00 2001 From: Thomas Goyne Date: Wed, 29 May 2024 14:00:45 -0700 Subject: [PATCH 5/5] Fix reading CT history from synchronized Realms in trawler --- CHANGELOG.md | 1 + src/realm/exec/realm_trawler.cpp | 63 +++++++++++++++++++------------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 888bfdcce0f..c0f962fba52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ ### Internals * Removed references to `stitch_` fields in access tokens in sync unit tests ([PR #7769](https://github.com/realm/realm-core/pull/7769)). * Added back iOS simulator testing to evergreen after Jenkins went away ([PR #7758](https://github.com/realm/realm-core/pull/7758)). +* `realm-trawler -c` did not work on Realm using SyncClient history ([PR #7734](https://github.com/realm/realm-core/pull/7734)). ---------------------------------------------- diff --git a/src/realm/exec/realm_trawler.cpp b/src/realm/exec/realm_trawler.cpp index 307fb4fc9f8..d6882d76192 100644 --- a/src/realm/exec/realm_trawler.cpp +++ b/src/realm/exec/realm_trawler.cpp @@ -65,7 +65,7 @@ void consolidate_lists(std::vector& list, std::vector& list2) for (auto it = list.begin() + 1; it != list.end(); ++it) { if (prev->start + prev->length != it->start) { if (prev->start + prev->length > it->start) { - std::cout << "*** Overlapping entries:" << std::endl; + std::cout << "*** Overlapping entries:\n"; std::cout << std::hex; std::cout << " 0x" << prev->start << "..0x" << prev->start + prev->length << std::endl; std::cout << " 0x" << it->start << "..0x" << it->start + it->length << std::endl; @@ -455,7 +455,16 @@ class Group : public Array { for (size_t n = 0; n < m_history.size(); n++) { ref = m_history.get_ref(n); Node node(m_alloc, ref); - ret.emplace_back(node.data(), node.size()); + if (!node.has_refs()) { + ret.emplace_back(node.data(), node.size()); + continue; + } + + Array array(m_alloc, ref); + for (size_t j = 0; j < array.size(); ++j) { + Node node(m_alloc, array.get_ref(j)); + ret.emplace_back(node.data(), node.size()); + } } } return ret; @@ -473,7 +482,7 @@ class Group : public Array { if (m_evacuation_info.valid()) { ostr << "Evacuation limit: " << size_t(m_evacuation_info.get_val(0)); if (m_evacuation_info.get_val(1)) { - ostr << " Scan done" << std::endl; + ostr << " Scan done\n"; } else { ostr << " Progress: ["; @@ -482,7 +491,7 @@ class Group : public Array { ostr << ','; ostr << m_evacuation_info.get_val(i); } - ostr << "]" << std::endl; + ostr << "]\n"; } } } @@ -588,14 +597,14 @@ std::ostream& operator<<(std::ostream& ostr, const Group& g) } } else { - ostr << "*** Invalid group ***" << std::endl; + ostr << "*** Invalid group ***\n"; } return ostr; } void Table::print_columns(const Group& group) const { - std::cout << " <" << m_table_type << ">" << std::endl; + std::cout << " <" << m_table_type << ">\n"; for (unsigned i = 0; i < m_column_names.size(); i++) { auto type = realm::ColumnType(m_column_types.get_val(i) & 0xFFFF); auto attr = realm::ColumnAttr(m_column_attributes.get_val(i)); @@ -646,7 +655,7 @@ void Table::print_columns(const Group& group) const void Group::print_schema() const { if (valid()) { - std::cout << "Tables: " << std::endl; + std::cout << "Tables: \n"; for (unsigned i = 0; i < get_nb_tables(); i++) { Table* table = get_table(i); @@ -808,7 +817,7 @@ void RealmFile::node_scan() } uint64_t bad_ref = 0; if (free_list.empty()) { - std::cout << "*** No free list - results may be unreliable ***" << std::endl; + std::cout << "*** No free list - results may be unreliable ***\n"; } std::cout << std::hex; while (ref < end) { @@ -847,7 +856,7 @@ void RealmFile::node_scan() << "Start: 0x" << bad_ref << "..0x" << end << std::endl; } std::cout << std::dec; - std::cout << "Allocated space:" << std::endl; + std::cout << "Allocated space:\n"; for (auto s : sizes) { std::cout << " Size: " << s.first << " count: " << s.second << std::endl; } @@ -870,7 +879,7 @@ void RealmFile::memory_leaks() consolidate_lists(nodes, free_blocks); auto it = nodes.begin(); if (nodes.size() > 1) { - std::cout << "Memory leaked:" << std::endl; + std::cout << "Memory leaked:\n"; auto prev = it; ++it; while (it != nodes.end()) { @@ -882,7 +891,7 @@ void RealmFile::memory_leaks() } else { REALM_ASSERT(it->length == m_group->get_file_size()); - std::cout << "No memory leaks" << std::endl; + std::cout << "No memory leaks\n"; } } } @@ -891,7 +900,7 @@ void RealmFile::free_list_info() const { std::map free_sizes; std::map pinned_sizes; - std::cout << "Free space:" << std::endl; + std::cout << "Free space:\n"; auto free_list = m_group->get_free_list(); uint64_t pinned_free_list_size = 0; uint64_t total_free_list_size = 0; @@ -911,11 +920,11 @@ void RealmFile::free_list_info() const ++it; } - std::cout << "Free space sizes:" << std::endl; + std::cout << "Free space sizes:\n"; for (auto s : free_sizes) { std::cout << " Size: " << s.first << " count: " << s.second << std::endl; } - std::cout << "Pinned sizes:" << std::endl; + std::cout << "Pinned sizes:\n"; for (auto s : pinned_sizes) { std::cout << " Size: " << s.first << " count: " << s.second << std::endl; } @@ -984,7 +993,7 @@ class HistoryLogger { bool dictionary_clear(size_t) { - std::cout << "Dictionary clear " << std::endl; + std::cout << "Dictionary clear \n"; return true; } @@ -1052,8 +1061,13 @@ void RealmFile::changes() const for (auto c : changesets) { realm::util::SimpleInputStream stream(c); - parser.parse(stream, logger); - std::cout << "--------------------------------------------" << std::endl; + try { + parser.parse(stream, logger); + } + catch (const std::exception& ex) { + std::cout << "Bad history: " << ex.what() << "\n"; + } + std::cout << "--------------------------------------------\n"; } } @@ -1161,15 +1175,14 @@ int main(int argc, const char* argv[]) } } else { - std::cout << "Usage: realm-trawler [-afmsw] [--keyfile file-with-binary-crypt-key] [--hexkey " + std::cout << "Usage: realm-trawler [-cfmsw] [--keyfile file-with-binary-crypt-key] [--hexkey " "crypt-key-in-hex] [--top " - "top_ref] " - << std::endl; - std::cout << " c : dump changelog" << std::endl; - std::cout << " f : free list analysis" << std::endl; - std::cout << " m : memory leak check" << std::endl; - std::cout << " s : schema dump" << std::endl; - std::cout << " w : node walk" << std::endl; + "top_ref] \n"; + std::cout << " c : dump changelog\n"; + std::cout << " f : free list analysis\n"; + std::cout << " m : memory leak check\n"; + std::cout << " s : schema dump\n"; + std::cout << " w : node walk\n"; } return 0;