From f9f99769f3153da9b27f6b7360f5675780615637 Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Tue, 10 Jul 2018 07:18:09 +0800 Subject: [PATCH 1/4] feat(language): shared user dictionary per language (Closes #184) --- src/rime/dict/user_dictionary.cc | 6 +++--- src/rime/dict/user_dictionary.h | 7 ++++--- src/rime/gear/memory.cc | 9 ++++++--- src/rime/gear/memory.h | 9 +++------ src/rime/gear/poet.h | 8 ++++---- src/rime/gear/table_translator.cc | 17 +++------------- src/rime/gear/table_translator.h | 14 +++++++------- src/rime/gear/translator_commons.h | 8 ++++---- src/rime/language.h | 31 ++++++++++++++++++++++++++++++ 9 files changed, 65 insertions(+), 44 deletions(-) create mode 100644 src/rime/language.h diff --git a/src/rime/dict/user_dictionary.cc b/src/rime/dict/user_dictionary.cc index 078d3d984..347518c9f 100644 --- a/src/rime/dict/user_dictionary.cc +++ b/src/rime/dict/user_dictionary.cc @@ -116,8 +116,8 @@ bool UserDictEntryIterator::Next() { // UserDictionary members -UserDictionary::UserDictionary(const an& db) - : db_(db) { +UserDictionary::UserDictionary(Language language, an db) + : language_(language), db_(db) { } UserDictionary::~UserDictionary() { @@ -519,7 +519,7 @@ UserDictionary* UserDictionaryComponent::Create(const Ticket& ticket) { db.reset(component->Create(dict_name)); db_pool_[dict_name] = db; } - return new UserDictionary(db); + return new UserDictionary(Language{dict_name}, db); } } // namespace rime diff --git a/src/rime/dict/user_dictionary.h b/src/rime/dict/user_dictionary.h index 67f1427fb..3b0427495 100644 --- a/src/rime/dict/user_dictionary.h +++ b/src/rime/dict/user_dictionary.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -50,7 +51,7 @@ struct Ticket; class UserDictionary : public Class { public: - explicit UserDictionary(const an& db); + UserDictionary(Language language, an db); virtual ~UserDictionary(); void Attach(const an& table, const an& prism); @@ -76,7 +77,7 @@ class UserDictionary : public Class { bool RevertRecentTransaction(); bool CommitPendingTransaction(); - const string& name() const { return name_; } + const Language* language() const { return &language_; } TickCount tick() const { return tick_; } static an CreateDictEntry(const string& key, @@ -94,7 +95,7 @@ class UserDictionary : public Class { DfsState* state); private: - string name_; + const Language language_; an db_; an
table_; an prism_; diff --git a/src/rime/gear/memory.cc b/src/rime/gear/memory.cc index 644bfd7bb..a0ae73407 100644 --- a/src/rime/gear/memory.cc +++ b/src/rime/gear/memory.cc @@ -92,6 +92,10 @@ bool Memory::DiscardSession() { return user_dict_ && user_dict_->RevertRecentTransaction(); } +const Language* Memory::language() const { + return user_dict_ ? user_dict_->language() : nullptr; +} + void Memory::OnCommit(Context* ctx) { if (!user_dict_|| user_dict_->readonly()) return; @@ -100,7 +104,7 @@ void Memory::OnCommit(Context* ctx) { for (auto& seg : ctx->composition()) { auto phrase = As(Candidate::GetGenuineCandidate( seg.GetSelectedCandidate())); - bool recognized = phrase && phrase->language() == language(); + bool recognized = Language::intelligible(phrase, this); if (recognized) { commit_entry.AppendPhrase(phrase); } @@ -119,8 +123,7 @@ void Memory::OnDeleteEntry(Context* ctx) { return; auto phrase = As(Candidate::GetGenuineCandidate( ctx->GetSelectedCandidate())); - bool recognized = phrase && phrase->language() == language(); - if (recognized) { + if (Language::intelligible(phrase, this)) { const DictEntry& entry(phrase->entry()); LOG(INFO) << "deleting entry: '" << entry.text << "'."; user_dict_->UpdateEntry(entry, -1); // mark as deleted in user dict diff --git a/src/rime/gear/memory.h b/src/rime/gear/memory.h index 5ec8145f7..01908d13a 100644 --- a/src/rime/gear/memory.h +++ b/src/rime/gear/memory.h @@ -17,6 +17,7 @@ class Context; class Engine; class Dictionary; class UserDictionary; +class Language; class Phrase; class Memory; @@ -31,9 +32,6 @@ struct CommitEntry : DictEntry { bool Save() const; }; -class Language { -}; - class Memory { public: Memory(const Ticket& ticket); @@ -45,11 +43,11 @@ class Memory { bool FinishSession(); bool DiscardSession(); - Language* language() { return &language_; } - Dictionary* dict() const { return dict_.get(); } UserDictionary* user_dict() const { return user_dict_.get(); } + const Language* language() const; + protected: void OnCommit(Context* ctx); void OnDeleteEntry(Context* ctx); @@ -62,7 +60,6 @@ class Memory { connection commit_connection_; connection delete_connection_; connection unhandled_key_connection_; - Language language_; }; } // namespace rime diff --git a/src/rime/gear/poet.h b/src/rime/gear/poet.h index 55f0c36b8..df27da549 100644 --- a/src/rime/gear/poet.h +++ b/src/rime/gear/poet.h @@ -21,12 +21,12 @@ class Language; class Poet { public: - Poet(Language* language) : language_(language) {} + Poet(const Language* language) : language_(language) {} + + an MakeSentence(const WordGraph& graph, size_t total_length); - an MakeSentence(const WordGraph& graph, - size_t total_length); protected: - Language* language_; + const Language* language_; }; } // namespace rime diff --git a/src/rime/gear/table_translator.cc b/src/rime/gear/table_translator.cc index 6f6d0c892..988a21577 100644 --- a/src/rime/gear/table_translator.cc +++ b/src/rime/gear/table_translator.cc @@ -28,21 +28,10 @@ static const char* kUnitySymbol = " \xe2\x98\xaf "; // TableTranslation TableTranslation::TableTranslation(TranslatorOptions* options, - Language* language, + const Language* language, const string& input, - size_t start, size_t end, - const string& preedit) - : options_(options), language_(language), - input_(input), start_(start), end_(end), preedit_(preedit) { - if (options_) - options_->preedit_formatter().Apply(&preedit_); - set_exhausted(true); -} - -TableTranslation::TableTranslation(TranslatorOptions* options, - Language* language, - const string& input, - size_t start, size_t end, + size_t start, + size_t end, const string& preedit, const DictEntryIterator& iter, const UserDictEntryIterator& uter) diff --git a/src/rime/gear/table_translator.h b/src/rime/gear/table_translator.h index 6b5767310..cd4b01dbe 100644 --- a/src/rime/gear/table_translator.h +++ b/src/rime/gear/table_translator.h @@ -50,13 +50,13 @@ class TableTranslator : public Translator, class TableTranslation : public Translation { public: - TableTranslation(TranslatorOptions* options, Language* language, - const string& input, size_t start, size_t end, - const string& preedit); - TableTranslation(TranslatorOptions* options, Language* language, - const string& input, size_t start, size_t end, + TableTranslation(TranslatorOptions* options, + const Language* language, + const string& input, + size_t start, + size_t end, const string& preedit, - const DictEntryIterator& iter, + const DictEntryIterator& iter = DictEntryIterator(), const UserDictEntryIterator& uter = UserDictEntryIterator()); virtual bool Next(); @@ -74,7 +74,7 @@ class TableTranslation : public Translation { } TranslatorOptions* options_; - Language* language_; + const Language* language_; string input_; size_t start_; size_t end_; diff --git a/src/rime/gear/translator_commons.h b/src/rime/gear/translator_commons.h index 566a522d2..3367bf41e 100644 --- a/src/rime/gear/translator_commons.h +++ b/src/rime/gear/translator_commons.h @@ -70,7 +70,7 @@ class Language; class Phrase : public Candidate { public: - Phrase(Language* language, + Phrase(const Language* language, const string& type, size_t start, size_t end, const an& entry) : Candidate(type, start, end), @@ -93,14 +93,14 @@ class Phrase : public Candidate { double weight() const { return entry_->weight; } Code& code() const { return entry_->code; } const DictEntry& entry() const { return *entry_; } - Language* language() const { return language_; } + const Language* language() const { return language_; } Spans spans() { return syllabifier_ ? syllabifier_->Syllabify(this) : Spans(); } protected: - Language* language_; + const Language* language_; an entry_; an syllabifier_; }; @@ -109,7 +109,7 @@ class Phrase : public Candidate { class Sentence : public Phrase { public: - Sentence(Language* language) + Sentence(const Language* language) : Phrase(language, "sentence", 0, 0, New()) { entry_->weight = 1.0; } diff --git a/src/rime/language.h b/src/rime/language.h new file mode 100644 index 000000000..07286a161 --- /dev/null +++ b/src/rime/language.h @@ -0,0 +1,31 @@ +// +// Copyright RIME Developers +// Distributed under the BSD License +// +#ifndef RIME_LANGUAGE_H_ +#define RIME_LANGUAGE_H_ + +#include + +namespace rime { + +class Language { + const string name_; + + public: + Language(const string& name) : name_(name) {} + string name() const { return name_; } + + bool operator== (const Language& other) const { + return name_ == other.name_; + } + + template + static bool intelligible(const T& t, const U& u) { + return t->language() && u->language() && *t->language() == *u->language(); + } +}; + +} // namespace rime + +#endif // RIME_LANGUAGE_H_ From 91775a624d414fa5476d34a9572e6ffdef7a8ae6 Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Wed, 11 Jul 2018 00:03:40 +0800 Subject: [PATCH 2/4] fix(language, memory, user_dictionary): a translator without user dictionary should also provide a language --- src/rime/dict/user_dictionary.cc | 13 ++++++------- src/rime/dict/user_dictionary.h | 7 +++---- src/rime/gear/memory.cc | 12 ++++++++---- src/rime/gear/memory.h | 3 ++- src/rime/language.cc | 18 ++++++++++++++++++ src/rime/language.h | 2 ++ 6 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 src/rime/language.cc diff --git a/src/rime/dict/user_dictionary.cc b/src/rime/dict/user_dictionary.cc index 347518c9f..2c7b9869a 100644 --- a/src/rime/dict/user_dictionary.cc +++ b/src/rime/dict/user_dictionary.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -116,8 +117,8 @@ bool UserDictEntryIterator::Next() { // UserDictionary members -UserDictionary::UserDictionary(Language language, an db) - : language_(language), db_(db) { +UserDictionary::UserDictionary(const string& name, an db) + : name_(name), db_(db) { } UserDictionary::~UserDictionary() { @@ -494,10 +495,8 @@ UserDictionary* UserDictionaryComponent::Create(const Ticket& ticket) { // user specified name } else if (config->GetString(ticket.name_space + "/dictionary", &dict_name)) { - // {dictionary: lunapinyin.extra} implies {user_dict: luna_pinyin} - size_t dot = dict_name.find('.'); - if (dot != string::npos && dot != 0) - dict_name.resize(dot); + // {dictionary: luna_pinyin.extra} implies {user_dict: luna_pinyin} + dict_name = Language::get_language_component(dict_name); } else { LOG(ERROR) << ticket.name_space << "/dictionary not specified in schema '" @@ -519,7 +518,7 @@ UserDictionary* UserDictionaryComponent::Create(const Ticket& ticket) { db.reset(component->Create(dict_name)); db_pool_[dict_name] = db; } - return new UserDictionary(Language{dict_name}, db); + return new UserDictionary(dict_name, db); } } // namespace rime diff --git a/src/rime/dict/user_dictionary.h b/src/rime/dict/user_dictionary.h index 3b0427495..4601a22f6 100644 --- a/src/rime/dict/user_dictionary.h +++ b/src/rime/dict/user_dictionary.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -51,7 +50,7 @@ struct Ticket; class UserDictionary : public Class { public: - UserDictionary(Language language, an db); + UserDictionary(const string& name, an db); virtual ~UserDictionary(); void Attach(const an
& table, const an& prism); @@ -77,7 +76,7 @@ class UserDictionary : public Class { bool RevertRecentTransaction(); bool CommitPendingTransaction(); - const Language* language() const { return &language_; } + const string& name() const { return name_; } TickCount tick() const { return tick_; } static an CreateDictEntry(const string& key, @@ -95,7 +94,7 @@ class UserDictionary : public Class { DfsState* state); private: - const Language language_; + string name_; an db_; an
table_; an prism_; diff --git a/src/rime/gear/memory.cc b/src/rime/gear/memory.cc index a0ae73407..bde9a4785 100644 --- a/src/rime/gear/memory.cc +++ b/src/rime/gear/memory.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,13 @@ Memory::Memory(const Ticket& ticket) { } } + // user dictionary is named after language; dictionary name may have an + // optional suffix separated from the language component by dot. + language_.reset(new Language{ + user_dict_ ? user_dict_->name() : + Language::get_language_component(dict_->name()) + }); + Context* ctx = ticket.engine->context(); commit_connection_ = ctx->commit_notifier().connect( [this](Context* ctx) { OnCommit(ctx); }); @@ -92,10 +100,6 @@ bool Memory::DiscardSession() { return user_dict_ && user_dict_->RevertRecentTransaction(); } -const Language* Memory::language() const { - return user_dict_ ? user_dict_->language() : nullptr; -} - void Memory::OnCommit(Context* ctx) { if (!user_dict_|| user_dict_->readonly()) return; diff --git a/src/rime/gear/memory.h b/src/rime/gear/memory.h index 01908d13a..0b065491f 100644 --- a/src/rime/gear/memory.h +++ b/src/rime/gear/memory.h @@ -46,7 +46,7 @@ class Memory { Dictionary* dict() const { return dict_.get(); } UserDictionary* user_dict() const { return user_dict_.get(); } - const Language* language() const; + const Language* language() const { return language_.get(); } protected: void OnCommit(Context* ctx); @@ -55,6 +55,7 @@ class Memory { the dict_; the user_dict_; + the language_; private: connection commit_connection_; diff --git a/src/rime/language.cc b/src/rime/language.cc new file mode 100644 index 000000000..5626499f8 --- /dev/null +++ b/src/rime/language.cc @@ -0,0 +1,18 @@ +// +// Copyright RIME Developers +// Distributed under the BSD License +// +#include +#include + +namespace rime { + +// "luna_pinyin.extra" has language component "luna_pinyin". +string Language::get_language_component(const string& name) { + size_t dot = name.find('.'); + if (dot != string::npos && dot != 0) + return name.substr(0, dot); + return name; +} + +} // namespace rime diff --git a/src/rime/language.h b/src/rime/language.h index 07286a161..7d1eee760 100644 --- a/src/rime/language.h +++ b/src/rime/language.h @@ -24,6 +24,8 @@ class Language { static bool intelligible(const T& t, const U& u) { return t->language() && u->language() && *t->language() == *u->language(); } + + static string get_language_component(const string& name); }; } // namespace rime From db519e692a0a641bb4606f597e27e6a1a03d1e76 Mon Sep 17 00:00:00 2001 From: nameoverflow Date: Thu, 2 Aug 2018 23:54:22 +0800 Subject: [PATCH 3/4] fix(memory): avoid null pointer access after dynamic casting --- src/rime/gear/memory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rime/gear/memory.cc b/src/rime/gear/memory.cc index bde9a4785..9c20bd268 100644 --- a/src/rime/gear/memory.cc +++ b/src/rime/gear/memory.cc @@ -108,7 +108,7 @@ void Memory::OnCommit(Context* ctx) { for (auto& seg : ctx->composition()) { auto phrase = As(Candidate::GetGenuineCandidate( seg.GetSelectedCandidate())); - bool recognized = Language::intelligible(phrase, this); + bool recognized = phrase != nullptr && Language::intelligible(phrase, this); if (recognized) { commit_entry.AppendPhrase(phrase); } From 4c2aa3c7a9b0becdfb03404bc50ea96a1aa4470c Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Fri, 3 Aug 2018 03:42:52 +0800 Subject: [PATCH 4/4] fix(language): add null check --- src/rime/gear/memory.cc | 2 +- src/rime/language.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rime/gear/memory.cc b/src/rime/gear/memory.cc index 9c20bd268..bde9a4785 100644 --- a/src/rime/gear/memory.cc +++ b/src/rime/gear/memory.cc @@ -108,7 +108,7 @@ void Memory::OnCommit(Context* ctx) { for (auto& seg : ctx->composition()) { auto phrase = As(Candidate::GetGenuineCandidate( seg.GetSelectedCandidate())); - bool recognized = phrase != nullptr && Language::intelligible(phrase, this); + bool recognized = Language::intelligible(phrase, this); if (recognized) { commit_entry.AppendPhrase(phrase); } diff --git a/src/rime/language.h b/src/rime/language.h index 7d1eee760..dbc8c0a2b 100644 --- a/src/rime/language.h +++ b/src/rime/language.h @@ -22,7 +22,8 @@ class Language { template static bool intelligible(const T& t, const U& u) { - return t->language() && u->language() && *t->language() == *u->language(); + return t && t->language() && u && u->language() && + *t->language() == *u->language(); } static string get_language_component(const string& name);