Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.12)
project(libime VERSION 1.1.11)
project(libime VERSION 1.1.12)
set(LibIME_VERSION ${PROJECT_VERSION})

set(REQUIRED_FCITX_VERSION 5.1.13)
Expand Down
12 changes: 12 additions & 0 deletions src/libime/core/lattice.h
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,18 @@ class LIBIMECORE_EXPORT LatticeNode : public WordNode {
LatticeNode *prev() const { return prev_; }
void setPrev(LatticeNode *prev) { prev_ = prev; }

template <typename T>
requires(std::is_base_of_v<LatticeNode, T>)
T &as() {
return static_cast<T &>(*this);
}

template <typename T>
requires(std::is_base_of_v<LatticeNode, T>)
const T &as() const {
return static_cast<const T &>(*this);
}

/// Return the full word till the begining of the sentence.
std::string fullWord() const {
size_t length = 0;
Expand Down
62 changes: 46 additions & 16 deletions src/libime/pinyin/pinyincontext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -180,16 +180,15 @@ class PinyinContextPrivate : public fcitx::QPtrHolder<PinyinContext> {
void select(const SentenceResult &sentence) {
FCITX_Q();
auto offset = q->selectedLength();
selectHelper([offset, &sentence,
this](std::vector<SelectedPinyin> &selection) {
for (const auto &p : sentence.sentence()) {
selection.emplace_back(
offset + p->to()->index(),
WordNode{p->word(), ime_->model()->index(p->word())},
static_cast<const PinyinLatticeNode *>(p)->encodedPinyin(),
false);
}
});
selectHelper(
[offset, &sentence, this](std::vector<SelectedPinyin> &selection) {
for (const auto &p : sentence.sentence()) {
selection.emplace_back(
offset + p->to()->index(),
WordNode{p->word(), ime_->model()->index(p->word())},
p->as<PinyinLatticeNode>().encodedPinyin(), false);
}
});
}

void selectCustom(size_t inputLength, std::string_view segment,
Expand Down Expand Up @@ -584,6 +583,8 @@ void PinyinContext::update() {
d->ime_->frameSize(), &d->matchState_);

d->clearCandidates();

// Add n-best result.
for (size_t i = 0, e = d->lattice_.sentenceSize(); i < e; i++) {
d->candidates_.push_back(d->lattice_.sentence(i));
d->candidatesSet_.insert(d->candidates_.back().toString());
Expand All @@ -597,8 +598,10 @@ void PinyinContext::update() {
float max = -std::numeric_limits<float>::max();
auto distancePenalty = d->ime_->model()->unknownPenalty() /
PINYIN_DISTANCE_PENALTY_FACTOR;
// Pull the phrase from lattice, this part is the word that's in the
// dict.

// Enumerate over all the lattice node, if from == bos, this is
// a dictionary word match.
// Add all words that does not contain pinyin correction.
for (const auto &graphNode : graph.nodes(i)) {
auto distance = graph.distanceToEnd(graphNode);
auto adjust = static_cast<float>(distance) * distancePenalty;
Expand All @@ -610,6 +613,7 @@ void PinyinContext::update() {
min = std::min(latticeNode.score(), min);
max = std::max(latticeNode.score(), max);
}
// Deduplcate.
if (d->candidatesSet_.contains(latticeNode.word())) {
continue;
}
Expand Down Expand Up @@ -667,7 +671,34 @@ void PinyinContext::update() {
}
}
std::sort(d->candidates_.begin() + beginSize, d->candidates_.end(),
std::greater<SentenceResult>());
std::greater<>());
if (const auto limit = d->ime_->wordCandidateLimit()) {
size_t count = 0;
auto &candidatesSet = d->candidatesSet_;
d->candidates_.erase(
std::remove_if(
d->candidates_.begin() + beginSize, d->candidates_.end(),
[&count, limit,
&candidatesSet](const SentenceResult &candidate) {
const bool isSinglePinyinWord =
candidate.sentence().size() == 1 &&
candidate.sentence()
.front()
->as<PinyinLatticeNode>()
.encodedPinyin()
.size() == 2;
if (!isSinglePinyinWord) {
if (count >= limit) {
candidatesSet.erase(candidate.toString());
return true;
}
count++;
}
return false;
}),
d->candidates_.end());
}

d->candidatesToCursorNeedUpdate_ = true;
}

Expand Down Expand Up @@ -767,8 +798,7 @@ PinyinContext::preeditWithCursor(PinyinPreeditMode mode) const {
std::string actualPinyin;
if (!syls.empty() && !syls.front().second.empty()) {
std::string_view candidatePinyin =
static_cast<const PinyinLatticeNode *>(node)
->encodedPinyin();
node->as<PinyinLatticeNode>().encodedPinyin();
auto nthPinyin = std::distance(node->path().begin(), iter);
PinyinInitial bestInitial = syls[0].first;
PinyinFinal bestFinal = syls[0].second[0].first;
Expand Down Expand Up @@ -918,7 +948,7 @@ PinyinContext::candidateFullPinyin(const SentenceResult &candidate) const {
pinyin.push_back('\'');
}
pinyin += PinyinEncoder::decodeFullPinyin(
static_cast<const PinyinLatticeNode *>(node)->encodedPinyin());
node->as<PinyinLatticeNode>().encodedPinyin());
}
}
return pinyin;
Expand Down
14 changes: 14 additions & 0 deletions src/libime/pinyin/pinyinime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class PinyinIMEPrivate : fcitx::QPtrHolder<PinyinIME> {
size_t beamSize_ = Decoder::beamSizeDefault;
size_t frameSize_ = Decoder::frameSizeDefault;
size_t partialLongWordLimit_ = 0;
size_t wordCandidateLimit_ = 15;
float maxDistance_ = std::numeric_limits<float>::max();
float minPath_ = -std::numeric_limits<float>::max();
PinyinPreeditMode preeditMode_ = PinyinPreeditMode::RawText;
Expand Down Expand Up @@ -139,6 +140,19 @@ void PinyinIME::setPartialLongWordLimit(size_t n) {
}
}

size_t PinyinIME::wordCandidateLimit() const {
FCITX_D();
return d->wordCandidateLimit_;
}

void PinyinIME::setWordCandidateLimit(size_t n) {
FCITX_D();
if (d->wordCandidateLimit_ != n) {
d->wordCandidateLimit_ = n;
emit<PinyinIME::optionChanged>();
}
}

void PinyinIME::setPreeditMode(PinyinPreeditMode mode) {
FCITX_D();
if (d->preeditMode_ != mode) {
Expand Down
16 changes: 16 additions & 0 deletions src/libime/pinyin/pinyinime.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,22 @@ class LIBIMEPINYIN_EXPORT PinyinIME : public fcitx::ConnectableObject {
void setFrameSize(size_t n);
size_t partialLongWordLimit() const;
void setPartialLongWordLimit(size_t n);
/**
* \brief The maximum number of candidates that is a word.
*
* Limit the non single character candidates to avoid need to scroll/next
* page too many characters.
*
* When is 0, it means no limit.
*
* Since 1.1.12
*/
size_t wordCandidateLimit() const;
/**
* \brief Set the maximum number of candidates that is a word.
* Since 1.1.12
*/
void setWordCandidateLimit(size_t n);
void setScoreFilter(float maxDistance = std::numeric_limits<float>::max(),
float minPath = -std::numeric_limits<float>::max());
void setShuangpinProfile(std::shared_ptr<const ShuangpinProfile> profile);
Expand Down
3 changes: 1 addition & 2 deletions test/testpinyincontext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,7 @@ int main() {
c.type("n");
for (const auto &candidate : c.candidates()) {
for (const auto *node : candidate.sentence()) {
const auto &pinyin =
static_cast<const PinyinLatticeNode *>(node)->encodedPinyin();
const auto &pinyin = node->as<PinyinLatticeNode>().encodedPinyin();
std::cout << node->word();
if (!pinyin.empty()) {
std::cout << " " << PinyinEncoder::decodeFullPinyin(pinyin);
Expand Down
3 changes: 1 addition & 2 deletions test/testpinyinime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,7 @@ int main(int argc, char *argv[]) {
std::cout << (count % 10) << ": ";
for (const auto *node : candidate.sentence()) {
const auto &pinyin =
static_cast<const PinyinLatticeNode *>(node)
->encodedPinyin();
node->as<PinyinLatticeNode>().encodedPinyin();
std::cout << node->word();
if (!pinyin.empty()) {
std::cout << " " << PinyinEncoder::decodeFullPinyin(pinyin);
Expand Down
Loading