mtfyun/ragflow
1
0 Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-11-13 14:51:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: harden SMTP, SQL, and tokenizer boundaries (#21558)

Browse Source
This commit is contained in:
buua436 authored and GitHub committed 2026-10-28 09:38:20 +08:00
1 parent 5966db46ab
commit 56582a9926
9 files changed
+397 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/binding/cpp/rag_analyzer.cpp
+8 -4
View File
@@ -1500,8 +1500,9 @@ void RAGAnalyzer::MergeWithPosition(const std::vector<std::string> &tokens,
void RAGAnalyzer::EnglishNormalize(const std::vector<std::string> &tokens, std::vector<std::string> &res) {
for (auto &t : tokens) {
if (re2::RE2::PartialMatch(t, pattern1_)) { //"[a-zA-Z_-]+$"
lowercase_string_buffer_.resize(t.size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(t.c_str(), t.size(), lowercase_term, term_string_buffer_limit_);
ToLower(t.c_str(), t.size(), lowercase_term, lowercase_string_buffer_.size());
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
@@ -1886,8 +1887,9 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
NLTKWordTokenizer::GetInstance().Tokenize(sentence, term_list);
}
for (unsigned i = 0; i < term_list.size(); ++i) {
lowercase_string_buffer_.resize(term_list[i].size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, term_string_buffer_limit_);
ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, lowercase_string_buffer_.size());
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
@@ -2034,8 +2036,9 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
unsigned start_pos = sentence_start_pos + static_cast<unsigned>(pos_in_sentence);
unsigned end_pos = start_pos + static_cast<unsigned>(term.size());
// Apply lowercase before lemmatization to match Python NLTK behavior
lowercase_string_buffer_.resize(term.size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term.c_str(), term.size(), lowercase_term, term_string_buffer_limit_);
ToLower(term.c_str(), term.size(), lowercase_term, lowercase_string_buffer_.size());
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
@@ -2367,8 +2370,9 @@ void RAGAnalyzer::EnglishNormalizeWithPosition(const std::vector<std::string> &t
const auto &[start_pos, end_pos] = positions[i];
if (re2::RE2::PartialMatch(token, pattern1_)) { //"[a-zA-Z_-]+$"
lowercase_string_buffer_.resize(token.size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(token.c_str(), token.size(), lowercase_term, term_string_buffer_limit_);
ToLower(token.c_str(), token.size(), lowercase_term, lowercase_string_buffer_.size());
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);