Skip to content
This repository has been archived by the owner on May 3, 2024. It is now read-only.

Commit

Permalink
styling
Browse files Browse the repository at this point in the history
  • Loading branch information
@Dronplane
Dronplane committed Jan 10, 2024
1 parent 9db9f4d commit 4de640a
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 52 deletions.
68 changes: 34 additions & 34 deletions core/analysis/multi_delimited_token_stream.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,32 +43,32 @@ namespace {
template<typename Derived>
class MultiDelimitedTokenStreamBase : public MultiDelimitedAnalyser {
public:
MultiDelimitedTokenStreamBase() { std::get<increment>(attrs_).value = 1; }
MultiDelimitedTokenStreamBase() { std::get<increment>(attrs).value = 1; }

bool next() override {
while (true) {
if (data_.begin() == data_.end()) {
if (data.begin() == data.end()) {
return false;
}

auto [next, skip] = static_cast<Derived*>(this)->FindNextDelim();

if (next == data_.begin()) {
if (next == data.begin()) {
// skip empty terms
data_ = bytes_view{next + skip, data_.end()};
data = bytes_view{next + skip, data.end()};
continue;
}

auto& term = std::get<term_attribute>(attrs_);
term.value = bytes_view{data_.begin(), next};
auto& offset = std::get<irs::offset>(attrs_);
offset.start = std::distance(start_, data_.data());
auto& term = std::get<term_attribute>(attrs);
term.value = bytes_view{data.begin(), next};
auto& offset = std::get<irs::offset>(attrs);
offset.start = std::distance(start, data.data());
offset.end = offset.start + term.value.size();

if (next == data_.end()) {
data_ = {};
if (next == data.end()) {
data = {};
} else {
data_ = bytes_view{next + skip, data_.end()};
data = bytes_view{next + skip, data.end()};
}

return true;
Expand Down Expand Up @@ -100,7 +100,7 @@ class MultiDelimitedTokenStreamSingleChars final
IRS_ASSERT(false);
}

auto FindNextDelim() { return this->data_.end(); }
auto FindNextDelim() { return this->data.end(); }
};

template<>
Expand All @@ -116,10 +116,10 @@ class MultiDelimitedTokenStreamSingleChars<1> final
}

auto FindNextDelim() {
if (auto pos = this->data_.find(delim); pos != bstring::npos) {
return this->data_.begin() + pos;
if (auto pos = this->data.find(delim); pos != bstring::npos) {
return this->data.begin() + pos;
}
return this->data_.end();
return this->data.end();
}

byte_type delim;
Expand All @@ -135,7 +135,7 @@ class MultiDelimitedTokenStreamSingleChars<0> final
IRS_ASSERT(opts.delimiters.empty());
}

auto FindNextDelim() { return this->data_.end(); }
auto FindNextDelim() { return this->data.end(); }
};

class MultiDelimitedTokenStreamGenericSingleChars final
Expand All @@ -145,26 +145,25 @@ class MultiDelimitedTokenStreamGenericSingleChars final
explicit MultiDelimitedTokenStreamGenericSingleChars(const Options& opts) {
for (const auto& delim : opts.delimiters) {
IRS_ASSERT(delim.size() == 1);
bytes_[delim[0]] = true;
bytes[delim[0]] = true;
}
}

auto FindNextDelim() {
return std::find_if(data_.begin(), data_.end(), [&](auto c) {
return std::find_if(data.begin(), data.end(), [&](auto c) {
if (c > SCHAR_MAX) {
return false;
}
IRS_ASSERT(c <= SCHAR_MAX);
return bytes_[c];
return bytes[c];
});
}
// TODO maybe use a bitset instead?
std::array<bool, SCHAR_MAX + 1> bytes_{};
std::array<bool, SCHAR_MAX + 1> bytes{};
};

struct TrieNode {
explicit TrieNode(int32_t stateId, int32_t depth)
: state_id(stateId), depth(depth) {}
explicit TrieNode(int32_t id, int32_t depth) : state_id(id), depth(depth) {}
int32_t state_id;
int32_t depth;
bool is_leaf{false};
Expand Down Expand Up @@ -202,7 +201,7 @@ void InsertErrorTransitions(const std::vector<bstring>& strings,
return;
}

for (size_t k = 0; k < 256; k++) {
for (size_t k = 0; k <= std::numeric_limits<byte>::max(); k++) {
if (auto it = node->simple_trie.find(k); it != node->simple_trie.end()) {
node->real_trie.emplace(k, it->second);
matched_word.push_back(k);
Expand Down Expand Up @@ -322,23 +321,23 @@ class MultiDelimitedTokenStreamGeneric final
auto FindNextDelim() {
auto state = matcher.GetFst().Start();
matcher.SetState(state);
for (size_t k = 0; k < data_.length(); k++) {
matcher.Find(data_[k]);
for (size_t k = 0; k < data.length(); k++) {
matcher.Find(data[k]);

state = matcher.Value().nextstate;

if (matcher.Final(state)) {
auto length = matcher.Final(state).Payload();
IRS_ASSERT(length <= k);

return std::make_pair(data_.begin() + (k - length),
return std::make_pair(data.begin() + (k - length),
static_cast<size_t>(length + 1));
}

matcher.SetState(state);
}

return std::make_pair(data_.end(), size_t{0});
return std::make_pair(data.end(), size_t{0});
}

automaton autom;
Expand All @@ -353,9 +352,9 @@ class MultiDelimitedTokenStreamSingle final
: delim(std::move(opts.delimiters[0])) {}

auto FindNextDelim() {
auto next = data_.end();
if (auto pos = this->data_.find(delim); pos != bstring::npos) {
next = this->data_.begin() + pos;
auto next = data.end();
if (auto pos = this->data.find(delim); pos != bstring::npos) {
next = this->data.begin() + pos;
}
return std::make_pair(next, delim.size());
}
Expand All @@ -372,7 +371,7 @@ class MultiDelimitedTokenStreamSingle final
searcher(delim.begin(), delim.end()) {}

auto FindNextDelim() {
auto next = std::search(data_.begin(), data_.end(), searcher);
auto next = std::search(data.begin(), data.end(), searcher);
return std::make_pair(next, delim.size());
}

Expand Down Expand Up @@ -402,7 +401,8 @@ irs::analysis::analyzer::ptr Make(MultiDelimitedAnalyser::Options&& opts) {
[](const auto& delim) { return delim.size() == 1; });
if (single_character_case) {
return MakeSingleChar<0>(std::move(opts));
} else if (opts.delimiters.size() == 1) {
}
if (opts.delimiters.size() == 1) {
return std::make_unique<MultiDelimitedTokenStreamSingle>(opts);
}
return std::make_unique<MultiDelimitedTokenStreamGeneric>(std::move(opts));
Expand Down Expand Up @@ -477,7 +477,7 @@ bool MakeVpackConfig(const MultiDelimitedAnalyser::Options& options,
irs::analysis::analyzer::ptr MakeVpack(VPackSlice slice) {
MultiDelimitedAnalyser::Options options;
if (ParseVpackOptions(slice, options)) {
return irs::analysis::MultiDelimitedAnalyser::make(std::move(options));
return irs::analysis::MultiDelimitedAnalyser::Make(std::move(options));
}
return nullptr;
}
Expand Down Expand Up @@ -523,7 +523,7 @@ void MultiDelimitedAnalyser::init() {
// normalize_json_config); // match registration above
}

analyzer::ptr MultiDelimitedAnalyser::make(
analyzer::ptr MultiDelimitedAnalyser::Make(
MultiDelimitedAnalyser::Options&& opts) {
return ::Make(std::move(opts));
}
Expand Down
18 changes: 9 additions & 9 deletions core/analysis/multi_delimited_token_stream.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,23 +43,23 @@ class MultiDelimitedAnalyser : public TypedAnalyzer<MultiDelimitedAnalyser>,
}
static void init();

static analyzer::ptr make(Options&&);
static analyzer::ptr Make(Options&&);

attribute* get_mutable(irs::type_info::type_id type) noexcept final {
return irs::get_mutable(attrs_, type);
return irs::get_mutable(attrs, type);
}

bool reset(std::string_view data) final {
data_ = ViewCast<byte_type>(data);
start_ = data_.data();
bool reset(std::string_view input) final {
data = ViewCast<byte_type>(input);
start = data.data();
return true;
}

protected:
using attributes = std::tuple<increment, offset, term_attribute>;
const byte_type* start_;
bytes_view data_;
attributes attrs_;
using Attributes = std::tuple<increment, offset, term_attribute>;
const byte_type* start{};
bytes_view data{};
Attributes attrs;
};

} // namespace analysis
Expand Down
18 changes: 9 additions & 9 deletions tests/analysis/multi_delimited_token_stream_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ TEST_F(multi_delimited_token_stream_tests, consts) {

TEST_F(multi_delimited_token_stream_tests, test_delimiter) {
auto stream =
irs::analysis::MultiDelimitedAnalyser::make({.delimiters = {"a"_b}});
irs::analysis::MultiDelimitedAnalyser::Make({.delimiters = {"a"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());

Expand Down Expand Up @@ -84,7 +84,7 @@ TEST_F(multi_delimited_token_stream_tests, test_delimiter) {

TEST_F(multi_delimited_token_stream_tests, test_delimiter_empty_match) {
auto stream =
irs::analysis::MultiDelimitedAnalyser::make({.delimiters = {"."_b}});
irs::analysis::MultiDelimitedAnalyser::Make({.delimiters = {"."_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());

Expand All @@ -97,7 +97,7 @@ TEST_F(multi_delimited_token_stream_tests, test_delimiter_empty_match) {
}

TEST_F(multi_delimited_token_stream_tests, test_delimiter_3) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make(
auto stream = irs::analysis::MultiDelimitedAnalyser::Make(
{.delimiters = {";"_b, ","_b, "|"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());
Expand Down Expand Up @@ -133,7 +133,7 @@ TEST_F(multi_delimited_token_stream_tests, test_delimiter_3) {
}

TEST_F(multi_delimited_token_stream_tests, test_delimiter_5) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make(
auto stream = irs::analysis::MultiDelimitedAnalyser::Make(
{.delimiters = {";"_b, ","_b, "|"_b, "."_b, ":"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());
Expand Down Expand Up @@ -170,7 +170,7 @@ TEST_F(multi_delimited_token_stream_tests, test_delimiter_5) {

TEST_F(multi_delimited_token_stream_tests, test_delimiter_single_long) {
auto stream =
irs::analysis::MultiDelimitedAnalyser::make({.delimiters = {"foo"_b}});
irs::analysis::MultiDelimitedAnalyser::Make({.delimiters = {"foo"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());

Expand All @@ -197,7 +197,7 @@ TEST_F(multi_delimited_token_stream_tests, test_delimiter_single_long) {
}

TEST_F(multi_delimited_token_stream_tests, no_delimiter) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make({.delimiters = {}});
auto stream = irs::analysis::MultiDelimitedAnalyser::Make({.delimiters = {}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());

Expand All @@ -216,7 +216,7 @@ TEST_F(multi_delimited_token_stream_tests, no_delimiter) {
}

TEST_F(multi_delimited_token_stream_tests, multi_words) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make(
auto stream = irs::analysis::MultiDelimitedAnalyser::Make(
{.delimiters = {"foo"_b, "bar"_b, "baz"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());
Expand All @@ -240,7 +240,7 @@ TEST_F(multi_delimited_token_stream_tests, multi_words) {
}

TEST_F(multi_delimited_token_stream_tests, multi_words_2) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make(
auto stream = irs::analysis::MultiDelimitedAnalyser::Make(
{.delimiters = {"foo"_b, "bar"_b, "baz"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());
Expand All @@ -254,7 +254,7 @@ TEST_F(multi_delimited_token_stream_tests, multi_words_2) {
}

TEST_F(multi_delimited_token_stream_tests, trick_matching_1) {
auto stream = irs::analysis::MultiDelimitedAnalyser::make(
auto stream = irs::analysis::MultiDelimitedAnalyser::Make(
{.delimiters = {"foo"_b, "ffa"_b}});
ASSERT_EQ(irs::type<irs::analysis::MultiDelimitedAnalyser>::id(),
stream->type());
Expand Down

0 comments on commit 4de640a

Please sign in to comment.