Skip to content
This repository has been archived by the owner on May 3, 2024. It is now read-only.

Commit

Permalink
Add ability to pass resource manager to prepare and execute
Browse files Browse the repository at this point in the history
  • Loading branch information
@MBkkt
MBkkt committed Aug 2, 2023
1 parent e051faa commit afce583
Show file tree
Hide file tree
Showing 28 changed files with 257 additions and 302 deletions.
12 changes: 6 additions & 6 deletions core/index/index_writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,13 @@ void RemoveFromExistingSegment(DocumentMask& deleted_docs,
return;
}

auto prepared = query.filter->prepare(reader);
auto prepared = query.filter->prepare({.index = reader});

if (IRS_UNLIKELY(!prepared)) {
return; // skip invalid prepared filters
}

auto itr = prepared->execute(reader);
auto itr = prepared->execute({.segment = reader});

if (IRS_UNLIKELY(!itr)) {
return; // skip invalid iterators
Expand All @@ -232,12 +232,12 @@ bool RemoveFromImportedSegment(DocumentMask& deleted_docs,
return false;
}

auto prepared = query.filter->prepare(reader);
auto prepared = query.filter->prepare({.index = reader});
if (IRS_UNLIKELY(!prepared)) {
return false; // skip invalid prepared filters
}

auto itr = prepared->execute(reader);
auto itr = prepared->execute({.segment = reader});
if (IRS_UNLIKELY(!itr)) {
return false; // skip invalid iterators
}
Expand Down Expand Up @@ -270,13 +270,13 @@ void FlushedSegmentContext::Remove(IndexWriter::QueryContext& query) {

auto& document_mask = flushed.document_mask;

auto prepared = query.filter->prepare(*reader);
auto prepared = query.filter->prepare({.index = *reader});

if (IRS_UNLIKELY(!prepared)) {
return; // Skip invalid prepared filters
}

auto itr = prepared->execute(*reader);
auto itr = prepared->execute({.segment = *reader});

if (IRS_UNLIKELY(!itr)) {
return; // Skip invalid iterators
Expand Down
10 changes: 4 additions & 6 deletions core/search/all_filter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,17 @@ class all_query : public filter::prepared {
bstring stats_;
};

filter::prepared::ptr all::prepare(const IndexReader& reader,
const Scorers& order, score_t filter_boost,
const attribute_provider* /*ctx*/) const {
filter::prepared::ptr all::prepare(const PrepareContext& ctx) const {
// skip field-level/term-level statistics because there are no explicit
// fields/terms, but still collect index-level statistics
// i.e. all fields and terms implicitly match
bstring stats(order.stats_size(), 0);
bstring stats(ctx.scorers.stats_size(), 0);
auto* stats_buf = stats.data();

PrepareCollectors(order.buckets(), stats_buf);
PrepareCollectors(ctx.scorers.buckets(), stats_buf);

return memory::make_managed<all_query>(std::move(stats),
this->boost() * filter_boost);
this->boost() * ctx.boost);
}

} // namespace irs
6 changes: 1 addition & 5 deletions core/search/all_filter.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,7 @@ namespace irs {
// Filter returning all documents
class all : public filter {
public:
using filter::prepare;

filter::prepared::ptr prepare(const IndexReader& reader, const Scorers& order,
score_t filter_boost,
const attribute_provider* ctx) const final;
filter::prepared::ptr prepare(const PrepareContext& ctx) const final;

irs::type_info::type_id type() const noexcept final {
return irs::type<all>::id();
Expand Down
125 changes: 76 additions & 49 deletions core/search/boolean_filter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,27 +180,28 @@ class BooleanQuery : public filter::prepared {
}
}

virtual void prepare(const IndexReader& rdr, const Scorers& ord,
score_t boost, ScoreMergeType merge_type,
const attribute_provider* ctx,
virtual void prepare(const PrepareContext& ctx, ScoreMergeType merge_type,
std::span<const filter* const> incl,
std::span<const filter* const> excl) {
BooleanQuery::queries_t queries;
queries.reserve(incl.size() + excl.size());

// apply boost to the current node
this->boost(boost);
this->boost(ctx.boost);

// prepare included
for (const auto* filter : incl) {
queries.emplace_back(filter->prepare(rdr, ord, boost, ctx));
queries.emplace_back(filter->prepare(ctx));
}

// prepare excluded
for (const auto* filter : excl) {
// exclusion part does not affect scoring at all
queries.emplace_back(
filter->prepare(rdr, Scorers::kUnordered, irs::kNoBoost, ctx));
queries.emplace_back(filter->prepare({
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.ctx = ctx.ctx,
}));
}

// nothrow block
Expand Down Expand Up @@ -329,22 +330,28 @@ bool boolean_filter::equals(const filter& rhs) const noexcept {
});
}

filter::prepared::ptr boolean_filter::prepare(
const IndexReader& rdr, const Scorers& ord, score_t boost,
const attribute_provider* ctx) const {
filter::prepared::ptr boolean_filter::prepare(const PrepareContext& ctx) const {
const auto size = filters_.size();

if (IRS_UNLIKELY(size == 0)) {
return prepared::empty();
}

const PrepareContext sub_ctx{
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.scorers = ctx.scorers,
.ctx = ctx.ctx,
.boost = ctx.boost * boost(),
};

if (size == 1) {
auto* filter = filters_.front().get();
IRS_ASSERT(filter);

// FIXME(gnusi): let Not handle everything?
if (filter->type() != irs::type<irs::Not>::id()) {
return filter->prepare(rdr, ord, boost * this->boost(), ctx);
return filter->prepare(sub_ctx);
}
}

Expand All @@ -363,7 +370,7 @@ filter::prepared::ptr boolean_filter::prepare(
incl.push_back(all_docs_no_boost.get());
}

return prepare(incl, excl, rdr, ord, boost, ctx);
return PrepareBoolean(incl, excl, sub_ctx);
}

void boolean_filter::group_filters(filter::ptr& all_docs_zero_boost,
Expand Down Expand Up @@ -414,21 +421,27 @@ void boolean_filter::group_filters(filter::ptr& all_docs_zero_boost,
}
}

filter::prepared::ptr And::prepare(std::vector<const filter*>& incl,
std::vector<const filter*>& excl,
const IndexReader& rdr, const Scorers& ord,
score_t boost,
const attribute_provider* ctx) const {
filter::prepared::ptr And::PrepareBoolean(std::vector<const filter*>& incl,
std::vector<const filter*>& excl,
const PrepareContext& ctx) const {
// optimization step
// if include group empty itself or has 'empty' -> this whole conjunction is
// empty
if (incl.empty() || incl.back()->type() == irs::type<irs::empty>::id()) {
return prepared::empty();
}

PrepareContext sub_ctx{
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.scorers = ctx.scorers,
.ctx = ctx.ctx,
.boost = ctx.boost * boost(),
};

// single node case
if (1 == incl.size() && excl.empty()) {
return incl.front()->prepare(rdr, ord, this->boost() * boost, ctx);
return incl.front()->prepare(sub_ctx);
}

auto cumulative_all = MakeAllDocsFilter(kNoBoost);
Expand Down Expand Up @@ -462,51 +475,60 @@ filter::prepared::ptr And::prepare(std::vector<const filter*>& incl,
// substitute new_boost back we will get ( boost * OR_BOOST * ALL_BOOST +
// boost * OR_BOOST * LEFT_BOOST) - original non-optimized boost value
auto left_boost = (*incl.begin())->boost();
if (this->boost() != 0 && left_boost != 0 && !ord.empty()) {
boost = (boost * this->boost() * all_boost +
boost * this->boost() * left_boost) /
(left_boost * this->boost());
if (boost() != 0 && left_boost != 0 && !ctx.scorers.empty()) {
sub_ctx.boost = (sub_ctx.boost * boost() * all_boost +
sub_ctx.boost * boost() * left_boost) /
(left_boost * boost());
} else {
boost = 0;
sub_ctx.boost = 0;
}
} else {
// create new 'all' with boost from all removed
cumulative_all->boost(all_boost);
incl.push_back(cumulative_all.get());
}
}
boost *= this->boost();
sub_ctx.boost *= this->boost();
if (1 == incl.size() && excl.empty()) {
// single node case
return incl.front()->prepare(rdr, ord, boost, ctx);
return incl.front()->prepare(sub_ctx);
}
auto q = memory::make_managed<AndQuery>();
q->prepare(rdr, ord, boost, merge_type(), ctx, incl, excl);
q->prepare(sub_ctx, merge_type(), incl, excl);
return q;
}

filter::prepared::ptr Or::prepare(const IndexReader& rdr, const Scorers& ord,
score_t boost,
const attribute_provider* ctx) const {
filter::prepared::ptr Or::prepare(const PrepareContext& ctx) const {
const PrepareContext sub_ctx{
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.scorers = ctx.scorers,
.ctx = ctx.ctx,
.boost = ctx.boost * boost(),
};

if (0 == min_match_count_) { // only explicit 0 min match counts!
// all conditions are satisfied
return MakeAllDocsFilter(kNoBoost)->prepare(rdr, ord, this->boost() * boost,
ctx);
return MakeAllDocsFilter(kNoBoost)->prepare(sub_ctx);
}

return boolean_filter::prepare(rdr, ord, boost, ctx);
return boolean_filter::prepare(sub_ctx);
}

filter::prepared::ptr Or::prepare(std::vector<const filter*>& incl,
std::vector<const filter*>& excl,
const IndexReader& rdr, const Scorers& ord,
score_t boost,
const attribute_provider* ctx) const {
boost *= this->boost();
filter::prepared::ptr Or::PrepareBoolean(std::vector<const filter*>& incl,
std::vector<const filter*>& excl,
const PrepareContext& ctx) const {
const PrepareContext sub_ctx{
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.scorers = ctx.scorers,
.ctx = ctx.ctx,
.boost = ctx.boost * boost(),
};

if (0 == min_match_count_) { // only explicit 0 min match counts!
// all conditions are satisfied
return MakeAllDocsFilter(kNoBoost)->prepare(rdr, ord, boost, ctx);
return MakeAllDocsFilter(kNoBoost)->prepare(sub_ctx);
}

if (!incl.empty() && incl.back()->type() == irs::type<irs::empty>::id()) {
Expand All @@ -519,7 +541,7 @@ filter::prepared::ptr Or::prepare(std::vector<const filter*>& incl,

// single node case
if (1 == incl.size() && excl.empty()) {
return incl.front()->prepare(rdr, ord, boost, ctx);
return incl.front()->prepare(sub_ctx);
}

auto cumulative_all = MakeAllDocsFilter(kNoBoost);
Expand All @@ -537,7 +559,8 @@ filter::prepared::ptr Or::prepare(std::vector<const filter*>& incl,
}
}
if (all_count != 0) {
if (ord.empty() && incl.size() > 1 && min_match_count_ <= all_count) {
if (ctx.scorers.empty() && incl.size() > 1 &&
min_match_count_ <= all_count) {
// if we have at least one all in include group - all other filters are
// not necessary in case there is no scoring and 'all' count satisfies
// min_match
Expand Down Expand Up @@ -574,7 +597,7 @@ filter::prepared::ptr Or::prepare(std::vector<const filter*>& incl,

if (1 == incl.size() && excl.empty()) {
// single node case
return incl.front()->prepare(rdr, ord, boost, ctx);
return incl.front()->prepare(sub_ctx);
}

IRS_ASSERT(adjusted_min_match_count > 0 &&
Expand All @@ -589,33 +612,37 @@ filter::prepared::ptr Or::prepare(std::vector<const filter*>& incl,
q = memory::make_managed<MinMatchQuery>(adjusted_min_match_count);
}

q->prepare(rdr, ord, boost, merge_type(), ctx, incl, excl);
q->prepare(sub_ctx, merge_type(), incl, excl);
return q;
}

filter::prepared::ptr Not::prepare(const IndexReader& rdr, const Scorers& ord,
score_t boost,
const attribute_provider* ctx) const {
filter::prepared::ptr Not::prepare(const PrepareContext& ctx) const {
const auto res = optimize_not(*this);

if (!res.first) {
return prepared::empty();
}

boost *= this->boost();
const PrepareContext sub_ctx{
.index = ctx.index,
.resource_manager = ctx.resource_manager,
.scorers = ctx.scorers,
.ctx = ctx.ctx,
.boost = ctx.boost * boost(),
};

if (res.second) {
auto all_docs = MakeAllDocsFilter(kNoBoost);
const std::array<const irs::filter*, 1> incl{all_docs.get()};
const std::array<const irs::filter*, 1> excl{res.first};

auto q = memory::make_managed<AndQuery>();
q->prepare(rdr, ord, boost, ScoreMergeType::kSum, ctx, incl, excl);
q->prepare(sub_ctx, ScoreMergeType::kSum, incl, excl);
return q;
}

// negation has been optimized out
return res.first->prepare(rdr, ord, boost, ctx);
return res.first->prepare(sub_ctx);
}

size_t Not::hash() const noexcept {
Expand Down
Loading

0 comments on commit afce583

Please sign in to comment.