Skip to content

Commit

Permalink
Support doc value only range queries (#800)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aprudhomme
aprudhomme authored Jan 6, 2025
1 parent 9a1613a commit bd45834
Show file tree
Hide file tree
Showing 20 changed files with 511 additions and 96 deletions.
4 changes: 2 additions & 2 deletions src/main/java/com/yelp/nrtsearch/server/doc/LoadedDocValues.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ public SingleFloat(NumericDocValues docValues) {
@Override
public void setDocId(int docID) throws IOException {
if (docValues.advanceExact(docID)) {
value = Float.intBitsToFloat((int) docValues.longValue());
value = NumericUtils.sortableIntToFloat((int) docValues.longValue());
isSet = true;
} else {
isSet = false;
Expand Down Expand Up @@ -324,7 +324,7 @@ public SingleDouble(NumericDocValues docValues) {
@Override
public void setDocId(int docID) throws IOException {
if (docValues.advanceExact(docID)) {
value = Double.longBitsToDouble(docValues.longValue());
value = NumericUtils.sortableLongToDouble(docValues.longValue());
isSet = true;
} else {
isSet = false;
Expand Down
3 changes: 0 additions & 3 deletions src/main/java/com/yelp/nrtsearch/server/field/BindingValuesSources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ public class BindingValuesSources {
// decoders to convert the long value read from doc value fields into a double
public static LongToDoubleFunction INT_DECODER = value -> (double) value;
public static LongToDoubleFunction LONG_DECODER = value -> (double) value;
public static LongToDoubleFunction FLOAT_DECODER =
value -> (double) Float.intBitsToFloat((int) value);
public static LongToDoubleFunction DOUBLE_DECODER = Double::longBitsToDouble;
public static LongToDoubleFunction SORTED_FLOAT_DECODER =
value -> (double) NumericUtils.sortableIntToFloat((int) value);
public static LongToDoubleFunction SORTED_DOUBLE_DECODER = NumericUtils::sortableLongToDouble;
Expand Down
21 changes: 14 additions & 7 deletions src/main/java/com/yelp/nrtsearch/server/field/DateTimeFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public Object parseLastValue(String value) {

@Override
public Query getRangeQuery(RangeQuery rangeQuery) {
verifySearchableOrDocValues("Range query");
long lower =
rangeQuery.getLower().isEmpty()
? Long.MIN_VALUE
Expand All @@ -122,14 +123,20 @@ public Query getRangeQuery(RangeQuery rangeQuery) {
}
ensureUpperIsMoreThanLower(rangeQuery, lower, upper);

Query pointQuery = LongPoint.newRangeQuery(rangeQuery.getField(), lower, upper);

if (!hasDocValues()) {
return pointQuery;
Query pointQuery = null;
Query dvQuery = null;
if (isSearchable()) {
pointQuery = LongPoint.newRangeQuery(rangeQuery.getField(), lower, upper);
if (!hasDocValues()) {
return pointQuery;
}
}
if (hasDocValues()) {
dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
if (!isSearchable()) {
return dvQuery;
}
}

Query dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
}

Expand Down
38 changes: 21 additions & 17 deletions src/main/java/com/yelp/nrtsearch/server/field/DoubleFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import java.util.ArrayList;
import java.util.List;
import java.util.function.LongToDoubleFunction;
import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.index.DocValuesType;
Expand All @@ -45,7 +45,7 @@ public DoubleFieldDef(
@Override
protected org.apache.lucene.document.Field getDocValueField(Number fieldValue) {
if (docValuesType == DocValuesType.NUMERIC) {
return new DoubleDocValuesField(getName(), fieldValue.doubleValue());
return new NumericDocValuesField(getName(), SORTED_DOUBLE_ENCODER.applyAsLong(fieldValue));
} else if (docValuesType == DocValuesType.SORTED_NUMERIC) {
return new SortedNumericDocValuesField(
getName(), SORTED_DOUBLE_ENCODER.applyAsLong(fieldValue));
Expand All @@ -71,11 +71,7 @@ protected LoadedDocValues<Double> getSortedNumericDocValues(SortedNumericDocValu

@Override
protected LongToDoubleFunction getBindingDecoder() {
if (isMultiValue()) {
return BindingValuesSources.SORTED_DOUBLE_DECODER;
} else {
return BindingValuesSources.DOUBLE_DECODER;
}
return BindingValuesSources.SORTED_DOUBLE_DECODER;
}

@Override
Expand Down Expand Up @@ -107,6 +103,7 @@ public String getType() {

@Override
public Query getRangeQuery(RangeQuery rangeQuery) {
verifySearchableOrDocValues("Range query");
double lower =
rangeQuery.getLower().isEmpty()
? Double.NEGATIVE_INFINITY
Expand All @@ -123,17 +120,24 @@ public Query getRangeQuery(RangeQuery rangeQuery) {
}
ensureUpperIsMoreThanLower(rangeQuery, lower, upper);

Query pointQuery = DoublePoint.newRangeQuery(rangeQuery.getField(), lower, upper);

if (!hasDocValues()) {
return pointQuery;
Query pointQuery = null;
Query dvQuery = null;
if (isSearchable()) {
pointQuery = DoublePoint.newRangeQuery(rangeQuery.getField(), lower, upper);
if (!hasDocValues()) {
return pointQuery;
}
}
if (hasDocValues()) {
dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(
rangeQuery.getField(),
NumericUtils.doubleToSortableLong(lower),
NumericUtils.doubleToSortableLong(upper));
if (!isSearchable()) {
return dvQuery;
}
}

Query dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(
rangeQuery.getField(),
NumericUtils.doubleToSortableLong(lower),
NumericUtils.doubleToSortableLong(upper));
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
}

Expand Down
38 changes: 21 additions & 17 deletions src/main/java/com/yelp/nrtsearch/server/field/FloatFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import java.util.ArrayList;
import java.util.List;
import java.util.function.LongToDoubleFunction;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.index.DocValuesType;
Expand All @@ -45,7 +45,7 @@ public FloatFieldDef(
@Override
protected org.apache.lucene.document.Field getDocValueField(Number fieldValue) {
if (docValuesType == DocValuesType.NUMERIC) {
return new FloatDocValuesField(getName(), fieldValue.floatValue());
return new NumericDocValuesField(getName(), SORTED_FLOAT_ENCODER.applyAsLong(fieldValue));
} else if (docValuesType == DocValuesType.SORTED_NUMERIC) {
return new SortedNumericDocValuesField(
getName(), SORTED_FLOAT_ENCODER.applyAsLong(fieldValue));
Expand All @@ -71,11 +71,7 @@ protected LoadedDocValues<Float> getSortedNumericDocValues(SortedNumericDocValue

@Override
protected LongToDoubleFunction getBindingDecoder() {
if (isMultiValue()) {
return BindingValuesSources.SORTED_FLOAT_DECODER;
} else {
return BindingValuesSources.FLOAT_DECODER;
}
return BindingValuesSources.SORTED_FLOAT_DECODER;
}

@Override
Expand Down Expand Up @@ -105,6 +101,7 @@ public String getType() {

@Override
public Query getRangeQuery(RangeQuery rangeQuery) {
verifySearchableOrDocValues("Range query");
float lower =
rangeQuery.getLower().isEmpty()
? Float.NEGATIVE_INFINITY
Expand All @@ -122,17 +119,24 @@ public Query getRangeQuery(RangeQuery rangeQuery) {
}
ensureUpperIsMoreThanLower(rangeQuery, lower, upper);

Query pointQuery = FloatPoint.newRangeQuery(rangeQuery.getField(), lower, upper);

if (!hasDocValues()) {
return pointQuery;
Query pointQuery = null;
Query dvQuery = null;
if (isSearchable()) {
pointQuery = FloatPoint.newRangeQuery(rangeQuery.getField(), lower, upper);
if (!hasDocValues()) {
return pointQuery;
}
}
if (hasDocValues()) {
dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(
rangeQuery.getField(),
NumericUtils.floatToSortableInt(lower),
NumericUtils.floatToSortableInt(upper));
if (!isSearchable()) {
return dvQuery;
}
}

Query dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(
rangeQuery.getField(),
NumericUtils.floatToSortableInt(lower),
NumericUtils.floatToSortableInt(upper));
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
}

Expand Down
75 changes: 59 additions & 16 deletions src/main/java/com/yelp/nrtsearch/server/field/IndexableFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,19 +145,21 @@ private void checkChildName(String name) {
}

/**
* Method called by {@link #IndexableFieldDef(String, Field, Class)} to validate the provided
* {@link Field}. Field definitions should define a version that checks for incompatible
* parameters and any other potential issues. It is recommended to also call the super version of
* this method, so that general checks do not need to be repeated everywhere.
* Method called by {@link #IndexableFieldDef(String, Field,
* FieldDefCreator.FieldDefCreatorContext, Class)} to validate the provided {@link Field}. Field
* definitions should define a version that checks for incompatible parameters and any other
* potential issues. It is recommended to also call the super version of this method, so that
* general checks do not need to be repeated everywhere.
*
* @param requestField field properties to validate
*/
protected void validateRequest(Field requestField) {}

/**
* Method called by {@link #IndexableFieldDef(String, Field, Class)} to determine the doc value
* type used by this field. Fields are not necessarily limited to one doc value, but this should
* represent the primary value that will be accessible to scripts and search through {@link
* Method called by {@link #IndexableFieldDef(String, Field,
* FieldDefCreator.FieldDefCreatorContext, Class)} to determine the doc value type used by this
* field. Fields are not necessarily limited to one doc value, but this should represent the
* primary value that will be accessible to scripts and search through {@link
* #getDocValues(LeafReaderContext)}. A value of NONE implies that the field does not support doc
* values.
*
Expand All @@ -169,9 +171,10 @@ protected DocValuesType parseDocValuesType(Field requestField) {
}

/**
* Method called by {@link #IndexableFieldDef(String, Field, Class)} to determine the facet value
* type for this field. The result of this method is exposed externally through {@link
* #getFacetValueType()}. A value of NO_FACETS implies that the field does not support facets.
* Method called by {@link #IndexableFieldDef(String, Field,
* FieldDefCreator.FieldDefCreatorContext, Class)} to determine the facet value type for this
* field. The result of this method is exposed externally through {@link #getFacetValueType()}. A
* value of NO_FACETS implies that the field does not support facets.
*
* @param requestField field from request
* @return field facet value type
Expand All @@ -181,12 +184,13 @@ protected FacetValueType parseFacetValueType(Field requestField) {
}

/**
* Method called by {@link #IndexableFieldDef(String, Field, Class)} to set the search properties
* on the given {@link FieldType}. The {@link FieldType#setStored(boolean)} has already been set
* to the value from {@link Field#getStore()}. This method should set any other needed properties,
* such as index options, tokenization, term vectors, etc. It likely should not set a doc value
* type, as those are usually added separately. The common use of this {@link FieldType} is to add
* a {@link FieldWithData} during indexing. This method should not freeze the field type.
* Method called by {@link #IndexableFieldDef(String, Field,
* FieldDefCreator.FieldDefCreatorContext, Class)} to set the search properties on the given
* {@link FieldType}. The {@link FieldType#setStored(boolean)} has already been set to the value
* from {@link Field#getStore()}. This method should set any other needed properties, such as
* index options, tokenization, term vectors, etc. It likely should not set a doc value type, as
* those are usually added separately. The common use of this {@link FieldType} is to add a {@link
* FieldWithData} during indexing. This method should not freeze the field type.
*
* @param fieldType type that needs search properties set
* @param requestField field from request
Expand Down Expand Up @@ -371,4 +375,43 @@ public DocValuesFormat getDocValuesFormat() {
public FieldType getFieldType() {
return fieldType;
}

/**
* Verify that the field is searchable or has doc values.
*
* @param featureName name of feature that requires searchable or doc values
* @throws IllegalStateException if field is not searchable and does not have doc values
*/
protected void verifySearchableOrDocValues(String featureName) {
if (!isSearchable() && !hasDocValues()) {
throw new IllegalStateException(
featureName + " requires field to be searchable or have doc values: " + getName());
}
}

/**
* Verify that the field is searchable.
*
* @param featureName name of feature that requires searchable
* @throws IllegalStateException if field is not searchable
*/
protected void verifySearchable(String featureName) {
if (!isSearchable()) {
throw new IllegalStateException(
featureName + " requires field to be searchable: " + getName());
}
}

/**
* Verify that the field has doc values.
*
* @param featureName name of feature that requires doc values
* @throws IllegalStateException if field does not have doc values
*/
protected void verifyDocValues(String featureName) {
if (!hasDocValues()) {
throw new IllegalStateException(
featureName + " requires field to have doc values: " + getName());
}
}
}
21 changes: 14 additions & 7 deletions src/main/java/com/yelp/nrtsearch/server/field/IntFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public String getType() {

@Override
public Query getRangeQuery(RangeQuery rangeQuery) {
verifySearchableOrDocValues("Range query");
int lower =
rangeQuery.getLower().isEmpty()
? Integer.MIN_VALUE
Expand All @@ -116,14 +117,20 @@ public Query getRangeQuery(RangeQuery rangeQuery) {
}
ensureUpperIsMoreThanLower(rangeQuery, lower, upper);

Query pointQuery = IntPoint.newRangeQuery(rangeQuery.getField(), lower, upper);

if (!hasDocValues()) {
return pointQuery;
Query pointQuery = null;
Query dvQuery = null;
if (isSearchable()) {
pointQuery = IntPoint.newRangeQuery(rangeQuery.getField(), lower, upper);
if (!hasDocValues()) {
return pointQuery;
}
}
if (hasDocValues()) {
dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
if (!isSearchable()) {
return dvQuery;
}
}

Query dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
}

Expand Down
21 changes: 14 additions & 7 deletions src/main/java/com/yelp/nrtsearch/server/field/LongFieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public String getType() {

@Override
public Query getRangeQuery(RangeQuery rangeQuery) {
verifySearchableOrDocValues("Range query");
long lower =
rangeQuery.getLower().isEmpty() ? Long.MIN_VALUE : Long.parseLong(rangeQuery.getLower());
long upper =
Expand All @@ -112,14 +113,20 @@ public Query getRangeQuery(RangeQuery rangeQuery) {
}
ensureUpperIsMoreThanLower(rangeQuery, lower, upper);

Query pointQuery = LongPoint.newRangeQuery(rangeQuery.getField(), lower, upper);

if (!hasDocValues()) {
return pointQuery;
Query pointQuery = null;
Query dvQuery = null;
if (isSearchable()) {
pointQuery = LongPoint.newRangeQuery(rangeQuery.getField(), lower, upper);
if (!hasDocValues()) {
return pointQuery;
}
}
if (hasDocValues()) {
dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
if (!isSearchable()) {
return dvQuery;
}
}

Query dvQuery =
SortedNumericDocValuesField.newSlowRangeQuery(rangeQuery.getField(), lower, upper);
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
}

Expand Down
Loading

0 comments on commit bd45834

Please sign in to comment.