Skip to content

Commit 6377afa

Browse files
martijnvgmartijnvg
committed
Multi-level Nested Sort with Filters
Allow multple levels of nested sorting where each level can have it's own filter. Backward compatible with previous single-level nested sort.
1 parent 3d07bce commit 6377afa

13 files changed

Lines changed: 824 additions & 158 deletions

File tree

‎core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.lucene.util.BitSet;
3737
import org.apache.lucene.util.BytesRef;
3838
import org.elasticsearch.common.Nullable;
39+
import org.elasticsearch.common.lucene.search.Queries;
3940
import org.elasticsearch.index.IndexComponent;
4041
import org.elasticsearch.index.IndexSettings;
4142
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -116,6 +117,14 @@ public Nested(BitSetProducer rootFilter, Query innerQuery) {
116117
this.innerQuery = innerQuery;
117118
}
118119

120+
public Query getInnerQuery() {
121+
return innerQuery;
122+
}
123+
124+
public BitSetProducer getRootFilter() {
125+
return rootFilter;
126+
}
127+
119128
/**
120129
* Get a {@link BitDocIdSet} that matches the root documents.
121130
*/

‎core/src/main/java/org/elasticsearch/search/MultiValueMode.java‎

Lines changed: 54 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -514,42 +514,41 @@ protected long pick(SortedNumericDocValues values) throws IOException {
514514
* NOTE: Calling the returned instance on docs that are not root docs is illegal
515515
* The returned instance can only be evaluate the current and upcoming docs
516516
*/
517-
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
518-
if (rootDocs == null || innerDocs == null) {
517+
public NumericDocValues select(final SortedNumericDocValues values, final long missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
518+
if (parentDocs == null || childDocs == null) {
519519
return select(DocValues.emptySortedNumeric(maxDoc), missingValue);
520520
}
521521

522522
return new AbstractNumericDocValues() {
523523

524-
int lastSeenRootDoc = -1;
524+
int lastSeenParentDoc = -1;
525525
long lastEmittedValue = missingValue;
526526

527527
@Override
528-
public boolean advanceExact(int rootDoc) throws IOException {
529-
assert rootDocs.get(rootDoc) : "can only sort root documents";
530-
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
531-
if (rootDoc == lastSeenRootDoc) {
528+
public boolean advanceExact(int parentDoc) throws IOException {
529+
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
530+
if (parentDoc == lastSeenParentDoc) {
532531
return true;
533-
} else if (rootDoc == 0) {
532+
} else if (parentDoc == 0) {
534533
lastEmittedValue = missingValue;
535534
return true;
536535
}
537-
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
538-
final int firstNestedDoc;
539-
if (innerDocs.docID() > prevRootDoc) {
540-
firstNestedDoc = innerDocs.docID();
536+
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
537+
final int firstChildDoc;
538+
if (childDocs.docID() > prevParentDoc) {
539+
firstChildDoc = childDocs.docID();
541540
} else {
542-
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
541+
firstChildDoc = childDocs.advance(prevParentDoc + 1);
543542
}
544543

545-
lastSeenRootDoc = rootDoc;
546-
lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc);
544+
lastSeenParentDoc = parentDoc;
545+
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
547546
return true;
548547
}
549548

550549
@Override
551550
public int docID() {
552-
return lastSeenRootDoc;
551+
return lastSeenParentDoc;
553552
}
554553

555554
@Override
@@ -624,33 +623,32 @@ protected double pick(SortedNumericDoubleValues values) throws IOException {
624623
* NOTE: Calling the returned instance on docs that are not root docs is illegal
625624
* The returned instance can only be evaluate the current and upcoming docs
626625
*/
627-
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
628-
if (rootDocs == null || innerDocs == null) {
626+
public NumericDoubleValues select(final SortedNumericDoubleValues values, final double missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
627+
if (parentDocs == null || childDocs == null) {
629628
return select(FieldData.emptySortedNumericDoubles(), missingValue);
630629
}
631630

632631
return new NumericDoubleValues() {
633632

634-
int lastSeenRootDoc = 0;
633+
int lastSeenParentDoc = 0;
635634
double lastEmittedValue = missingValue;
636635

637636
@Override
638-
public boolean advanceExact(int rootDoc) throws IOException {
639-
assert rootDocs.get(rootDoc) : "can only sort root documents";
640-
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
641-
if (rootDoc == lastSeenRootDoc) {
637+
public boolean advanceExact(int parentDoc) throws IOException {
638+
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming parent docs";
639+
if (parentDoc == lastSeenParentDoc) {
642640
return true;
643641
}
644-
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
645-
final int firstNestedDoc;
646-
if (innerDocs.docID() > prevRootDoc) {
647-
firstNestedDoc = innerDocs.docID();
642+
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
643+
final int firstChildDoc;
644+
if (childDocs.docID() > prevParentDoc) {
645+
firstChildDoc = childDocs.docID();
648646
} else {
649-
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
647+
firstChildDoc = childDocs.advance(prevParentDoc + 1);
650648
}
651649

652-
lastSeenRootDoc = rootDoc;
653-
lastEmittedValue = pick(values, missingValue, innerDocs, firstNestedDoc, rootDoc);
650+
lastSeenParentDoc = parentDoc;
651+
lastEmittedValue = pick(values, missingValue, childDocs, firstChildDoc, parentDoc);
654652
return true;
655653
}
656654

@@ -733,8 +731,8 @@ protected BytesRef pick(SortedBinaryDocValues values) throws IOException {
733731
* NOTE: Calling the returned instance on docs that are not root docs is illegal
734732
* The returned instance can only be evaluate the current and upcoming docs
735733
*/
736-
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet rootDocs, final DocIdSetIterator innerDocs, int maxDoc) throws IOException {
737-
if (rootDocs == null || innerDocs == null) {
734+
public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef missingValue, final BitSet parentDocs, final DocIdSetIterator childDocs, int maxDoc) throws IOException {
735+
if (parentDocs == null || childDocs == null) {
738736
return select(FieldData.emptySortedBinary(), missingValue);
739737
}
740738
final BinaryDocValues selectedValues = select(values, null);
@@ -743,27 +741,26 @@ public BinaryDocValues select(final SortedBinaryDocValues values, final BytesRef
743741

744742
final BytesRefBuilder builder = new BytesRefBuilder();
745743

746-
int lastSeenRootDoc = 0;
744+
int lastSeenParentDoc = 0;
747745
BytesRef lastEmittedValue = missingValue;
748746

749747
@Override
750-
public boolean advanceExact(int rootDoc) throws IOException {
751-
assert rootDocs.get(rootDoc) : "can only sort root documents";
752-
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
753-
if (rootDoc == lastSeenRootDoc) {
748+
public boolean advanceExact(int parentDoc) throws IOException {
749+
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming root docs";
750+
if (parentDoc == lastSeenParentDoc) {
754751
return true;
755752
}
756753

757-
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
758-
final int firstNestedDoc;
759-
if (innerDocs.docID() > prevRootDoc) {
760-
firstNestedDoc = innerDocs.docID();
754+
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
755+
final int firstChildDoc;
756+
if (childDocs.docID() > prevParentDoc) {
757+
firstChildDoc = childDocs.docID();
761758
} else {
762-
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
759+
firstChildDoc = childDocs.advance(prevParentDoc + 1);
763760
}
764761

765-
lastSeenRootDoc = rootDoc;
766-
lastEmittedValue = pick(selectedValues, builder, innerDocs, firstNestedDoc, rootDoc);
762+
lastSeenParentDoc = parentDoc;
763+
lastEmittedValue = pick(selectedValues, builder, childDocs, firstChildDoc, parentDoc);
767764
if (lastEmittedValue == null) {
768765
lastEmittedValue = missingValue;
769766
}
@@ -850,16 +847,16 @@ protected int pick(SortedSetDocValues values) throws IOException {
850847
* NOTE: Calling the returned instance on docs that are not root docs is illegal
851848
* The returned instance can only be evaluate the current and upcoming docs
852849
*/
853-
public SortedDocValues select(final SortedSetDocValues values, final BitSet rootDocs, final DocIdSetIterator innerDocs) throws IOException {
854-
if (rootDocs == null || innerDocs == null) {
850+
public SortedDocValues select(final SortedSetDocValues values, final BitSet parentDocs, final DocIdSetIterator childDocs) throws IOException {
851+
if (parentDocs == null || childDocs == null) {
855852
return select(DocValues.emptySortedSet());
856853
}
857854
final SortedDocValues selectedValues = select(values);
858855

859856
return new AbstractSortedDocValues() {
860857

861858
int docID = -1;
862-
int lastSeenRootDoc = 0;
859+
int lastSeenParentDoc = 0;
863860
int lastEmittedOrd = -1;
864861

865862
@Override
@@ -873,23 +870,22 @@ public int getValueCount() {
873870
}
874871

875872
@Override
876-
public boolean advanceExact(int rootDoc) throws IOException {
877-
assert rootDocs.get(rootDoc) : "can only sort root documents";
878-
assert rootDoc >= lastSeenRootDoc : "can only evaluate current and upcoming root docs";
879-
if (rootDoc == lastSeenRootDoc) {
873+
public boolean advanceExact(int parentDoc) throws IOException {
874+
assert parentDoc >= lastSeenParentDoc : "can only evaluate current and upcoming root docs";
875+
if (parentDoc == lastSeenParentDoc) {
880876
return lastEmittedOrd != -1;
881877
}
882878

883-
final int prevRootDoc = rootDocs.prevSetBit(rootDoc - 1);
884-
final int firstNestedDoc;
885-
if (innerDocs.docID() > prevRootDoc) {
886-
firstNestedDoc = innerDocs.docID();
879+
final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1);
880+
final int firstChildDoc;
881+
if (childDocs.docID() > prevParentDoc) {
882+
firstChildDoc = childDocs.docID();
887883
} else {
888-
firstNestedDoc = innerDocs.advance(prevRootDoc + 1);
884+
firstChildDoc = childDocs.advance(prevParentDoc + 1);
889885
}
890886

891-
docID = lastSeenRootDoc = rootDoc;
892-
lastEmittedOrd = pick(selectedValues, innerDocs, firstNestedDoc, rootDoc);
887+
docID = lastSeenParentDoc = parentDoc;
888+
lastEmittedOrd = pick(selectedValues, childDocs, firstChildDoc, parentDoc);
893889
return lastEmittedOrd != -1;
894890
}
895891

‎core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,15 @@
1919

2020
package org.elasticsearch.search.sort;
2121

22+
import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;
23+
2224
import org.apache.lucene.search.SortField;
25+
import org.elasticsearch.Version;
2326
import org.elasticsearch.common.ParseField;
2427
import org.elasticsearch.common.io.stream.StreamInput;
2528
import org.elasticsearch.common.io.stream.StreamOutput;
29+
import org.elasticsearch.common.logging.DeprecationLogger;
30+
import org.elasticsearch.common.logging.Loggers;
2631
import org.elasticsearch.common.xcontent.ObjectParser;
2732
import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
2833
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -45,6 +50,8 @@
4550
* A sort builder to sort based on a document field.
4651
*/
4752
public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
53+
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(FieldSortBuilder.class));
54+
4855
public static final String NAME = "field_sort";
4956
public static final ParseField MISSING = new ParseField("missing");
5057
public static final ParseField SORT_MODE = new ParseField("mode");
@@ -71,6 +78,8 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
7178

7279
private String nestedPath;
7380

81+
private NestedSortBuilder nestedSort;
82+
7483
/** Copy constructor. */
7584
public FieldSortBuilder(FieldSortBuilder template) {
7685
this(template.fieldName);
@@ -82,6 +91,7 @@ public FieldSortBuilder(FieldSortBuilder template) {
8291
}
8392
this.setNestedFilter(template.getNestedFilter());
8493
this.setNestedPath(template.getNestedPath());
94+
this.setNestedSort(template.getNestedSort());
8595
}
8696

8797
/**
@@ -108,6 +118,9 @@ public FieldSortBuilder(StreamInput in) throws IOException {
108118
order = in.readOptionalWriteable(SortOrder::readFromStream);
109119
sortMode = in.readOptionalWriteable(SortMode::readFromStream);
110120
unmappedType = in.readOptionalString();
121+
if (in.getVersion().onOrAfter(Version.V_6_1_0)) {
122+
nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
123+
}
111124
}
112125

113126
@Override
@@ -119,6 +132,9 @@ public void writeTo(StreamOutput out) throws IOException {
119132
out.writeOptionalWriteable(order);
120133
out.writeOptionalWriteable(sortMode);
121134
out.writeOptionalString(unmappedType);
135+
if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
136+
out.writeOptionalWriteable(nestedSort);
137+
}
122138
}
123139

124140
/** Returns the document field this sort should be based on. */
@@ -221,6 +237,15 @@ public String getNestedPath() {
221237
return this.nestedPath;
222238
}
223239

240+
public NestedSortBuilder getNestedSort() {
241+
return this.nestedSort;
242+
}
243+
244+
public FieldSortBuilder setNestedSort(final NestedSortBuilder nestedSort) {
245+
this.nestedSort = nestedSort;
246+
return this;
247+
}
248+
224249
@Override
225250
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
226251
builder.startObject();
@@ -241,6 +266,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
241266
if (nestedPath != null) {
242267
builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
243268
}
269+
if (nestedSort != null) {
270+
builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
271+
}
244272
builder.endObject();
245273
builder.endObject();
246274
return builder;
@@ -274,7 +302,14 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
274302
localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
275303
}
276304

277-
final Nested nested = resolveNested(context, nestedPath, nestedFilter);
305+
final Nested nested;
306+
if (nestedSort != null) {
307+
// new nested sorts takes priority
308+
nested = resolveNested(context, nestedSort);
309+
} else {
310+
nested = resolveNested(context, nestedPath, nestedFilter);
311+
}
312+
278313
IndexFieldData<?> fieldData = context.getForField(fieldType);
279314
if (fieldData instanceof IndexNumericFieldData == false
280315
&& (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
@@ -299,12 +334,13 @@ public boolean equals(Object other) {
299334
return (Objects.equals(this.fieldName, builder.fieldName) && Objects.equals(this.nestedFilter, builder.nestedFilter)
300335
&& Objects.equals(this.nestedPath, builder.nestedPath) && Objects.equals(this.missing, builder.missing)
301336
&& Objects.equals(this.order, builder.order) && Objects.equals(this.sortMode, builder.sortMode)
302-
&& Objects.equals(this.unmappedType, builder.unmappedType));
337+
&& Objects.equals(this.unmappedType, builder.unmappedType) && Objects.equals(this.nestedSort, builder.nestedSort));
303338
}
304339

305340
@Override
306341
public int hashCode() {
307-
return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.missing, this.order, this.sortMode, this.unmappedType);
342+
return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.nestedSort, this.missing, this.order, this.sortMode,
343+
this.unmappedType);
308344
}
309345

310346
@Override
@@ -329,11 +365,18 @@ public static FieldSortBuilder fromXContent(XContentParser parser, String fieldN
329365

330366
static {
331367
PARSER.declareField(FieldSortBuilder::missing, p -> p.objectText(), MISSING, ValueType.VALUE);
332-
PARSER.declareString(FieldSortBuilder::setNestedPath , NESTED_PATH_FIELD);
368+
PARSER.declareString((fieldSortBuilder, nestedPath) -> {
369+
DEPRECATION_LOGGER.deprecated("[nested_path] has been deprecated in favor of the [nested] parameter");
370+
fieldSortBuilder.setNestedPath(nestedPath);
371+
}, NESTED_PATH_FIELD);
333372
PARSER.declareString(FieldSortBuilder::unmappedType , UNMAPPED_TYPE);
334373
PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)) , ORDER_FIELD);
335374
PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORT_MODE);
336-
PARSER.declareObject(FieldSortBuilder::setNestedFilter, (p, c) -> SortBuilder.parseNestedFilter(p), NESTED_FILTER_FIELD);
375+
PARSER.declareObject(FieldSortBuilder::setNestedFilter, (p, c) -> {
376+
DEPRECATION_LOGGER.deprecated("[nested_filter] has been deprecated in favour for the [nested] parameter");
377+
return SortBuilder.parseNestedFilter(p);
378+
}, NESTED_FILTER_FIELD);
379+
PARSER.declareObject(FieldSortBuilder::setNestedSort, (p, c) -> NestedSortBuilder.fromXContent(p), NESTED_FIELD);
337380
}
338381

339382
@Override

0 commit comments

Comments
 (0)