Skip to content

Commit 272ca11

Browse files
idegtiarenkoidegtiarenko
authored
Expose _tier metadata attribute in ESQL (#139894)
1 parent aff2320 commit 272ca11

11 files changed

Lines changed: 198 additions & 47 deletions

File tree

‎docs/changelog/139894.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 139894
2+
summary: Expose `_tier` in ESQL
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/reference/query-languages/esql/esql-metadata-fields.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ The following metadata fields are available in {{esql}}:
2626
| [`_id`](/reference/elasticsearch/mapping-reference/mapping-id-field.md) | [keyword](/reference/elasticsearch/mapping-reference/keyword.md) | Unique document ID. |
2727
| [`_ignored`](/reference/elasticsearch/mapping-reference/mapping-ignored-field.md) | [keyword](/reference/elasticsearch/mapping-reference/keyword.md) | Names every field in a document that was ignored when the document was indexed. |
2828
| [`_index`](/reference/elasticsearch/mapping-reference/mapping-index-field.md) | [keyword](/reference/elasticsearch/mapping-reference/keyword.md) | Index name. |
29+
| [`_tier`](/reference/elasticsearch/mapping-reference/mapping-tier-field.md) | [keyword](/reference/elasticsearch/mapping-reference/keyword.md) | Index top tier preference. |
2930
| `_index_mode` | [keyword](/reference/elasticsearch/mapping-reference/keyword.md) | [Index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting). For example: `standard`, `lookup`, or `logsdb`. |
3031
| `_score` | [`float`](/reference/elasticsearch/mapping-reference/number.md) | Query relevance score (when enabled). Scores are updated when using [full text search functions](/reference/query-languages/esql/functions-operators/search-functions.md). |
3132
| [`_source`](/reference/elasticsearch/mapping-reference/mapping-source-field.md) | Special `_source` type | Original JSON document body passed at index time (or a reconstructed version if [synthetic `_source`](/reference/elasticsearch/mapping-reference/mapping-source-field.md#synthetic-source) is enabled). |
@@ -83,4 +84,4 @@ FROM products METADATA _score
8384

8485
:::{tip}
8586
Refer to [{{esql}} for search](docs-content://solutions/search/esql-for-search.md#esql-for-search-scoring) for more information on relevance scoring and how to use `_score` in your queries.
86-
:::
87+
:::

‎server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -623,16 +623,18 @@ public PointInTimeBuilder getPointInTimeBuilder() {
623623
*/
624624
@Nullable
625625
public String getTierPreference() {
626-
Settings settings = getIndexSettings().getSettings();
627-
String value = DataTier.TIER_PREFERENCE_SETTING.get(settings);
626+
return getFirstTierPreference(getIndexSettings().getSettings(), null);
627+
}
628628

629+
public static String getFirstTierPreference(Settings settings, String defaultTierPreference) {
630+
String value = DataTier.TIER_PREFERENCE_SETTING.get(settings);
629631
if (Strings.hasText(value) == false) {
630-
return null;
632+
return defaultTierPreference;
631633
}
632-
633634
// Tier preference can be a comma-delimited list of tiers, ordered by preference
634635
// It was decided we should only test the first of these potentially multiple preferences.
635-
return value.split(",")[0].trim();
636+
int separatorPosition = value.indexOf(',');
637+
return (separatorPosition != -1 ? value.substring(0, separatorPosition) : value).trim();
636638
}
637639

638640
public QueryRewriteInterceptor getQueryRewriteInterceptor() {

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@
88
package org.elasticsearch.xpack.cluster.routing.allocation.mapper;
99

1010
import org.apache.lucene.search.Query;
11+
import org.apache.lucene.util.BytesRef;
1112
import org.elasticsearch.common.Strings;
1213
import org.elasticsearch.common.lucene.search.Queries;
1314
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.index.mapper.BlockLoader;
1416
import org.elasticsearch.index.mapper.ConstantFieldType;
1517
import org.elasticsearch.index.mapper.KeywordFieldMapper;
1618
import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -73,6 +75,12 @@ public Query existsQuery(SearchExecutionContext context) {
7375
return Queries.ALL_DOCS_INSTANCE;
7476
}
7577

78+
@Override
79+
public BlockLoader blockLoader(BlockLoaderContext blContext) {
80+
final String tierPreference = SearchExecutionContext.getFirstTierPreference(blContext.indexSettings().getSettings(), "");
81+
return BlockLoader.constantBytes(new BytesRef(tierPreference));
82+
}
83+
7684
@Override
7785
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
7886
if (format != null) {

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata.csv-spec‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,3 +184,19 @@ FROM ul_logs, apps METADATA _index, _version
184184

185185
// end::multipleIndices-result[]
186186
;
187+
188+
indexMetadata
189+
required_capability: metadata_tier_field
190+
FROM employees METADATA _index | KEEP _index | LIMIT 1;
191+
192+
_index:keyword
193+
employees
194+
;
195+
196+
tierMetadata
197+
required_capability: metadata_tier_field
198+
FROM employees METADATA _tier | KEEP _tier | LIMIT 1;
199+
200+
_tier:keyword
201+
data_content
202+
;

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/CanMatchIT.java‎

Lines changed: 66 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,23 @@
1111
import org.elasticsearch.action.index.IndexRequest;
1212
import org.elasticsearch.action.support.WriteRequest;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.cluster.routing.allocation.DataTier;
1415
import org.elasticsearch.common.settings.Settings;
15-
import org.elasticsearch.common.util.CollectionUtils;
1616
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
17+
import org.elasticsearch.index.mapper.MetadataFieldMapper;
1718
import org.elasticsearch.index.query.MatchQueryBuilder;
1819
import org.elasticsearch.index.query.RangeQueryBuilder;
20+
import org.elasticsearch.plugins.MapperPlugin;
1921
import org.elasticsearch.plugins.Plugin;
2022
import org.elasticsearch.test.transport.MockTransportService;
2123
import org.elasticsearch.transport.RemoteClusterAware;
2224
import org.elasticsearch.transport.TransportService;
25+
import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper;
2326
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
2427
import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo;
2528
import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
2629

30+
import java.util.ArrayList;
2731
import java.util.Collection;
2832
import java.util.HashMap;
2933
import java.util.List;
@@ -43,7 +47,17 @@ public class CanMatchIT extends AbstractEsqlIntegTestCase {
4347

4448
@Override
4549
protected Collection<Class<? extends Plugin>> nodePlugins() {
46-
return CollectionUtils.appendToCopy(super.nodePlugins(), MockTransportService.TestPlugin.class);
50+
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
51+
plugins.add(MockTransportService.TestPlugin.class);
52+
plugins.add(DataTierFieldMapperPlugin.class);
53+
return plugins;
54+
}
55+
56+
public static class DataTierFieldMapperPlugin extends Plugin implements MapperPlugin {
57+
@Override
58+
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
59+
return Map.of(DataTierFieldMapper.NAME, DataTierFieldMapper.PARSER);
60+
}
4761
}
4862

4963
/**
@@ -71,19 +85,7 @@ public void testCanMatch() {
7185
.add(new IndexRequest().source("@timestamp", "2023-03-25", "uid", "u1"))
7286
.get();
7387
try {
74-
Set<String> queriedIndices = ConcurrentCollections.newConcurrentSet();
75-
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
76-
as(transportService, MockTransportService.class).addRequestHandlingBehavior(
77-
ComputeService.DATA_ACTION_NAME,
78-
(handler, request, channel, task) -> {
79-
DataNodeRequest dataNodeRequest = (DataNodeRequest) request;
80-
for (DataNodeRequest.Shard shard : dataNodeRequest.shards()) {
81-
queriedIndices.add(shard.shardId().getIndexName());
82-
}
83-
handler.messageReceived(request, channel, task);
84-
}
85-
);
86-
}
88+
var queriedIndices = captureQueriedIndices();
8789
try (
8890
EsqlQueryResponse resp = run(
8991
syncEsqlQueryRequest("from events_*").pragmas(randomPragmas())
@@ -160,9 +162,7 @@ public void testCanMatch() {
160162
queriedIndices.clear();
161163
}
162164
} finally {
163-
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
164-
as(transportService, MockTransportService.class).clearAllRules();
165-
}
165+
cleanAllTransportRules();
166166
}
167167
}
168168

@@ -372,6 +372,48 @@ public void testSkipOnIndexName() {
372372
bulk.get();
373373
indexToNumDocs.put(index, docs);
374374
}
375+
var queriedIndices = captureQueriedIndices();
376+
try {
377+
for (int i = 0; i < numIndices; i++) {
378+
queriedIndices.clear();
379+
String index = "events-" + i;
380+
try (EsqlQueryResponse resp = run("from events* METADATA _index | WHERE _index == \"" + index + "\" | KEEP timestamp")) {
381+
assertThat(getValuesList(resp), hasSize(indexToNumDocs.get(index)));
382+
}
383+
assertThat(queriedIndices, equalTo(Set.of(index)));
384+
}
385+
} finally {
386+
cleanAllTransportRules();
387+
}
388+
}
389+
390+
public void testSkipOnTierName() {
391+
var tiers = List.of("hot", "warm", "cold");
392+
for (String tier : tiers) {
393+
assertAcked(
394+
client().admin()
395+
.indices()
396+
.prepareCreate("index-" + tier)
397+
.setSettings(Settings.builder().put(DataTier.TIER_PREFERENCE, "data_" + tier))
398+
);
399+
indexRandom(true, "index-" + tier, 1);
400+
}
401+
402+
var queriedIndices = captureQueriedIndices();
403+
try {
404+
for (String tier : tiers) {
405+
queriedIndices.clear();
406+
try (EsqlQueryResponse resp = run("from index-* METADATA _tier | WHERE _tier == \"data_" + tier + "\"")) {
407+
assertThat(getValuesList(resp), hasSize(1));
408+
assertThat(queriedIndices, equalTo(Set.of("index-" + tier)));
409+
}
410+
}
411+
} finally {
412+
cleanAllTransportRules();
413+
}
414+
}
415+
416+
private static Set<String> captureQueriedIndices() {
375417
Set<String> queriedIndices = ConcurrentCollections.newConcurrentSet();
376418
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
377419
as(transportService, MockTransportService.class).addRequestHandlingBehavior(
@@ -385,19 +427,12 @@ public void testSkipOnIndexName() {
385427
}
386428
);
387429
}
388-
try {
389-
for (int i = 0; i < numIndices; i++) {
390-
queriedIndices.clear();
391-
String index = "events-" + i;
392-
try (EsqlQueryResponse resp = run("from events* METADATA _index | WHERE _index == \"" + index + "\" | KEEP timestamp")) {
393-
assertThat(getValuesList(resp), hasSize(indexToNumDocs.get(index)));
394-
}
395-
assertThat(queriedIndices, equalTo(Set.of(index)));
396-
}
397-
} finally {
398-
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
399-
as(transportService, MockTransportService.class).clearAllRules();
400-
}
430+
return queriedIndices;
431+
}
432+
433+
private static void cleanAllTransportRules() {
434+
for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
435+
as(transportService, MockTransportService.class).clearAllRules();
401436
}
402437
}
403438
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1836,6 +1836,10 @@ public enum Cap {
18361836
*/
18371837
ENABLE_REDUCE_NODE_LATE_MATERIALIZATION(Build.current().isSnapshot()),
18381838

1839+
/**
1840+
* Support for requesting the "_tier" metadata field.
1841+
*/
1842+
METADATA_TIER_FIELD,
18391843
/**
18401844
* Fix folding of coalesce function
18411845
* https://github.com/elastic/elasticsearch/issues/139887

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
1616
import org.elasticsearch.index.mapper.IndexModeFieldMapper;
1717
import org.elasticsearch.index.mapper.SourceFieldMapper;
18+
import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper;
1819
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
1920
import org.elasticsearch.xpack.esql.core.tree.Source;
2021
import org.elasticsearch.xpack.esql.core.type.DataType;
@@ -46,6 +47,7 @@ public class MetadataAttribute extends TypedAttribute {
4647
Map.entry(IgnoredFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
4748
Map.entry(SourceFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.SOURCE, false)),
4849
Map.entry(IndexModeFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
50+
Map.entry(DataTierFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
4951
Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)),
5052
Map.entry(TSID_FIELD, new MetadataAttributeConfiguration(DataType.TSID_DATA_TYPE, false))
5153
);

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2989,6 +2989,83 @@ public void testDontPushDownMetadataIndexInInequality() {
29892989
}
29902990
}
29912991

2992+
/*
2993+
* LimitExec[1000[INTEGER],...]
2994+
* \_ExchangeExec[[...],false]
2995+
* \_ProjectExec[[...]]
2996+
* \_FieldExtractExec[_meta_field{f}#8586, ...]<[],[]>
2997+
* \_EsQueryExec[test], query={"term":{"_tier":{"value":"data_hot","boost":0.0}}}
2998+
*/
2999+
public void testPushDownMetadataTierInEquality() {
3000+
var plan = physicalPlan("""
3001+
from test metadata _tier
3002+
| where _tier == "data_hot"
3003+
""");
3004+
3005+
var optimized = optimizedPlan(plan);
3006+
var limit = as(optimized, LimitExec.class);
3007+
var exchange = asRemoteExchange(limit.child());
3008+
var project = as(exchange.child(), ProjectExec.class);
3009+
var extract = as(project.child(), FieldExtractExec.class);
3010+
var source = source(extract.child());
3011+
3012+
var tq = as(source.query(), TermQueryBuilder.class);
3013+
assertThat(tq.fieldName(), is("_tier"));
3014+
assertThat(tq.value(), is("data_hot"));
3015+
}
3016+
3017+
/*
3018+
* LimitExec[1000[INTEGER],..]
3019+
* \_ExchangeExec[[...],false]
3020+
* \_ProjectExec[[...]]
3021+
* \_FieldExtractExec[_meta_field{f}#9140, ...]<[],[]>
3022+
* \_EsQueryExec[test], query={"bool":{"must_not":[{"term":{"_tier":{"value":"data_hot","boost":0.0}}}],"boost":1.0}}
3023+
*/
3024+
public void testPushDownMetadataTierInNotEquality() {
3025+
var plan = physicalPlan("""
3026+
from test metadata _tier
3027+
| where _tier != "data_hot"
3028+
""");
3029+
3030+
var optimized = optimizedPlan(plan);
3031+
var limit = as(optimized, LimitExec.class);
3032+
var exchange = asRemoteExchange(limit.child());
3033+
var project = as(exchange.child(), ProjectExec.class);
3034+
var extract = as(project.child(), FieldExtractExec.class);
3035+
var source = source(extract.child());
3036+
3037+
var bq = as(source.query(), BoolQueryBuilder.class);
3038+
assertThat(bq.mustNot().size(), is(1));
3039+
var tq = as(bq.mustNot().get(0), TermQueryBuilder.class);
3040+
assertThat(tq.fieldName(), is("_tier"));
3041+
assertThat(tq.value(), is("data_hot"));
3042+
}
3043+
3044+
/*
3045+
* LimitExec[1000[INTEGER],...]
3046+
* \_ExchangeExec[[...],false]
3047+
* \_ProjectExec[[_meta_field{f}#1816, ..., _tier{m}#1808]]
3048+
* \_FieldExtractExec[_meta_field{f}#1816, ...]<[],[]>
3049+
* \_EsQueryExec[test], query={"wildcard":{"_tier":{"wildcard":"data_*","boost":0.0}}}
3050+
*/
3051+
public void testPushDownMetadataTierInWildcard() {
3052+
var plan = physicalPlan("""
3053+
from test metadata _tier
3054+
| where _tier like "data_*"
3055+
""");
3056+
3057+
var optimized = optimizedPlan(plan);
3058+
var limit = as(optimized, LimitExec.class);
3059+
var exchange = asRemoteExchange(limit.child());
3060+
var project = as(exchange.child(), ProjectExec.class);
3061+
var extract = as(project.child(), FieldExtractExec.class);
3062+
var source = source(extract.child());
3063+
3064+
var tq = as(source.query(), WildcardQueryBuilder.class);
3065+
assertThat(tq.fieldName(), is("_tier"));
3066+
assertThat(tq.value(), is("data_*"));
3067+
}
3068+
29923069
public void testDontPushDownMetadataVersionAndId() {
29933070
for (var t : List.of(tuple("_version", "2"), tuple("_id", "\"2\""))) {
29943071
var plan = physicalPlan("from test metadata " + t.v1() + " | where " + t.v1() + " == " + t.v2());

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1249,10 +1249,6 @@ public void testMetadataFieldMultipleDeclarations() {
12491249
expectError("from test metadata _index, _version, _index", "1:38: metadata field [_index] already declared [@1:20]");
12501250
}
12511251

1252-
public void testMetadataFieldUnsupportedPrimitiveType() {
1253-
expectError("from test metadata _tier", "line 1:20: unsupported metadata field [_tier]");
1254-
}
1255-
12561252
public void testMetadataFieldUnsupportedCustomType() {
12571253
expectError("from test metadata _feature", "line 1:20: unsupported metadata field [_feature]");
12581254
}

0 commit comments

Comments
 (0)