Skip to content

Commit ee5cc54

Browse files
astefanastefan
authored
QL: "fields" api implementation in QL (#68802)
* Integrate "fields" API into QL (#68467) * QL: retry SQL and EQL requests in a mixed-node (rolling upgrade) cluster (#68602) * Adapt nested fields extraction from "fields" API output to the new un-flattened structure (#68745)
1 parent bae65dd commit ee5cc54

44 files changed

Lines changed: 1364 additions & 1092 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/reference/sql/endpoints/translate.asciidoc‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,22 @@ Which returns:
2222
--------------------------------------------------
2323
{
2424
"size": 10,
25-
"docvalue_fields": [
25+
"_source": false,
26+
"fields": [
27+
{
28+
"field": "author"
29+
},
30+
{
31+
"field": "name"
32+
},
33+
{
34+
"field": "page_count"
35+
},
2636
{
2737
"field": "release_date",
2838
"format": "epoch_millis"
2939
}
3040
],
31-
"_source": {
32-
"includes": [
33-
"author",
34-
"name",
35-
"page_count"
36-
],
37-
"excludes": []
38-
},
3941
"sort": [
4042
{
4143
"page_count": {

‎x-pack/plugin/eql/qa/mixed-node/build.gradle‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
apply plugin: 'elasticsearch.testclusters'
2+
apply plugin: 'elasticsearch.standalone-rest-test'
3+
apply from : "$rootDir/gradle/bwc-test.gradle"
4+
apply plugin: 'elasticsearch.rest-test'
5+
6+
import org.elasticsearch.gradle.Version
7+
import org.elasticsearch.gradle.VersionProperties
8+
import org.elasticsearch.gradle.info.BuildParams
9+
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
10+
11+
dependencies {
12+
testImplementation project(':x-pack:qa')
13+
testImplementation(project(xpackModule('ql:test')))
14+
testImplementation project(path: xpackModule('eql'), configuration: 'default')
15+
}
16+
17+
tasks.named("integTest").configure{ enabled = false}
18+
19+
for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible.findAll { it.onOrAfter('7.10.0') }) {
20+
if (bwcVersion == VersionProperties.getElasticsearchVersion()) {
21+
// Not really a mixed cluster
22+
continue;
23+
}
24+
25+
String baseName = "v${bwcVersion}"
26+
27+
testClusters {
28+
"${baseName}" {
29+
versions = [bwcVersion.toString(), project.version]
30+
numberOfNodes = 3
31+
testDistribution = 'DEFAULT'
32+
setting 'xpack.security.enabled', 'false'
33+
setting 'xpack.watcher.enabled', 'false'
34+
setting 'xpack.ml.enabled', 'false'
35+
setting 'xpack.eql.enabled', 'true'
36+
setting 'xpack.license.self_generated.type', 'trial'
37+
// for debugging purposes
38+
// setting 'logger.org.elasticsearch.xpack.eql.plugin.TransportEqlSearchAction', 'TRACE'
39+
}
40+
}
41+
42+
tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
43+
useCluster testClusters."${baseName}"
44+
mustRunAfter("precommit")
45+
doFirst {
46+
// Getting the endpoints causes a wait for the cluster
47+
println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
48+
println "Upgrading one node to create a mixed cluster"
49+
testClusters."${baseName}".nextNodeToNextVersion()
50+
51+
println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
52+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
53+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
54+
}
55+
onlyIf { project.bwc_tests_enabled }
56+
}
57+
58+
tasks.register(bwcTaskName(bwcVersion)) {
59+
dependsOn "${baseName}#mixedClusterTest"
60+
}
61+
62+
// run these bwc tests as part of the "check" task
63+
tasks.named("check").configure {
64+
dependsOn "${baseName}#mixedClusterTest"
65+
}
66+
}

‎x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.eql.qa.mixed_node;
9+
10+
import org.apache.http.HttpHost;
11+
import org.elasticsearch.client.Request;
12+
import org.elasticsearch.client.Response;
13+
import org.elasticsearch.client.RestClient;
14+
import org.elasticsearch.cluster.metadata.IndexMetadata;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.xcontent.XContentHelper;
17+
import org.elasticsearch.common.xcontent.json.JsonXContent;
18+
import org.elasticsearch.test.NotEqualMessageBuilder;
19+
import org.elasticsearch.test.rest.ESRestTestCase;
20+
import org.elasticsearch.xpack.ql.TestNode;
21+
import org.elasticsearch.xpack.ql.TestNodes;
22+
import org.junit.After;
23+
import org.junit.Before;
24+
25+
import java.io.IOException;
26+
import java.io.InputStream;
27+
import java.util.ArrayList;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
32+
import static java.util.Arrays.asList;
33+
import static java.util.Collections.emptyMap;
34+
import static java.util.Collections.singletonList;
35+
import static java.util.Collections.singletonMap;
36+
import static java.util.Collections.unmodifiableList;
37+
import static org.elasticsearch.xpack.ql.TestUtils.buildNodeAndVersions;
38+
import static org.elasticsearch.xpack.ql.TestUtils.readResource;
39+
40+
/**
41+
* Class testing the behavior of events and sequence queries in a mixed cluster scenario (during rolling upgrade).
42+
* The test is against a three-node cluster where one node is upgraded, the other two are on the old version.
43+
*
44+
*/
45+
public class EqlSearchIT extends ESRestTestCase {
46+
47+
private static final String index = "test_eql_mixed_versions";
48+
private static int numShards;
49+
private static int numReplicas = 1;
50+
private static int numDocs;
51+
private static TestNodes nodes;
52+
private static List<TestNode> newNodes;
53+
private static List<TestNode> bwcNodes;
54+
55+
@Before
56+
public void createIndex() throws IOException {
57+
nodes = buildNodeAndVersions(client());
58+
numShards = nodes.size();
59+
numDocs = randomIntBetween(numShards, 15);
60+
newNodes = new ArrayList<>(nodes.getNewNodes());
61+
bwcNodes = new ArrayList<>(nodes.getBWCNodes());
62+
63+
String mappings = readResource(EqlSearchIT.class.getResourceAsStream("/eql_mapping.json"));
64+
createIndex(
65+
index,
66+
Settings.builder()
67+
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards)
68+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas)
69+
.build(),
70+
mappings
71+
);
72+
}
73+
74+
@After
75+
public void cleanUpIndex() throws IOException {
76+
if (indexExists(index)) {
77+
deleteIndex(index);
78+
}
79+
}
80+
81+
public void testEventsWithRequestToOldNodes() throws Exception {
82+
assertEventsQueryOnNodes(bwcNodes);
83+
}
84+
85+
public void testEventsWithRequestToUpgradedNodes() throws Exception {
86+
assertEventsQueryOnNodes(newNodes);
87+
}
88+
89+
public void testSequencesWithRequestToOldNodes() throws Exception {
90+
assertSequncesQueryOnNodes(bwcNodes);
91+
}
92+
93+
public void testSequencesWithRequestToUpgradedNodes() throws Exception {
94+
assertSequncesQueryOnNodes(newNodes);
95+
}
96+
97+
private void assertEventsQueryOnNodes(List<TestNode> nodesList) throws Exception {
98+
final String event = randomEvent();
99+
Map<String, Object> expectedResponse = prepareEventsTestData(event);
100+
try (
101+
RestClient client = buildClient(restClientSettings(),
102+
nodesList.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new))
103+
) {
104+
// filter only the relevant bits of the response
105+
String filterPath = "filter_path=hits.events._source.@timestamp,hits.events._source.event_type,hits.events._source.sequence";
106+
107+
Request request = new Request("POST", index + "/_eql/search?" + filterPath);
108+
request.setJsonEntity("{\"query\":\"" + event + " where true\"}");
109+
assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); });
110+
}
111+
}
112+
113+
private void assertSequncesQueryOnNodes(List<TestNode> nodesList) throws Exception {
114+
Map<String, Object> expectedResponse = prepareSequencesTestData();
115+
try (
116+
RestClient client = buildClient(restClientSettings(),
117+
nodesList.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new))
118+
) {
119+
String filterPath = "filter_path=hits.sequences.join_keys,hits.sequences.events._id,hits.sequences.events._source";
120+
String query = "sequence by `sequence` with maxspan=100ms [success where true] by correlation_success1, correlation_success2 "
121+
+ "[failure where true] by correlation_failure1, correlation_failure2";
122+
String filter = "{\"range\":{\"@timestamp\":{\"gte\":\"1970-05-01\"}}}";
123+
124+
Request request = new Request("POST", index + "/_eql/search?" + filterPath);
125+
request.setJsonEntity("{\"query\":\"" + query + "\",\"filter\":" + filter + "}");
126+
assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); });
127+
}
128+
}
129+
130+
private String randomEvent() {
131+
return randomFrom("success", "failure");
132+
}
133+
134+
private Map<String, Object> prepareEventsTestData(String event) throws IOException {
135+
List<Map<String, Object>> sourceEvents = new ArrayList<Map<String, Object>>();
136+
Map<String, Object> expectedResponse = singletonMap("hits", singletonMap("events", sourceEvents));
137+
138+
for (int i = 0; i < numDocs; i++) {
139+
StringBuilder builder = new StringBuilder();
140+
final String randomEvent = randomEvent();
141+
builder.append("{");
142+
builder.append("\"@timestamp\":" + i + ",");
143+
builder.append("\"event_type\":\"" + randomEvent + "\",");
144+
builder.append("\"sequence\":" + i);
145+
builder.append("}");
146+
if (randomEvent.equals(event)) {
147+
Map<String, Object> eventSource = new HashMap<>();
148+
eventSource.put("@timestamp", i);
149+
eventSource.put("event_type", randomEvent);
150+
eventSource.put("sequence", i);
151+
sourceEvents.add(singletonMap("_source", eventSource));
152+
}
153+
154+
Request request = new Request("PUT", index + "/_doc/" + i);
155+
request.setJsonEntity(builder.toString());
156+
assertOK(client().performRequest(request));
157+
}
158+
if (sourceEvents.isEmpty()) {
159+
return emptyMap();
160+
}
161+
return expectedResponse;
162+
}
163+
164+
/*
165+
* Output to compare with looks like this:
166+
* {
167+
* "hits": {
168+
* "sequences": [
169+
* {
170+
* "join_keys": [
171+
* 44,
172+
* "C",
173+
* "D"
174+
* ],
175+
* "events": [
176+
* {
177+
* "_id": "14",
178+
* "_source": {
179+
* ...
180+
* }
181+
* }
182+
* ]
183+
* }
184+
* }
185+
* }
186+
*
187+
*/
188+
private Map<String, Object> prepareSequencesTestData() throws IOException {
189+
Map<String, Object> event14 = new HashMap<>();
190+
Map<String, Object> event14Source = new HashMap<>();
191+
event14.put("_id", "14");
192+
event14.put("_source", event14Source);
193+
event14Source.put("@timestamp", "12345678914");
194+
event14Source.put("event_type", "success");
195+
event14Source.put("sequence", 44);
196+
event14Source.put("correlation_success1", "C");
197+
event14Source.put("correlation_success2", "D");
198+
199+
Map<String, Object> event15 = new HashMap<>();
200+
Map<String, Object> event15Source = new HashMap<>();
201+
event15.put("_id", "15");
202+
event15.put("_source", event15Source);
203+
event15Source.put("@timestamp", "12345678999");
204+
event15Source.put("event_type", "failure");
205+
event15Source.put("sequence", 44);
206+
event15Source.put("correlation_failure1", "C");
207+
event15Source.put("correlation_failure2", "D");
208+
209+
Map<String, Object> sequence = new HashMap<>();
210+
List<Map<String, Object>> events = unmodifiableList(asList(event14, event15));
211+
List<Map<String, Object>> sequences = singletonList(sequence);
212+
Map<String, Object> expectedResponse = singletonMap("hits", singletonMap("sequences", sequences));
213+
214+
sequence.put("join_keys", asList(44, "C", "D"));
215+
sequence.put("events", events);
216+
217+
final String bulkEntries = readResource(EqlSearchIT.class.getResourceAsStream("/eql_data.json"));
218+
Request request = new Request("POST", index + "/_bulk?refresh");
219+
request.setJsonEntity(bulkEntries);
220+
assertOK(client().performRequest(request));
221+
222+
return expectedResponse;
223+
}
224+
225+
private void assertResponse(Map<String, Object> expected, Map<String, Object> actual) {
226+
if (false == expected.equals(actual)) {
227+
NotEqualMessageBuilder message = new NotEqualMessageBuilder();
228+
message.compareMaps(actual, expected);
229+
fail("Response does not match:\n" + message.toString());
230+
}
231+
}
232+
233+
private Map<String, Object> runEql(RestClient client, Request request) throws IOException {
234+
Response response = client.performRequest(request);
235+
try (InputStream content = response.getEntity().getContent()) {
236+
return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false);
237+
}
238+
}
239+
}

‎x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{"index":{"_id":1}}
2+
{"@timestamp":"1234567891","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"B"}
3+
{"index":{"_id":2}}
4+
{"@timestamp":"1234567892","event_type":"failure","sequence":2,"correlation_failure1":"A","correlation_failure2":"B"}
5+
{"index":{"_id":3}}
6+
{"@timestamp":"1234567893","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"}
7+
{"index":{"_id":4}}
8+
{"@timestamp":"1234567894","event_type":"success","sequence":4,"correlation_success1":"C","correlation_success2":"C"}
9+
{"index":{"_id":5}}
10+
{"@timestamp":"1234567895","event_type":"failure","sequence":5,"correlation_failure1":"B","correlation_failure2":"C"}
11+
{"index":{"_id":6}}
12+
{"@timestamp":"1234567896","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"A"}
13+
{"index":{"_id":7}}
14+
{"@timestamp":"1234567897","event_type":"failure","sequence":1,"correlation_failure1":"A","correlation_failure2":"A"}
15+
{"index":{"_id":8}}
16+
{"@timestamp":"1234567898","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"}
17+
{"index":{"_id":9}}
18+
{"@timestamp":"1234567899","event_type":"success","sequence":4,"correlation_success1":"C","correlation_success2":"B"}
19+
{"index":{"_id":10}}
20+
{"@timestamp":"12345678910","event_type":"failure","sequence":4,"correlation_failure1":"B","correlation_failure2":"B"}
21+
{"index":{"_id":11}}
22+
{"@timestamp":"12345678911","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"A"}
23+
{"index":{"_id":12}}
24+
{"@timestamp":"12345678912","event_type":"failure","sequence":1,"correlation_failure1":"A","correlation_failure2":"B"}
25+
{"index":{"_id":13}}
26+
{"@timestamp":"12345678913","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"}
27+
{"index":{"_id":14}}
28+
{"@timestamp":"12345678914","event_type":"success","sequence":44,"correlation_success1":"C","correlation_success2":"D"}
29+
{"index":{"_id":15}}
30+
{"@timestamp":"12345678999","event_type":"failure","sequence":44,"correlation_failure1":"C","correlation_failure2":"D"}

0 commit comments

Comments
 (0)