Skip to content

Commit f63cbdb

Browse files
AthenaErymaAthenaEryma
authored
Dep. check for ECS changes to User Agent processor (elastic#38362)
The User Agent ingest processor is changing to align with ECS. Users need to be made aware that any pipelines using this processor will behave differently in 7.0.
1 parent 0970035 commit f63cbdb

4 files changed

Lines changed: 99 additions & 1 deletion

File tree

‎server/src/main/java/org/elasticsearch/ingest/IngestService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ public String getType() {
335335
return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor));
336336
}
337337

338-
static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
338+
public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
339339
IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE);
340340
Map<String, PipelineConfiguration> pipelines;
341341
if (currentIngestMetadata != null) {

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,16 @@
88

99
import org.elasticsearch.cluster.ClusterState;
1010
import org.elasticsearch.cluster.metadata.MetaData;
11+
import org.elasticsearch.ingest.ConfigurationUtils;
12+
import org.elasticsearch.ingest.IngestService;
13+
import org.elasticsearch.ingest.PipelineConfiguration;
1114
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
1215

16+
import java.util.List;
17+
import java.util.Map;
18+
import java.util.Objects;
19+
import java.util.stream.Collectors;
20+
1321
public class ClusterDeprecationChecks {
1422

1523
static DeprecationIssue checkShardLimit(ClusterState state) {
@@ -40,4 +48,36 @@ static DeprecationIssue checkClusterName(ClusterState state) {
4048
}
4149
return null;
4250
}
51+
52+
@SuppressWarnings("unchecked")
53+
static DeprecationIssue checkUserAgentPipelines(ClusterState state) {
54+
List<PipelineConfiguration> pipelines = IngestService.getPipelines(state);
55+
56+
List<String> pipelinesWithDeprecatedEcsConfig = pipelines.stream()
57+
.filter(Objects::nonNull)
58+
.filter(pipeline -> {
59+
Map<String, Object> pipelineConfig = pipeline.getConfigAsMap();
60+
61+
List<Map<String, Map<String, Object>>> processors =
62+
(List<Map<String, Map<String, Object>>>) pipelineConfig.get("processors");
63+
return processors.stream()
64+
.filter(Objects::nonNull)
65+
.filter(processor -> processor.containsKey("user_agent"))
66+
.map(processor -> processor.get("user_agent"))
67+
.anyMatch(processorConfig ->
68+
false == ConfigurationUtils.readBooleanProperty(null, null, processorConfig, "ecs", false));
69+
})
70+
.map(PipelineConfiguration::getId)
71+
.sorted() // Make the warning consistent for testing purposes
72+
.collect(Collectors.toList());
73+
if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) {
74+
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
75+
"User-Agent ingest plugin will use ECS-formatted output",
76+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
77+
"#ingest-user-agent-ecs-always",
78+
"Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " will change to using ECS output format in 7.0");
79+
}
80+
return null;
81+
82+
}
4383
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ private DeprecationChecks() {
3333

3434
static List<Function<ClusterState, DeprecationIssue>> CLUSTER_SETTINGS_CHECKS =
3535
Collections.unmodifiableList(Arrays.asList(
36+
ClusterDeprecationChecks::checkUserAgentPipelines,
3637
ClusterDeprecationChecks::checkShardLimit,
3738
ClusterDeprecationChecks::checkClusterName
3839
));

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,18 @@
66
package org.elasticsearch.xpack.deprecation;
77

88
import org.elasticsearch.Version;
9+
import org.elasticsearch.action.ingest.PutPipelineRequest;
910
import org.elasticsearch.cluster.ClusterName;
1011
import org.elasticsearch.cluster.ClusterState;
1112
import org.elasticsearch.cluster.metadata.IndexMetaData;
1213
import org.elasticsearch.cluster.metadata.MetaData;
1314
import org.elasticsearch.cluster.node.DiscoveryNode;
1415
import org.elasticsearch.cluster.node.DiscoveryNodes;
16+
import org.elasticsearch.common.bytes.BytesArray;
1517
import org.elasticsearch.common.settings.Settings;
1618
import org.elasticsearch.common.transport.TransportAddress;
19+
import org.elasticsearch.common.xcontent.XContentType;
20+
import org.elasticsearch.ingest.IngestService;
1721
import org.elasticsearch.test.ESTestCase;
1822
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
1923

@@ -83,4 +87,57 @@ public void testCheckShardLimit() {
8387
issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(goodState));
8488
assertTrue(issues.isEmpty());
8589
}
90+
91+
public void testUserAgentEcsCheck() {
92+
PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false",
93+
new BytesArray("{\n" +
94+
" \"description\" : \"This has ecs set to false\",\n" +
95+
" \"processors\" : [\n" +
96+
" {\n" +
97+
" \"user_agent\" : {\n" +
98+
" \"field\" : \"agent\",\n" +
99+
" \"ecs\" : false\n" +
100+
" }\n" +
101+
" }\n" +
102+
" ]\n" +
103+
"}"), XContentType.JSON);
104+
PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null",
105+
new BytesArray("{\n" +
106+
" \"description\" : \"This has ecs set to false\",\n" +
107+
" \"processors\" : [\n" +
108+
" {\n" +
109+
" \"user_agent\" : {\n" +
110+
" \"field\" : \"agent\"\n" +
111+
" }\n" +
112+
" }\n" +
113+
" ]\n" +
114+
"}"), XContentType.JSON);
115+
PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true",
116+
new BytesArray("{\n" +
117+
" \"description\" : \"This has ecs set to false\",\n" +
118+
" \"processors\" : [\n" +
119+
" {\n" +
120+
" \"user_agent\" : {\n" +
121+
" \"field\" : \"agent\",\n" +
122+
" \"ecs\" : true\n" +
123+
" }\n" +
124+
" }\n" +
125+
" ]\n" +
126+
"}"), XContentType.JSON);
127+
128+
ClusterState state = ClusterState.builder(new ClusterName("test")).build();
129+
state = IngestService.innerPut(ecsTrueRequest, state);
130+
state = IngestService.innerPut(ecsFalseRequest, state);
131+
state = IngestService.innerPut(ecsNullRequest, state);
132+
133+
final ClusterState finalState = state;
134+
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState));
135+
136+
DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING,
137+
"User-Agent ingest plugin will use ECS-formatted output",
138+
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
139+
"#ingest-user-agent-ecs-always",
140+
"Ingest pipelines [ecs_false, ecs_null] will change to using ECS output format in 7.0");
141+
assertEquals(singletonList(expected), issues);
142+
}
86143
}

0 commit comments

Comments
 (0)