Skip to content

Commit 32f46f2

Browse files
stu-elasticstu-elastic
authored
Scripting: stats per context in nodes stats (#54008)
Adds script cache stats to `_node/stats`. If using the general cache: ``` "script_cache": { "sum": { "compilations": 12, "cache_evictions": 9, "compilation_limit_triggered": 5 } } ``` If using context caches: ``` "script_cache": { "sum": { "compilations": 13, "cache_evictions": 9, "compilation_limit_triggered": 5 }, "contexts": [ { "context": "aggregation_selector", "compilations": 8, "cache_evictions": 6, "compilation_limit_triggered": 3 }, { "context": "aggs", "compilations": 5, "cache_evictions": 3, "compilation_limit_triggered": 2 }, ``` Refs: #50152
1 parent 744f4eb commit 32f46f2

18 files changed

Lines changed: 338 additions & 31 deletions

File tree

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.action.admin.cluster.node.stats;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.action.support.nodes.BaseNodeResponse;
2324
import org.elasticsearch.cluster.node.DiscoveryNode;
2425
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
@@ -37,6 +38,7 @@
3738
import org.elasticsearch.monitor.os.OsStats;
3839
import org.elasticsearch.monitor.process.ProcessStats;
3940
import org.elasticsearch.node.AdaptiveSelectionStats;
41+
import org.elasticsearch.script.ScriptCacheStats;
4042
import org.elasticsearch.script.ScriptStats;
4143
import org.elasticsearch.threadpool.ThreadPoolStats;
4244
import org.elasticsearch.transport.TransportStats;
@@ -81,6 +83,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
8183
@Nullable
8284
private ScriptStats scriptStats;
8385

86+
@Nullable
87+
private ScriptCacheStats scriptCacheStats;
88+
8489
@Nullable
8590
private DiscoveryStats discoveryStats;
8691

@@ -108,6 +113,11 @@ public NodeStats(StreamInput in) throws IOException {
108113
discoveryStats = in.readOptionalWriteable(DiscoveryStats::new);
109114
ingestStats = in.readOptionalWriteable(IngestStats::new);
110115
adaptiveSelectionStats = in.readOptionalWriteable(AdaptiveSelectionStats::new);
116+
if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
117+
scriptCacheStats = in.readOptionalWriteable(ScriptCacheStats::new);
118+
} else {
119+
scriptCacheStats = null;
120+
}
111121
}
112122

113123
public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats indices,
@@ -117,7 +127,8 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
117127
@Nullable ScriptStats scriptStats,
118128
@Nullable DiscoveryStats discoveryStats,
119129
@Nullable IngestStats ingestStats,
120-
@Nullable AdaptiveSelectionStats adaptiveSelectionStats) {
130+
@Nullable AdaptiveSelectionStats adaptiveSelectionStats,
131+
@Nullable ScriptCacheStats scriptCacheStats) {
121132
super(node);
122133
this.timestamp = timestamp;
123134
this.indices = indices;
@@ -133,6 +144,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
133144
this.discoveryStats = discoveryStats;
134145
this.ingestStats = ingestStats;
135146
this.adaptiveSelectionStats = adaptiveSelectionStats;
147+
this.scriptCacheStats = scriptCacheStats;
136148
}
137149

138150
public long getTimestamp() {
@@ -227,6 +239,11 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() {
227239
return adaptiveSelectionStats;
228240
}
229241

242+
@Nullable
243+
public ScriptCacheStats getScriptCacheStats() {
244+
return scriptCacheStats;
245+
}
246+
230247
@Override
231248
public void writeTo(StreamOutput out) throws IOException {
232249
super.writeTo(out);
@@ -249,6 +266,9 @@ public void writeTo(StreamOutput out) throws IOException {
249266
out.writeOptionalWriteable(discoveryStats);
250267
out.writeOptionalWriteable(ingestStats);
251268
out.writeOptionalWriteable(adaptiveSelectionStats);
269+
if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
270+
out.writeOptionalWriteable(scriptCacheStats);
271+
}
252272
}
253273

254274
@Override
@@ -312,6 +332,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
312332
if (getAdaptiveSelectionStats() != null) {
313333
getAdaptiveSelectionStats().toXContent(builder, params);
314334
}
335+
if (getScriptCacheStats() != null) {
336+
getScriptCacheStats().toXContent(builder, params);
337+
}
315338
return builder;
316339
}
317340
}

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ public enum Metric {
219219
SCRIPT("script"),
220220
DISCOVERY("discovery"),
221221
INGEST("ingest"),
222-
ADAPTIVE_SELECTION("adaptive_selection");
222+
ADAPTIVE_SELECTION("adaptive_selection"),
223+
SCRIPT_CACHE("script_cache");
223224

224225
private String metricName;
225226

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,14 @@ public NodesStatsRequestBuilder setAdaptiveSelection(boolean adaptiveSelection)
149149
return this;
150150
}
151151

152+
/**
153+
* Should script context cache statistics be returned
154+
*/
155+
public NodesStatsRequestBuilder setScriptCache(boolean scriptCache) {
156+
addOrRemoveMetric(scriptCache, NodesStatsRequest.Metric.SCRIPT_CACHE);
157+
return this;
158+
}
159+
152160
/**
153161
* Helper method for adding metrics to a request
154162
*/

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest, Task task)
8383
NodesStatsRequest.Metric.SCRIPT.containedIn(metrics),
8484
NodesStatsRequest.Metric.DISCOVERY.containedIn(metrics),
8585
NodesStatsRequest.Metric.INGEST.containedIn(metrics),
86-
NodesStatsRequest.Metric.ADAPTIVE_SELECTION.containedIn(metrics));
86+
NodesStatsRequest.Metric.ADAPTIVE_SELECTION.containedIn(metrics),
87+
NodesStatsRequest.Metric.SCRIPT_CACHE.containedIn(metrics));
8788
}
8889

8990
public static class NodeStatsRequest extends BaseNodeRequest {

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce
9898
protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest, Task task) {
9999
NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
100100
NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE,
101-
true, true, true, false, true, false, false, false, false, false, true, false);
101+
true, true, true, false, true, false, false, false, false, false, true, false, false);
102102
List<ShardStats> shardsStats = new ArrayList<>();
103103
for (IndexService indexService : indicesService) {
104104
for (IndexShard indexShard : indexService) {

‎server/src/main/java/org/elasticsearch/node/NodeService.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm,
103103

104104
public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool,
105105
boolean fs, boolean transport, boolean http, boolean circuitBreaker,
106-
boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection) {
106+
boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection, boolean scriptCache) {
107107
// for indices stats we want to include previous allocated shards stats as well (it will
108108
// only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
109109
return new NodeStats(transportService.getLocalNode(), System.currentTimeMillis(),
@@ -119,7 +119,8 @@ public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, bo
119119
script ? scriptService.stats() : null,
120120
discoveryStats ? discovery.stats() : null,
121121
ingest ? ingestService.stats() : null,
122-
adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null
122+
adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null,
123+
scriptCache ? scriptService.cacheStats() : null
123124
);
124125
}
125126

‎server/src/main/java/org/elasticsearch/script/ScriptCache.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,17 @@ public class ScriptCache {
5656
final TimeValue cacheExpire;
5757
final Tuple<Integer, TimeValue> rate;
5858
private final double compilesAllowedPerNano;
59+
private final String contextRateSetting;
5960

6061
ScriptCache(
6162
int cacheMaxSize,
6263
TimeValue cacheExpire,
63-
Tuple<Integer, TimeValue> maxCompilationRate
64+
Tuple<Integer, TimeValue> maxCompilationRate,
65+
String contextRateSetting
6466
) {
6567
this.cacheSize = cacheMaxSize;
6668
this.cacheExpire = cacheExpire;
69+
this.contextRateSetting = contextRateSetting;
6770

6871
CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
6972
if (this.cacheSize >= 0) {
@@ -175,7 +178,7 @@ void checkCompilationLimit() {
175178
// Otherwise reject the request
176179
throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" +
177180
rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " +
178-
"this limit can be changed by the [script.max_compilations_rate] setting",
181+
"this limit can be changed by the [" + contextRateSetting + "] setting",
179182
CircuitBreaker.Durability.TRANSIENT);
180183
}
181184
}

‎server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.script;
21+
22+
import org.elasticsearch.common.io.stream.StreamInput;
23+
import org.elasticsearch.common.io.stream.StreamOutput;
24+
import org.elasticsearch.common.io.stream.Writeable;
25+
import org.elasticsearch.common.xcontent.ToXContentFragment;
26+
import org.elasticsearch.common.xcontent.XContentBuilder;
27+
28+
import java.io.IOException;
29+
import java.util.Collections;
30+
import java.util.HashMap;
31+
import java.util.Map;
32+
import java.util.Objects;
33+
import java.util.stream.Collectors;
34+
35+
public class ScriptCacheStats implements Writeable, ToXContentFragment {
36+
private final Map<String,ScriptStats> context;
37+
private final ScriptStats general;
38+
39+
public ScriptCacheStats(Map<String, ScriptStats> context) {
40+
this.context = Collections.unmodifiableMap(context);
41+
this.general = null;
42+
}
43+
44+
public ScriptCacheStats(ScriptStats general) {
45+
this.general = Objects.requireNonNull(general);
46+
this.context = null;
47+
}
48+
49+
public ScriptCacheStats(StreamInput in) throws IOException {
50+
boolean isContext = in.readBoolean();
51+
if (isContext == false) {
52+
general = new ScriptStats(in);
53+
context = null;
54+
return;
55+
}
56+
57+
general = null;
58+
int size = in.readInt();
59+
Map<String,ScriptStats> context = new HashMap<>(size);
60+
for (int i=0; i < size; i++) {
61+
String name = in.readString();
62+
context.put(name, new ScriptStats(in));
63+
}
64+
this.context = Collections.unmodifiableMap(context);
65+
}
66+
67+
@Override
68+
public void writeTo(StreamOutput out) throws IOException {
69+
if (general != null) {
70+
out.writeBoolean(false);
71+
general.writeTo(out);
72+
return;
73+
}
74+
75+
out.writeBoolean(true);
76+
out.writeInt(context.size());
77+
for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
78+
out.writeString(name);
79+
context.get(name).writeTo(out);
80+
}
81+
}
82+
83+
@Override
84+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
85+
builder.startObject(Fields.SCRIPT_CACHE_STATS);
86+
builder.startObject(Fields.SUM);
87+
if (general != null) {
88+
builder.field(ScriptStats.Fields.COMPILATIONS, general.getCompilations());
89+
builder.field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions());
90+
builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered());
91+
builder.endObject().endObject();
92+
return builder;
93+
}
94+
95+
ScriptStats sum = sum();
96+
builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations());
97+
builder.field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions());
98+
builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered());
99+
builder.endObject();
100+
101+
builder.startArray(Fields.CONTEXTS);
102+
for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
103+
ScriptStats stats = context.get(name);
104+
builder.startObject();
105+
builder.field(Fields.CONTEXT, name);
106+
builder.field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations());
107+
builder.field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions());
108+
builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered());
109+
builder.endObject();
110+
}
111+
builder.endArray();
112+
builder.endObject();
113+
114+
return builder;
115+
}
116+
117+
/**
118+
* Get the context specific stats, null if using general cache
119+
*/
120+
public Map<String, ScriptStats> getContextStats() {
121+
return context;
122+
}
123+
124+
/**
125+
* Get the general stats, null if using context cache
126+
*/
127+
public ScriptStats getGeneralStats() {
128+
return general;
129+
}
130+
131+
/**
132+
* The sum of all script stats, either the general stats or the sum of all stats of the context stats.
133+
*/
134+
public ScriptStats sum() {
135+
if (general != null) {
136+
return general;
137+
}
138+
return ScriptStats.sum(context.values());
139+
}
140+
141+
static final class Fields {
142+
static final String SCRIPT_CACHE_STATS = "script_cache";
143+
static final String CONTEXT = "context";
144+
static final String SUM = "sum";
145+
static final String CONTEXTS = "contexts";
146+
}
147+
}

‎server/src/main/java/org/elasticsearch/script/ScriptService.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,10 @@ public ScriptStats stats() {
567567
return cacheHolder.get().stats();
568568
}
569569

570+
public ScriptCacheStats cacheStats() {
571+
return cacheHolder.get().cacheStats();
572+
}
573+
570574
@Override
571575
public void applyClusterState(ClusterChangedEvent event) {
572576
clusterState = event.state();
@@ -602,7 +606,8 @@ static class CacheHolder {
602606
SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
603607
compilationLimitsEnabled ?
604608
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) :
605-
SCRIPT_COMPILATION_RATE_ZERO);
609+
SCRIPT_COMPILATION_RATE_ZERO,
610+
SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey());
606611
}
607612
}
608613

@@ -626,7 +631,8 @@ private static ScriptCache contextFromSettings(Settings settings, ScriptContext<
626631

627632
return new ScriptCache(cacheSize.existsOrFallbackExists(settings) ? cacheSize.get(settings) : context.cacheSizeDefault,
628633
cacheExpire.existsOrFallbackExists(settings) ? cacheExpire.get(settings) : context.cacheExpireDefault,
629-
compileRate);
634+
compileRate,
635+
SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name).getKey());
630636
}
631637

632638
/**
@@ -677,6 +683,17 @@ ScriptStats stats() {
677683
return ScriptStats.sum(contextCache.values().stream().map(AtomicReference::get).map(ScriptCache::stats)::iterator);
678684
}
679685

686+
ScriptCacheStats cacheStats() {
687+
if (general != null) {
688+
return new ScriptCacheStats(general.stats());
689+
}
690+
Map<String, ScriptStats> context = new HashMap<>(contextCache.size());
691+
for (ScriptContext<?> ctx: contexts) {
692+
context.put(ctx.name, contextCache.get(ctx.name).get().stats());
693+
}
694+
return new ScriptCacheStats(context);
695+
}
696+
680697
/**
681698
* Update settings for the context cache, if we're in the context cache mode otherwise no-op.
682699
*/

0 commit comments

Comments
 (0)