Skip to content

Commit 4ebb748

Browse files
zhubotang-wqzhubotang-wq
authored
Es 13868 Enforce Soft Index Limit Warning (#143711)
* ES-13868 Enforce index count warning
1 parent 354f8ea commit 4ebb748

4 files changed

Lines changed: 52 additions & 12 deletions

File tree

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,15 @@ public MetadataCreateIndexService(
244244
}
245245
}
246246

247+
public static long getTotalUserIndices(SystemIndices systemIndices, ProjectMetadata projectMetadata) {
248+
return projectMetadata.stream()
249+
.filter(
250+
indexMetadata -> indexMetadata.isSystem() == false
251+
&& systemIndices.isFeatureAssociatedIndex(indexMetadata.getIndex().getName()) == false
252+
)
253+
.count();
254+
}
255+
247256
public void validateIndexLimit(ProjectMetadata projectMetadata, CreateIndexClusterStateUpdateRequest request) {
248257
if (maxIndicesPerProjectEnabled == false) {
249258
return;
@@ -257,12 +266,7 @@ public void validateIndexLimit(ProjectMetadata projectMetadata, CreateIndexClust
257266
return;
258267
}
259268

260-
var totalUserIndices = projectMetadata.stream()
261-
.filter(
262-
indexMetadata -> indexMetadata.isSystem() == false
263-
&& systemIndices.isFeatureAssociatedIndex(indexMetadata.getIndex().getName()) == false
264-
)
265-
.count();
269+
var totalUserIndices = getTotalUserIndices(systemIndices, projectMetadata);
266270
if (totalUserIndices >= maxIndicesPerProject) {
267271
throw new IndexLimitExceededException(
268272
"This action would add an index, but this project currently has ["

‎server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,18 @@
1313
import org.apache.logging.log4j.Logger;
1414
import org.apache.lucene.store.AlreadyClosedException;
1515
import org.elasticsearch.cluster.routing.ShardRouting;
16+
import org.elasticsearch.cluster.service.ClusterService;
1617
import org.elasticsearch.common.component.AbstractLifecycleComponent;
1718
import org.elasticsearch.common.util.SingleObjectCache;
19+
import org.elasticsearch.core.FixForMultiProject;
1820
import org.elasticsearch.core.TimeValue;
1921
import org.elasticsearch.index.IndexMode;
2022
import org.elasticsearch.index.IndexService;
2123
import org.elasticsearch.index.shard.DocsStats;
2224
import org.elasticsearch.index.shard.IllegalIndexShardStateException;
2325
import org.elasticsearch.index.shard.IndexShard;
2426
import org.elasticsearch.indices.IndicesService;
27+
import org.elasticsearch.indices.SystemIndices;
2528
import org.elasticsearch.telemetry.metric.LongWithAttributes;
2629
import org.elasticsearch.telemetry.metric.MeterRegistry;
2730

@@ -33,27 +36,46 @@
3336
import java.util.concurrent.atomic.AtomicLong;
3437
import java.util.function.Supplier;
3538

39+
import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.getTotalUserIndices;
40+
3641
/**
3742
* {@link IndicesMetrics} monitors index statistics on an Elasticsearch node and exposes them as metrics
3843
* through the provided {@link MeterRegistry}. It tracks the current total number of indices, document count, and
3944
* store size (in bytes) for each index mode.
4045
*/
4146
public class IndicesMetrics extends AbstractLifecycleComponent {
47+
public static final String USER_INDEX_TOTAL_METRIC_NAME = "es.indices.users.total";
4248
private final Logger logger = LogManager.getLogger(IndicesMetrics.class);
4349
private final MeterRegistry registry;
4450
private final List<AutoCloseable> metrics = new ArrayList<>();
4551
private final IndicesStatsCache stateCache;
52+
private final ClusterService clusterService;
53+
private final SystemIndices systemIndices;
4654

47-
public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService, TimeValue metricsInterval) {
55+
public IndicesMetrics(
56+
MeterRegistry meterRegistry,
57+
IndicesService indicesService,
58+
TimeValue metricsInterval,
59+
ClusterService clusterService,
60+
SystemIndices systemIndices
61+
) {
4862
this.registry = meterRegistry;
4963
// Use half of the update interval to ensure that results aren't cached across updates,
5064
// while preventing the cache from expiring when reading different gauges within the same update.
5165
var cacheExpiry = new TimeValue(metricsInterval.getMillis() / 2);
5266
this.stateCache = new IndicesStatsCache(indicesService, cacheExpiry);
67+
this.clusterService = clusterService;
68+
this.systemIndices = systemIndices;
5369
}
5470

55-
private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) {
56-
final int TOTAL_METRICS = 52;
71+
@FixForMultiProject(description = "When multi-project arrives we should add project ID to the USER_INDEX_TOTAL_METRIC_NAME.")
72+
private static List<AutoCloseable> registerAsyncMetrics(
73+
MeterRegistry registry,
74+
IndicesStatsCache cache,
75+
ClusterService clusterService,
76+
SystemIndices systemIndices
77+
) {
78+
final int TOTAL_METRICS = 53;
5779
List<AutoCloseable> metrics = new ArrayList<>(TOTAL_METRICS);
5880
for (IndexMode indexMode : IndexMode.values()) {
5981
String name = indexMode.getName();
@@ -165,6 +187,14 @@ private static List<AutoCloseable> registerAsyncMetrics(MeterRegistry registry,
165187
)
166188
);
167189
}
190+
metrics.add(registry.registerLongGauge(USER_INDEX_TOTAL_METRIC_NAME, "Total number of user indices", "index", () -> {
191+
if (clusterService.state().clusterRecovered() == false || clusterService.state().nodes().isLocalNodeElectedMaster() == false) {
192+
return null;
193+
}
194+
return new LongWithAttributes(
195+
getTotalUserIndices(systemIndices, clusterService.state().getMetadata().projects().values().iterator().next())
196+
);
197+
}));
168198
assert metrics.size() == TOTAL_METRICS : "total number of metrics has changed";
169199
return metrics;
170200
}
@@ -180,7 +210,7 @@ static Supplier<LongWithAttributes> diffGauge(Supplier<Long> currentValue) {
180210

181211
@Override
182212
protected void doStart() {
183-
metrics.addAll(registerAsyncMetrics(registry, stateCache));
213+
metrics.addAll(registerAsyncMetrics(registry, stateCache, clusterService, systemIndices));
184214
}
185215

186216
@Override

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1272,7 +1272,13 @@ public <T extends TransportResponse> void sendRequest(
12721272

12731273
final TimeValue metricsInterval = settings.getAsTime("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(10));
12741274
final NodeMetrics nodeMetrics = new NodeMetrics(telemetryProvider.getMeterRegistry(), nodeService, metricsInterval);
1275-
final IndicesMetrics indicesMetrics = new IndicesMetrics(telemetryProvider.getMeterRegistry(), indicesService, metricsInterval);
1275+
final IndicesMetrics indicesMetrics = new IndicesMetrics(
1276+
telemetryProvider.getMeterRegistry(),
1277+
indicesService,
1278+
metricsInterval,
1279+
clusterService,
1280+
systemIndices
1281+
);
12761282
final SystemMetrics systemMetrics = new SystemMetrics(telemetryProvider.getMeterRegistry());
12771283

12781284
OnlinePrewarmingService onlinePrewarmingService = pluginsService.loadSingletonServiceProvider(

‎test/framework/src/main/java/org/elasticsearch/telemetry/RecordingInstruments.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ public static class RecordingLongGauge extends CallbackRecordingInstrument imple
207207
public RecordingLongGauge(String name, Supplier<Collection<LongWithAttributes>> observer, MetricRecorder<Instrument> recorder) {
208208
super(name, () -> {
209209
var observation = observer.get();
210-
return observation.stream().map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
210+
return observation.stream().filter(Objects::nonNull).map(o -> new Tuple<>((Number) o.value(), o.attributes())).toList();
211211
}, recorder);
212212
}
213213
}

0 commit comments

Comments
 (0)