Skip to content

Commit 1146b88

Browse files
dnhatndnhatn
authored
Revert "Partition time-series source (#140475)" (#141229)
The competitive benchmark shows a significant performance regression from #140475. This is likely due to the high number of CPUs in the benchmark, which results in many small slices and adds significant overhead. It is better to revert this change for now and adjust the partitioning for cases with high CPU and more than one shard before merging again. Relates #14047
1 parent 5d0bfe1 commit 1146b88

13 files changed

Lines changed: 57 additions & 712 deletions

File tree

‎docs/changelog/140475.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterTimeSeriesIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
5353
public class MultiClusterTimeSeriesIT extends ESRestTestCase {
5454

55-
static final List<String> REQUIRED_CAPABILITIES = List.of("ts_command_v0", "rate_fix_resets_multiple_segments");
55+
static final List<String> REQUIRED_CAPABILITIES = List.of("ts_command_v0");
5656

5757
static ElasticsearchCluster remoteCluster = Clusters.remoteCluster();
5858
static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster);

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,6 @@ public static LogicalOptimizerContext unboundLogicalOptimizerContext() {
592592
public static final PlannerSettings TEST_PLANNER_SETTINGS = new PlannerSettings(
593593
DataPartitioning.AUTO,
594594
ByteSizeValue.ofMb(1),
595-
ByteSizeValue.ofMb(2),
596595
10_000,
597596
ByteSizeValue.ofMb(1)
598597
);

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec‎

Lines changed: 3 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ max_bytes:long | cluster: keyword
2424
maxRate
2525
required_capability: ts_command_v0
2626
required_capability: double_quotes_source_enclosing
27-
required_capability: rate_fix_resets_multiple_segments
2827
TS k8s | STATS max=max(rate(network.total_bytes_in)) | EVAL max=ROUND(max, 6) | KEEP max;
2928

3029
max: double
@@ -34,7 +33,6 @@ max: double
3433
maxRatePerMinute
3534
required_capability: ts_command_v0
3635
required_capability: double_quotes_source_enclosing
37-
required_capability: rate_fix_resets_multiple_segments
3836
TS k8s | STATS max=max(60 * rate(network.total_bytes_in)) | EVAL max=ROUND(max, 6) | KEEP max;
3937

4038
max: double
@@ -43,7 +41,6 @@ max: double
4341

4442
maxCost
4543
required_capability: ts_command_v0
46-
required_capability: rate_fix_resets_multiple_segments
4744
TS k8s | STATS max_cost=max(rate(network.total_cost)) | EVAL max_cost=ROUND(max_cost, 6) | KEEP max_cost;
4845

4946
max_cost: double
@@ -52,7 +49,6 @@ max_cost: double
5249

5350
maxRateAndBytes
5451
required_capability: ts_command_v0
55-
required_capability: rate_fix_resets_multiple_segments
5652
TS k8s | STATS max_rate=max(60 * rate(network.total_bytes_in)), max_bytes=max(network.bytes_in) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_bytes;
5753

5854
max_rate: double | max_bytes: long
@@ -61,7 +57,6 @@ max_rate: double | max_bytes: long
6157

6258
maxRateAndBytesExplicit
6359
required_capability: ts_command_v0
64-
required_capability: rate_fix_resets_multiple_segments
6560
TS k8s | STATS max_rate=max(60 * rate(network.total_bytes_in)), max_bytes=max(last_over_time(network.bytes_in)) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_bytes;
6661

6762
max_rate: double | max_bytes: long
@@ -70,7 +65,6 @@ max_rate: double | max_bytes: long
7065

7166
maxRateAndMarkupBytes
7267
required_capability: ts_command_v0
73-
required_capability: rate_fix_resets_multiple_segments
7468
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), max_markup=max(network.bytes_in * 1.05) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_markup;
7569

7670
max_rate: double | max_markup: double
@@ -79,7 +73,6 @@ max_rate: double | max_markup: double
7973

8074
maxRateAndMarkupBytesExplicit
8175
required_capability: ts_command_v0
82-
required_capability: rate_fix_resets_multiple_segments
8376
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), max_bytes_in = max(last_over_time(network.bytes_in) * 1.05) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_bytes_in;
8477

8578
max_rate: double | max_bytes_in: double
@@ -88,7 +81,6 @@ max_rate: double | max_bytes_in: double
8881

8982
maxRateAndLastBytesIn
9083
required_capability: ts_command_v0
91-
required_capability: rate_fix_resets_multiple_segments
9284
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), max_bytes_in = max(last_over_time(network.bytes_in * 1.05)) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_bytes_in;
9385

9486
max_rate: double | max_bytes_in: double
@@ -97,7 +89,6 @@ max_rate: double | max_bytes_in: double
9789

9890
maxRateAndBytesAndCost
9991
required_capability: ts_command_v0
100-
required_capability: rate_fix_resets_multiple_segments
10192
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), max_bytes=max(max_over_time(network.bytes_in)), max_cost=max(rate(network.total_cost)) | EVAL max_rate=ROUND(max_rate, 6), max_cost=ROUND(max_cost, 6) | KEEP max_rate, max_bytes, max_cost;
10293

10394
max_rate:double | max_bytes:long | max_cost:double
@@ -106,7 +97,6 @@ max_rate:double | max_bytes:long | max_cost:double
10697

10798
sumRate
10899
required_capability: ts_command_v0
109-
required_capability: rate_fix_resets_multiple_segments
110100
TS k8s | STATS bytes=sum(rate(network.total_bytes_in)), sum_cost=sum(rate(network.total_cost)) BY cluster | EVAL bytes=ROUND(bytes, 6), sum_cost=ROUND(sum_cost, 6) | KEEP bytes, sum_cost, cluster | SORT cluster;
111101

112102
bytes: double | sum_cost: double | cluster: keyword
@@ -118,7 +108,6 @@ bytes: double | sum_cost: double | cluster: keyword
118108
oneRateWithBucket
119109
required_capability: ts_command_v0
120110
required_capability: rate_with_interpolation_v2
121-
required_capability: rate_fix_resets_multiple_segments
122111

123112
// tag::rate[]
124113
TS k8s
@@ -136,8 +125,8 @@ max_rate: double | time_bucket:date
136125

137126
oneRateWithTBucket
138127
required_capability: ts_command_v0
128+
required_capability: tbucket
139129
required_capability: rate_with_interpolation_v2
140-
required_capability: rate_fix_resets_multiple_segments
141130

142131
TS k8s
143132
| STATS max_rate=max(rate(network.total_bytes_in)) BY time_bucket = tbucket(5minute)
@@ -152,7 +141,6 @@ max_rate: double | time_bucket:date
152141
oneRateWithPromql
153142
required_capability: promql_command_v0
154143
required_capability: rate_with_interpolation_v2
155-
required_capability: rate_fix_resets_multiple_segments
156144

157145
PROMQL index=k8s step=5m max(rate(network.total_bytes_in[5m]))
158146
| SORT step DESC | LIMIT 2;
@@ -165,8 +153,7 @@ max(rate(network.total_bytes_in[5m])):double | step:datetime
165153

166154
oneRateWithSingleTBucket
167155
required_capability: ts_command_v0
168-
required_capability: rate_fix_resets_multiple_segments
169-
156+
required_capability: tbucket
170157
TS k8s
171158
| STATS max_rate=max(rate(network.total_bytes_in)) BY time_bucket = tbucket(1h)
172159
| EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, time_bucket;
@@ -177,7 +164,6 @@ max_rate: double | time_bucket:date
177164

178165
oneRateWithSingleStepPromql
179166
required_capability: promql_command_v0
180-
required_capability: rate_fix_resets_multiple_segments
181167
PROMQL index=k8s step=1h max(rate(network.total_bytes_in[1h]));
182168

183169
max(rate(network.total_bytes_in[1h])):double | step:datetime
@@ -188,7 +174,6 @@ max(rate(network.total_bytes_in[1h])):double | step:datetime
188174
twoRatesWithBucket
189175
required_capability: ts_command_v0
190176
required_capability: rate_with_interpolation_v2
191-
required_capability: rate_fix_resets_multiple_segments
192177

193178
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), sum_rate=sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | EVAL max_rate=ROUND(max_rate, 6), sum_rate=ROUND(sum_rate, 6) | KEEP max_rate, sum_rate, time_bucket | SORT time_bucket DESC | LIMIT 3;
194179

@@ -200,9 +185,8 @@ max_rate:double | sum_rate:double | time_bucket:datetime
200185

201186
twoRatesWithTBucket
202187
required_capability: ts_command_v0
188+
required_capability: tbucket
203189
required_capability: rate_with_interpolation_v2
204-
required_capability: rate_fix_resets_multiple_segments
205-
206190
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), sum_rate=sum(rate(network.total_bytes_in)) BY time_bucket = tbucket(5minute) | EVAL max_rate=ROUND(max_rate, 6), sum_rate=ROUND(sum_rate, 6) | KEEP max_rate, sum_rate, time_bucket | SORT time_bucket DESC | LIMIT 3;
207191

208192
max_rate:double | sum_rate:double | time_bucket:datetime
@@ -214,8 +198,6 @@ max_rate:double | sum_rate:double | time_bucket:datetime
214198
oneRateWithBucketAndCluster
215199
required_capability: ts_command_v0
216200
required_capability: rate_with_interpolation_v2
217-
required_capability: rate_fix_resets_multiple_segments
218-
219201
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, time_bucket, cluster | SORT time_bucket DESC, cluster | LIMIT 6;
220202

221203
max_rate:double | time_bucket:datetime | cluster:keyword
@@ -230,7 +212,6 @@ max_rate:double | time_bucket:datetime | cluster:keyword
230212
BytesAndCostByBucketAndCluster
231213
required_capability: ts_command_v0
232214
required_capability: rate_with_interpolation_v2
233-
required_capability: rate_fix_resets_multiple_segments
234215

235216
TS k8s | STATS max_rate=max(rate(network.total_bytes_in)), max_cost=max(max_over_time(network.cost)) BY time_bucket = bucket(@timestamp,5minute), cluster | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, max_cost, time_bucket, cluster | SORT time_bucket DESC, cluster | LIMIT 6;
236217

@@ -247,7 +228,6 @@ max_rate:double | max_cost:double | time_bucket:date | cluster: keyword
247228
oneRateWithBucketAndClusterThenFilter
248229
required_capability: ts_command_v0
249230
required_capability: rate_with_interpolation_v2
250-
required_capability: rate_fix_resets_multiple_segments
251231

252232
TS k8s | WHERE cluster=="prod" | STATS max_rate=max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, time_bucket, cluster | SORT time_bucket DESC | LIMIT 3;
253233

@@ -260,8 +240,6 @@ max_rate:double | time_bucket:datetime | cluster:keyword
260240

261241
maxRateWithInlineFilter
262242
required_capability: ts_command_v0
263-
required_capability: rate_fix_resets_multiple_segments
264-
265243
TS k8s | STATS max_rate = max(rate(network.total_bytes_in)) WHERE cluster=="prod" BY cluster | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, cluster | SORT cluster;
266244

267245
max_rate:double | cluster:keyword
@@ -272,8 +250,6 @@ null | staging
272250

273251
maxRateWithPreFilter
274252
required_capability: ts_command_v0
275-
required_capability: rate_fix_resets_multiple_segments
276-
277253
TS k8s | WHERE cluster=="prod" | STATS max_rate = max(rate(network.total_bytes_in)) BY cluster | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, cluster | SORT cluster;
278254

279255
max_rate:double | cluster:keyword
@@ -283,7 +259,6 @@ max_rate:double | cluster:keyword
283259
notEnoughSamples
284260
required_capability: ts_command_v0
285261
required_capability: rate_with_interpolation
286-
required_capability: rate_fix_resets_multiple_segments
287262

288263
TS k8s | WHERE @timestamp <= "2024-05-10T00:06:14.000Z" | STATS max_rate=max(rate(network.total_bytes_in)) BY pod, time_bucket = bucket(@timestamp,1minute) | EVAL max_rate=ROUND(max_rate, 6) | KEEP max_rate, pod, time_bucket | SORT pod, time_bucket DESC | LIMIT 10;
289264

@@ -541,7 +516,6 @@ distincts:long | distincts_imprecise:long | cluster:keyword | time_bucket:dateti
541516
two_rates
542517
required_capability: ts_command_v0
543518
required_capability: rate_with_interpolation
544-
required_capability: rate_fix_resets_multiple_segments
545519

546520
TS k8s | STATS cost_per_mb=max(rate(network.total_bytes_in) / 1024 * 1024 * rate(network.total_cost)) BY cluster, time_bucket = bucket(@timestamp,5minute) | EVAL cost_per_mb=ROUND(cost_per_mb, 6) | KEEP cost_per_mb, cluster, time_bucket | SORT cost_per_mb DESC, cluster, time_bucket DESC | LIMIT 5;
547521

@@ -742,7 +716,6 @@ sd:double | var:double | sd_squared:double | cluster:keyword | time_bucket:datet
742716

743717
Max of Rate with Bucket
744718
required_capability: ts_command_v0
745-
required_capability: rate_fix_resets_multiple_segments
746719

747720
TS k8s
748721
| STATS maxRate = max(rate(network.total_cost)) BY tbucket = bucket(@timestamp, 1hour)
@@ -754,7 +727,6 @@ maxRate:double | tbucket:datetime
754727

755728
Max of Rate with Bucket, rename _tsid
756729
required_capability: ts_command_v0
757-
required_capability: rate_fix_resets_multiple_segments
758730

759731
TS k8s METADATA _tsid
760732
| RENAME _tsid AS newTsid
@@ -769,7 +741,6 @@ maxRate:double | tbucket:datetime
769741

770742
Max of Rate with Bucket, drop _tsid
771743
required_capability: ts_command_v0
772-
required_capability: rate_fix_resets_multiple_segments
773744

774745
TS k8s METADATA _tsid
775746
| DROP _tsid
@@ -1129,7 +1100,6 @@ max:double | inc:double | time_bucket:date_nanos | cluster:keyword
11291100
dateNanosOneRateIncreaseWithBucketAndClusterThenFilter
11301101
required_capability: ts_command_v0
11311102
required_capability: ts_rate_datenanos_2
1132-
required_capability: rate_fix_resets_multiple_segments
11331103

11341104
TS datenanos-k8s
11351105
| WHERE cluster=="prod"
@@ -1178,7 +1148,6 @@ required_capability: ts_command_v0
11781148
required_capability: ts_linreg_derivative
11791149
required_capability: ts_deriv_datenanos
11801150
required_capability: rate_with_interpolation_v2
1181-
required_capability: rate_fix_resets_multiple_segments
11821151

11831152
TS k8s
11841153
| STATS max_deriv = max(deriv(to_long(network.total_bytes_in))), max_rate = max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster

0 commit comments

Comments
 (0)