PromQL: engine panics with @ modifier on empty ranges for several range functions #18018
Description
What did you do?
I upgraded Prometheus to v3.8.1 and noticed that some queries causing panic from the Cortex's fuzz tests. Also the same panic is reproducible on latest release v3.9.1
execution: unexpected error: runtime error: index out of range [0] with length 0
What did you expect to see?
The query should return no data instead of panic.
What did you see instead? Under which circumstances?
I ran several queries with range promql functions, subquery and @ modifier. The @ modifier timestamp doesn't have any data.
quantile_over_time(scalar(up) + 1, {__name__="up"}[1h:1m] @ 1111111)
deriv({__name__="up"}[1h:1m] @ 1111111)
changes({__name__="up"}[1h:1m] @ 1111111)
Those queries will panic and throw the error shared above. I also asked AI to help create a repro test. The panic seems caused by the change from #16797.
package promqltest
import (
"context"
"testing"
"time"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/stretchr/testify/require"
)
func TestAtModifierEmptyData_PanicQueries(t *testing.T) {
// Data only at 0s, 10s, 20s. Eval at 1111111s → no data at that time.
loadInput := `load 10s
up 1 2 3
`
storage := LoadedStorage(t, loadInput)
defer storage.Close()
opts := promql.EngineOpts{
Timeout: 10 * time.Second,
MaxSamples: 1e6,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)
ctx := context.Background()
tsNoData := time.Unix(1111111, 0)
// --- Queries that PANIC without fix ---
t.Run("quantile_over_time_vector_arg_at_no_data", func(t *testing.T) {
// First arg is vector (scalar(up)+1). At @ 1111111: no series → empty evalVals → engine may panic on evalVals[j][0].
// Or function gets empty vectorVals/matrix and panics on vectorVals[0][0] / matrixVal[0].
query, err := engine.NewInstantQuery(ctx, storage, nil,
`quantile_over_time(scalar(up) + 1, {__name__="up"} [1h:1m] @ 1111111)`,
tsNoData)
require.NoError(t, err)
result := query.Exec(ctx)
require.NoError(t, result.Err)
query.Close()
})
t.Run("predict_linear_at_no_data", func(t *testing.T) {
// Range at @ 111111111 has no data → empty matrix → function may panic on matrixVal[0] or vectorVals[0][0].
query, err := engine.NewInstantQuery(ctx, storage, nil,
`predict_linear({__name__="up"} [1h:1m] @ 111111111, 0.1)`,
tsNoData)
require.NoError(t, err)
result := query.Exec(ctx)
require.NoError(t, result.Err)
query.Close()
})
}
// TestAtModifierEmptyData_OtherMatrixFunctions runs queries that use the same
// unsafe pattern (matrixVal[0] or vectorVals[*][0] without length check). With
// the current engine they do not panic (engine skips the call when range is empty).
// These tests document expected behavior and guard against regressions; adding
// empty-input guards in these functions would make them robust if the engine
// ever passed an empty matrix.
func TestAtModifierEmptyData_OtherMatrixFunctions(t *testing.T) {
// double_exponential_smoothing is experimental; enable for that subtest.
defer func() { parser.EnableExperimentalFunctions = false }()
loadInput := `load 10s
up 1 2 3
`
storage := LoadedStorage(t, loadInput)
defer storage.Close()
opts := promql.EngineOpts{
Timeout: 10 * time.Second,
MaxSamples: 1e6,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)
ctx := context.Background()
tsNoData := time.Unix(1111111, 0)
queries := []struct {
name string
query string
}{
{"deriv", `deriv({__name__="up"} [1h:1m] @ 1111111)`},
{"first_over_time", `first_over_time({__name__="up"} [1h:1m] @ 1111111)`},
{"last_over_time", `last_over_time({__name__="up"} [1h:1m] @ 1111111)`},
{"resets", `resets({__name__="up"} [1h:1m] @ 1111111)`},
{"changes", `changes({__name__="up"} [1h:1m] @ 1111111)`},
{"sum_over_time", `sum_over_time({__name__="up"} [1h:1m] @ 1111111)`},
{"avg_over_time", `avg_over_time({__name__="up"} [1h:1m] @ 1111111)`},
{"double_exponential_smoothing", `double_exponential_smoothing({__name__="up"} [1h:1m] @ 1111111, 0.5, 0.3)`},
}
for _, q := range queries {
t.Run(q.name, func(t *testing.T) {
if q.name == "double_exponential_smoothing" {
parser.EnableExperimentalFunctions = true
}
query, err := engine.NewInstantQuery(ctx, storage, nil, q.query, tsNoData)
require.NoError(t, err)
result := query.Exec(ctx)
require.NoError(t, result.Err)
query.Close()
})
}
}
func TestAtModifierEmptyData_QueriesThatWork(t *testing.T) {
loadInput := `load 10s
up 1 2 3
`
storage := LoadedStorage(t, loadInput)
defer storage.Close()
opts := promql.EngineOpts{
Timeout: 10 * time.Second,
MaxSamples: 1e6,
EnableAtModifier: true,
EnableNegativeOffset: true,
}
engine := promql.NewEngine(opts)
ctx := context.Background()
tsNoData := time.Unix(1111111, 0)
// --- Queries that do NOT panic (scalar first arg or no @) ---
t.Run("quantile_over_time_scalar_arg_at_no_data", func(t *testing.T) {
query, err := engine.NewInstantQuery(ctx, storage, nil,
`quantile_over_time(0.1, {__name__="up"} [1h:1m] @ 1111111)`,
tsNoData)
require.NoError(t, err)
result := query.Exec(ctx)
require.NoError(t, result.Err)
query.Close()
})
t.Run("quantile_over_time_no_at", func(t *testing.T) {
query, err := engine.NewInstantQuery(ctx, storage, nil,
`quantile_over_time(0.1, {__name__="up"} [1h:1m])`,
tsNoData)
require.NoError(t, err)
result := query.Exec(ctx)
require.NoError(t, result.Err)
query.Close()
})
}
Example panic stack trace
ts=2026-02-04T07:36:49.155683215Z caller=handler.go:87 level=error caller=engine.go:1123 \
time=2026-02-04T07:36:49.155674062Z msg="runtime panic during query evaluation" \
expr="predict_linear({__name__=\"test_series_a\",status_code!=\"404\"} offset 2m9s[1h:1m] @ 1770183402.434 offset 1m40s, -time())" \
err="runtime error: index out of range [0] with length 0"
stacktrace:
goroutine 55914 [running]:
github.com/prometheus/prometheus/promql.(*evaluator).recover(
0xc000aa4c00,
{0x40c7640, 0xc0011c4a80},
0xc000dfe9a0,
0xc000dfe9b8,
)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/promql/engine.go:1121 +0x28b
panic({0x37dfba0?, 0xc000fc8138?})
/usr/local/go/src/runtime/panic.go:783 +0x132
github.com/prometheus/prometheus/promql.funcPredictLinear(
{0xc001418690?, 0x37?, 0x4?},
{0x0?, 0x0?, 0x0?},
{0xc000a5f220?, 0x0?, 0xc001535000?},
0xc00186c5a0,
)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/promql/functions.go:1504 +0x885
github.com/prometheus/prometheus/promql.(*evaluator).eval.func3(
{0xc001418690?, 0x0?, 0x0?},
{0x0?, 0x0?, 0xc000aa4c00?},
{0x19c272723c2?, 0xc000a6bdb0?, 0x1?},
0xc00186c5a0,
)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/promql/engine.go:1854 +0x5e
github.com/prometheus/prometheus/promql.(*evaluator).rangeEval(
0xc000aa4c00,
{0x40b7fa8, 0xc0011c4d80},
0x0,
0xc000dfe718,
{0xc000a5f220, 0x2, 0x100004d3dc5?},
)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/promql/engine.go:1338 +0xa5a
github.com/prometheus/prometheus/promql.(*evaluator).eval(
0xc000aa4c00,
{0x40b7fa8, 0.c0011c4d20},
{0x40c7640, 0xc0011c4a80},
)
/__w/cortex/cortex/vendor/github.com/prometheus/prometheus/promql/engine.go:1853 +0x374a