fix: make OOM expression a bit less sensitive

smira · smira · commit 7ddb37b1f3e2 · 2026-01-21T16:27:58.000+04:00
In addition to derivative of full PSI for the affected cgroups, also look at avg10 value to provide some hysteresis against small spikes. Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com> (cherry picked from commit 38e280c)
diff --git a/pkg/machinery/config/types/runtime/testdata/oom.yaml b/pkg/machinery/config/types/runtime/testdata/oom.yaml
@@ -1,7 +1,8 @@
 apiVersion: v1alpha1
 kind: OOMConfig
 triggerExpression: |-
-    multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 ||
+    multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 &&
+    multiply_qos_vectors(qos_memory_full_avg10, {System: 1.0, Podruntime: 1.0}) > 5.0 ||
     memory_full_avg10 > 75.0 && time_since_trigger > duration("10s")
 cgroupRankingExpression: 'memory_max.hasValue() ? 0.0 : ({Besteffort: 1.0, Burstable: 0.5, Guaranteed: 0.0, Podruntime: 0.0, System: 0.0}[class] * double(memory_current.orValue(0u)))'
 sampleInterval: 100ms
diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -1320,7 +1320,8 @@ const (
 	ContainerMarkerFilePath = "/usr/etc/in-container"
 
 	// DefaultOOMTriggerExpression is the default CEL expression used to determine whether to trigger OOM.
-	DefaultOOMTriggerExpression = `(multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0) ||
+	DefaultOOMTriggerExpression = `(multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 &&
+	     multiply_qos_vectors(qos_memory_full_avg10, {System: 1.0, Podruntime: 1.0}) > 5.0) ||
 		(memory_full_avg10 > 75.0 && time_since_trigger > duration("10s"))`
 
 	// DefaultOOMCgroupRankingExpression is the default CEL expression used to rank cgroups for OOM killer.
diff --git a/website/content/v1.12/reference/configuration/runtime/oomconfig.md b/website/content/v1.12/reference/configuration/runtime/oomconfig.md
@@ -17,7 +17,8 @@ title: OOMConfig
 apiVersion: v1alpha1
 kind: OOMConfig
 triggerExpression: |- # This expression defines when to trigger OOM action.
-    multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 ||
+    multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 &&
+    multiply_qos_vectors(qos_memory_full_avg10, {System: 1.0, Podruntime: 1.0}) > 5.0 ||
     memory_full_avg10 > 75.0 && time_since_trigger > duration("10s")
 cgroupRankingExpression: 'memory_max.hasValue() ? 0.0 : ({Besteffort: 1.0, Burstable: 0.5, Guaranteed: 0.0, Podruntime: 0.0, System: 0.0}[class] * double(memory_current.orValue(0u)))' # This expression defines how to rank cgroups for OOM handler.
 sampleInterval: 100ms # How often should the trigger expression be evaluated.
