Use information from cgroup (if applicable) to adjust memory tracker

azat · azat · commit 80888be41a41 · 2025-07-18T15:45:36.000+02:00
Right now `memory_worker_correct_memory_tracker` always uses information
from jemalloc to update the `MemoryTracking`, but, this may be not good
enough in some cases (i.e. when server requested more memory then it
use) and may lead to `MEMORY_LIMIT_EXCEEDED` errors with
`MemoryTracking` &gt; `RSS`.

So use information from cgroup if applicable, and if not, use
information from `jemalloc.allocated`.
diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp
@@ -9,6 +9,7 @@
 #include <Common/ProfileEvents.h>
 #include <Common/formatReadable.h>
 #include <Common/logger_useful.h>
+#include <Common/setThreadName.h>
 
 #include <fmt/ranges.h>
 
@@ -303,6 +304,7 @@ uint64_t MemoryWorker::getMemoryUsage()
             return cgroups_reader != nullptr ? cgroups_reader->readMemoryUsage() : 0;
         case MemoryUsageSource::Jemalloc:
 #if USE_JEMALLOC
+            epoch_mib.setValue(0);
             return resident_mib.getValue();
 #else
             return 0;
@@ -314,6 +316,8 @@ uint64_t MemoryWorker::getMemoryUsage()
 
 void MemoryWorker::backgroundThread()
 {
+    setThreadName("MemoryWorker");
+
     std::chrono::milliseconds chrono_period_ms{period_ms};
     [[maybe_unused]] bool first_run = true;
     std::unique_lock lock(mutex);
@@ -325,11 +329,6 @@ void MemoryWorker::backgroundThread()
 
         Stopwatch total_watch;
 
-#if USE_JEMALLOC
-        if (source == MemoryUsageSource::Jemalloc)
-            epoch_mib.setValue(0);
-#endif
-
         Int64 resident = getMemoryUsage();
         MemoryTracker::updateRSS(resident);
 
@@ -350,19 +349,9 @@ void MemoryWorker::backgroundThread()
         ///  - MemoryTracker stores a negative value
         ///  - `correct_tracker` is set to true
         if (unlikely(first_run || total_memory_tracker.get() < 0))
-        {
-            if (source != MemoryUsageSource::Jemalloc)
-                epoch_mib.setValue(0);
-
-            MemoryTracker::updateAllocated(allocated_mib.getValue(), /*log_change=*/true);
-        }
+            MemoryTracker::updateAllocated(resident, /*log_change=*/true);
         else if (correct_tracker)
-        {
-            if (source != MemoryUsageSource::Jemalloc)
-                epoch_mib.setValue(0);
-
-            MemoryTracker::updateAllocated(allocated_mib.getValue(), /*log_change=*/false);
-        }
+            MemoryTracker::updateAllocated(resident, /*log_change=*/false);
 #else
         /// we don't update in the first run if we don't have jemalloc
         /// because we can only use resident memory information
@@ -371,7 +360,6 @@ void MemoryWorker::backgroundThread()
         /// before MemoryTracker initialization
         if (unlikely(total_memory_tracker.get() < 0) || correct_tracker)
             MemoryTracker::updateAllocated(resident, /*log_change=*/false);
-
 #endif
 
         ProfileEvents::increment(ProfileEvents::MemoryWorkerRun);
diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h
@@ -77,7 +77,6 @@ class MemoryWorker
 #if USE_JEMALLOC
     JemallocMibCache<uint64_t> epoch_mib{"epoch"};
     JemallocMibCache<size_t> resident_mib{"stats.resident"};
-    JemallocMibCache<size_t> allocated_mib{"stats.allocated"};
 
 #define STRINGIFY_HELPER(x) #x
 #define STRINGIFY(x) STRINGIFY_HELPER(x)
