Disable obj store pages in raylet coredump (ray-project#30592)

Disable object store pages from appearing in Raylet coredumps using madvise(MADV_DONTDUMP). See ray-project#30150 for more context.
Signed-off-by: Cade Daniel <[email protected]>

src/ray/common/ray_config_def.h

@@ -715,6 +715,7 @@ RAY_CONFIG(int64_t, health_check_period_ms, 3000)
 RAY_CONFIG(int64_t, health_check_timeout_ms, 10000)
 RAY_CONFIG(int64_t, health_check_failure_threshold, 5)
 
-/// Use madvise to prevent worker coredump from including the mapped plasma pages
-/// in the worker processes.
+/// Use madvise to prevent worker/raylet coredumps from including
+/// the mapped plasma pages.
 RAY_CONFIG(bool, worker_core_dump_exclude_plasma_store, true)
+RAY_CONFIG(bool, raylet_core_dump_exclude_plasma_store, true)

src/ray/object_manager/plasma/dlmalloc.cc

@@ -219,6 +219,21 @@ void create_and_mmap_buffer(int64_t size, void **pointer, int *fd) {
     initial_region_ptr = static_cast<char *>(*pointer);
     initial_region_size = size;
   }
+
+#ifdef __linux__
+  if (RayConfig::instance().raylet_core_dump_exclude_plasma_store()) {
+    int rval = madvise(initial_region_ptr, initial_region_size, MADV_DONTDUMP);
+    if (rval) {
+      RAY_LOG(WARNING) << "madvise(MADV_DONTDUMP) call failed: " << rval << ", "
+                       << strerror(errno);
+    } else {
+      RAY_LOG(DEBUG) << "madvise(MADV_DONTDUMP) call succeeded.";
+    }
+  } else {
+    RAY_LOG(DEBUG) << "worker_core_dump_exclude_plasma_store disabled, raylet coredumps "
+                      "will contain the object store mappings.";
+  }
+#endif /* __linux__ */
 }
 
 #endif

src/ray/object_manager/plasma/shared_memory.cc

@@ -43,8 +43,7 @@ ClientMmapTableEntry::ClientMmapTableEntry(MEMFD_TYPE fd, int64_t map_size)
 void ClientMmapTableEntry::MaybeMadviseDontdump() {
   if (!RayConfig::instance().worker_core_dump_exclude_plasma_store()) {
     RAY_LOG(DEBUG) << "worker_core_dump_exclude_plasma_store disabled, worker coredumps "
-                      "will contain "
-                   << "the object store mappings.";
+                      "will contain the object store mappings.";
     return;
   }