tcmalloc: switch the interface.

Previously, we used function pointers. Fun for me to learn about, icky
to actually have!
Now we use our own wrapper functions with two implementations -- one
for with tcmalloc and one without. Make those programs which
are tcmalloc-aware build with the appropriate implementation source
at compile-time, but leave the wrapper function stubs in
no matter what.

While we're at it, implement two of the "MallocExtension" calls in
the OSD.

Signed-off-by: Greg Farnum <gregory.farnum@dreamhost.com>

src/Makefile.am

@@ -129,12 +129,17 @@ if WITH_TCMALLOC
 tcmalloc_safety_flags = -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
 cosd_LDADD += -ltcmalloc
 cosd_CXXFLAGS += ${tcmalloc_safety_flags}
+cosd_SOURCES += perfglue/heap_profiler.cc
 cmds_LDADD += -ltcmalloc
 cmds_CXXFLAGS += ${tcmalloc_safety_flags}
+cmds_SOURCES += perfglue/heap_profiler.cc
 if WITH_FUSE
 cfuse_LDADD += -ltcmalloc
 cfuse_CXXFLAGS += ${tcmalloc_safety_flags}
 endif #WITH_FUSE
+else
+cosd_SOURCES += perfglue/disabled_heap_profiler.cc
+cmds_SOURCES += perfglue/disabled_heap_profiler.cc
 endif # WITH_TCMALLOC
 
 # debug targets?
@@ -561,6 +566,7 @@ else
 libcommon_files += perfglue/disabled_stubs.cc
 endif
 
+
 libcrush_a_SOURCES = \
 	crush/builder.c \
 	crush/mapper.c \
@@ -974,6 +980,7 @@ noinst_HEADERS = \
         osdc/ObjectCacher.h\
         osdc/Objecter.h\
         perfglue/cpu_profiler.h\
+        perfglue/heap_profiler.h\
 	rgw/rgw_access.h\
 	rgw/rgw_acl.h\
 	rgw/rgw_fs.h\

src/cmds.cc

@@ -65,13 +65,6 @@ int main(int argc, const char **argv)
   argv_to_vec(argc, argv, args);
   env_to_vec(args);
 
-#ifdef HAVE_LIBTCMALLOC
-  g_conf.profiler_start = HeapProfilerStart;
-  g_conf.profiler_running = IsHeapProfilerRunning;
-  g_conf.profiler_stop = HeapProfilerStop;
-  g_conf.profiler_dump = HeapProfilerDump;
-  g_conf.tcmalloc_have = true;
-#endif //HAVE_LIBTCMALLOC
   common_init(args, "mds", STARTUP_FLAG_INIT_KEYS | STARTUP_FLAG_DAEMON);
 
   // mds specific args

src/common/common_init.cc

@@ -168,20 +168,6 @@ void common_init(std::vector<const char*>& args, const char *module_type, int fl
   parse_config_options(args);
   install_standard_sighandlers();
 
-#ifdef HAVE_LIBTCMALLOC
-  if (g_conf.tcmalloc_profiler_run && g_conf.tcmalloc_have) {
-    char profile_name[PATH_MAX];
-    sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name);
-    char *val = new char[sizeof(int)*8+1];
-    sprintf(val, "%i", g_conf.profiler_allocation_interval);
-    setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval);
-    sprintf(val, "%i", g_conf.profiler_highwater_interval);
-    setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval);
-    generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl;
-    g_conf.profiler_start(profile_name);
-  }
-#endif //HAVE_LIBTCMALLOC
-
   if (flags & STARTUP_FLAG_INIT_KEYS)  {
     if (is_supported_auth(CEPH_AUTH_CEPHX))
       keyring_init(g_conf.keyring);

src/common/config.h

@@ -31,10 +31,6 @@ extern struct ceph_file_layout g_default_file_layout;
 
 #include "msg/msg_types.h"
 
-#ifdef HAVE_LIBTCMALLOC
-#include <google/heap-profiler.h>
-#endif //HAVE_LIBTCMALLOC
-
 struct EntityName;
 
 enum log_to_stderr_t {
@@ -68,12 +64,7 @@ struct md_config_t {
   bool daemonize;
 
   //profiling
-  bool tcmalloc_have;
   bool tcmalloc_profiler_run;
-  void (*profiler_start)(const char*);
-  bool (*profiler_running)();
-  void (*profiler_stop)();
-  void (*profiler_dump)(const char*);
   int profiler_allocation_interval;
   int profiler_highwater_interval;
 

src/cosd.cc

@@ -37,6 +37,8 @@ using namespace std;
 #include "include/color.h"
 #include "common/errno.h"
 
+#include "perfglue/heap_profiler.h"
+
 void usage() 
 {
   derr << "usage: cosd -i osdid [--osd-data=path] [--osd-journal=path] "
@@ -62,14 +64,8 @@ int main(int argc, const char **argv)
     } 
   }
 
-#ifdef HAVE_LIBTCMALLOC
-  g_conf.profiler_start = HeapProfilerStart;
-  g_conf.profiler_running = IsHeapProfilerRunning;
-  g_conf.profiler_stop = HeapProfilerStop;
-  g_conf.profiler_dump = HeapProfilerDump;
-  g_conf.tcmalloc_have = true;
-#endif //HAVE_LIBTCMALLOC
   common_init(args, "osd", startup_flags);
+  ceph_heap_profiler_init();
 
   // osd specific args
   bool mkfs = false;

src/mds/MDS.cc

@@ -71,6 +71,7 @@
 #include "common/config.h"
 
 #include "perfglue/cpu_profiler.h"
+#include "perfglue/heap_profiler.h"
 
 
 #define DOUT_SUBSYS mds
@@ -791,23 +792,20 @@ void MDS::handle_command(MMonCommand *m)
     clog.info() << g_conf.name << " set heap variables from current config\n";
   }
   else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_start") {
-    char location[PATH_MAX];
-    snprintf(location, sizeof(location),
-	     "%s/%s", g_conf.log_dir, g_conf.name);
-    g_conf.profiler_start(location);
+    ceph_heap_profiler_start();
     clog.info() << g_conf.name << " started profiler\n";
   }
   else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_stop") {
-    g_conf.profiler_stop();
+    ceph_heap_profiler_stop();
     clog.info() << g_conf.name << " stopped profiler\n";
   }
   else if (m->cmd.size() == 1 && m->cmd[0] == "heap_profiler_dump"){
-    if (g_conf.tcmalloc_have) {
-      if (!g_conf.profiler_running()) {
+    if (ceph_using_tcmalloc()) {
+      if (!ceph_heap_profiler_running()) {
         clog.info() << g_conf.name << " can't dump heap: profiler not running\n";
       } else {
         clog.info() << g_conf.name << " dumping heap profile now\n";
-        g_conf.profiler_dump("admin request");
+        ceph_heap_profiler_dump("admin request");
       }
     } else {
       clog.info() << "tcmalloc not enabled, can't use profiler\n";

src/osd/OSD.cc

@@ -82,6 +82,7 @@
 #include "common/LogClient.h"
 #include "common/safe_io.h"
 #include "perfglue/cpu_profiler.h"
+#include "perfglue/heap_profiler.h"
 
 #include "common/ClassHandler.h"
 
@@ -2083,12 +2084,15 @@ void OSD::handle_command(MMonCommand *m)
   } else if (m->cmd.size() == 2 && m->cmd[0] == "logger" && m->cmd[1] == "reopen") {
     logger_reopen_all();
   } else if (m->cmd.size() == 1 && m->cmd[0] == "heapdump") {
-    if (g_conf.tcmalloc_have) {
-      if (!g_conf.profiler_running()) {
+    if (ceph_using_tcmalloc()) {
+      if (!ceph_heap_profiler_running()) {
 	clog.info() << "can't dump heap: profiler not running\n";
       } else {
-        clog.info() << g_conf.name << "dumping heap profile now\n";
-        g_conf.profiler_dump("admin request");
+	char *heap_stats = new char[1024];
+	ceph_heap_profiler_stats(heap_stats, 1024);
+        clog.info() << g_conf.name << "dumping heap profile now.\n"
+		    << heap_stats << std::endl;
+        ceph_heap_profiler_dump("admin request");
       }
     } else {
       clog.info() << g_conf.name << " does not have tcmalloc, "
@@ -2102,17 +2106,19 @@ void OSD::handle_command(MMonCommand *m)
     setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval);
     clog.info() << g_conf.name << " set heap variables from current config";
   } else if (m->cmd.size() == 1 && m->cmd[0] == "start_profiler") {
-    char location[PATH_MAX];
-    snprintf(location, sizeof(location),
-	     "%s/%s", g_conf.log_dir, g_conf.name);
-    g_conf.profiler_start(location);
-    clog.info() << g_conf.name << " started profiler with output "
-      << location << "\n";
+    ceph_heap_profiler_start();
+    clog.info() << g_conf.name << " started profiler \n";
   } else if (m->cmd.size() == 1 && m->cmd[0] == "stop_profiler") {
-    g_conf.profiler_stop();
+    ceph_heap_profiler_stop();
     clog.info() << g_conf.name << " stopped profiler\n";
-  }
-  else if (m->cmd.size() > 1 && m->cmd[0] == "debug") {
+  } else if (m->cmd.size() == 1 && m->cmd[0] == "release_heap") {
+    if (ceph_using_tcmalloc()) {
+      ceph_heap_release_free_memory();
+      clog.info() << g_conf.name << " releasing free RAM back to system.\n";
+    } else {
+      clog.warn() << "can't release RAM: not using tcmalloc!";
+    }
+  } else if (m->cmd.size() > 1 && m->cmd[0] == "debug") {
     if (m->cmd.size() == 3 && m->cmd[1] == "dump_missing") {
       const string &file_name(m->cmd[2]);
       std::ofstream fout(file_name.c_str());

src/perfglue/disabled_heap_profiler.cc

@@ -0,0 +1,30 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+#include "heap_profiler.h"
+
+bool ceph_using_tcmalloc() { return false; }
+
+void ceph_heap_profiler_init() { return; }
+
+void ceph_heap_profiler_stats(char *buf, int length) { return; }
+
+void ceph_heap_release_free_memory() { return; }
+
+bool ceph_heap_profiler_running() { return false; }
+
+void ceph_heap_profiler_start() { return; }
+
+void ceph_heap_profiler_stop() { return; }
+
+void ceph_heap_profiler_dump(const char *reason) { return; }

src/perfglue/heap_profiler.cc

@@ -0,0 +1,75 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#include <google/heap-profiler.h>
+#include <google/malloc_extension.h>
+#include "heap_profiler.h"
+
+bool ceph_using_tcmalloc()
+{
+  return true;
+}
+
+void ceph_heap_profiler_init()
+{
+  char profile_name[PATH_MAX];
+  sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name);
+  char *val = new char[sizeof(int)*8+1];
+  sprintf(val, "%i", g_conf.profiler_allocation_interval);
+  setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval);
+  sprintf(val, "%i", g_conf.profiler_highwater_interval);
+  setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval);
+  if (g_conf.tcmalloc_profiler_run) {
+    generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl;
+    HeapProfilerStart(profile_name);
+  }
+}
+
+void ceph_heap_profiler_stats(char *buf, int length)
+{
+  MallocExtension::instance()->GetStats(buf, length);
+}
+
+void ceph_heap_release_free_memory()
+{
+  MallocExtension::instance()->ReleaseFreeMemory();
+}
+
+bool ceph_heap_profiler_running()
+{
+  return IsHeapProfilerRunning();
+}
+
+void ceph_heap_profiler_start()
+{
+  char profile_name[PATH_MAX];
+  sprintf(profile_name, "%s/%s", g_conf.log_dir, g_conf.name);
+  char *val = new char[sizeof(int)*8+1];
+  sprintf(val, "%i", g_conf.profiler_allocation_interval);
+  setenv("HEAP_PROFILE_ALLOCATION_INTERVAL", val, g_conf.profiler_allocation_interval);
+  sprintf(val, "%i", g_conf.profiler_highwater_interval);
+  setenv("HEAP_PROFILE_INUSE_INTERVAL", val, g_conf.profiler_highwater_interval);
+  generic_dout(0) << "turning on heap profiler with prefix " << profile_name << dendl;
+  HeapProfilerStart(profile_name);
+}
+
+void ceph_heap_profiler_stop()
+{
+  HeapProfilerStop();
+}
+
+void ceph_heap_profiler_dump(const char *reason)
+{
+  HeapProfilerDump(reason);
+}

src/perfglue/heap_profiler.h

@@ -0,0 +1,44 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network/Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ */
+#ifndef HEAP_PROFILER_H_
+#define HEAP_PROFILER_H_
+
+#include "common/config.h"
+
+/*
+ * Ceph glue for the Google perftools heap profiler, included
+ * as part of tcmalloc. This replaces ugly function pointers
+ * and #ifdef hacks!
+ */
+bool ceph_using_tcmalloc();
+
+/*
+ * Configure the heap profiler
+ */
+void ceph_heap_profiler_init();
+
+void ceph_heap_profiler_stats(char *buf, int length);
+
+void ceph_heap_release_free_memory();
+
+bool ceph_heap_profiler_running();
+
+void ceph_heap_profiler_start();
+
+void ceph_heap_profiler_stop();
+
+void ceph_heap_profiler_dump(const char *reason);
+
+
+
+#endif /* HEAP_PROFILER_H_ */