Implement utility function purge_all_jemalloc_arenas() to be used l…

…ater by fast secondary index creation

Summary: This change is complimentary to https://reviews.facebook.net/D61779 and can be landed either before or after D61779. However, once D61779 lands then at the end of the `ha_rocksdb::inplace_populate_sk(...)` we'll need to call `purge_all_jemalloc_arenas()` to work around the problems which happen when `CREATE INDEX` statements are applied to large (and I mean really large :-)) amounts of data.

Test Plan:
- Usual Sandcastle testing.
- Manual performance testing internally to make sure that there'll be no memory leaks and `jemalloc` won't hold on to memory till thread state changes.

Reviewers: yoshinorim, alexyang

Reviewed By: alexyang

Subscribers: webscalesql-eng

Differential Revision: https://reviews.facebook.net/D63723

libmysqld/examples/CMakeLists.txt

@@ -46,7 +46,6 @@ ENDIF()
 MYSQL_ADD_EXECUTABLE(mysqltest_embedded ../../client/mysqltest.cc)
 TARGET_LINK_LIBRARIES(mysqltest_embedded mysqlserver)
 
-
 IF(CMAKE_GENERATOR MATCHES "Xcode")
 # It does not seem possible to tell Xcode the resulting target might need
 # to be linked with C++ runtime. The project needs to have at least one C++
@@ -79,3 +78,31 @@ IF(NOT INSTALL_LAYOUT MATCHES "SVR4")
   SET_TARGET_PROPERTIES(mysql_client_test_embedded PROPERTIES ENABLE_EXPORTS TRUE)
   ENDIF()
 ENDIF()
+
+#
+# Necessary to make sure that we can use the jemalloc API calls. We need to do
+# this for every binary which which will link to MyRocks. The trick here is to
+# use the linker flags which already get set in WITH_MYSQLD_LDFLAGS because
+# `mysqld` itself sets options for jemalloc. This way we don't have to define
+# a separate flag for every binary.
+#
+GET_TARGET_PROPERTY(mysqltest_embedded LINK_FLAGS PREV_LINK_FLAGS)
+IF(NOT PREV_LINK_FLAGS)
+  SET(PREV_LINK_FLAGS)
+ENDIF()
+SET_TARGET_PROPERTIES(mysqltest_embedded PROPERTIES LINK_FLAGS
+"${PREV_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}")
+
+GET_TARGET_PROPERTY(mysql_embedded LINK_FLAGS PREV_LINK_FLAGS)
+IF(NOT PREV_LINK_FLAGS)
+  SET(PREV_LINK_FLAGS)
+ENDIF()
+SET_TARGET_PROPERTIES(mysql_embedded PROPERTIES LINK_FLAGS
+"${PREV_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}")
+
+GET_TARGET_PROPERTY(mysql_client_test_embedded LINK_FLAGS PREV_LINK_FLAGS)
+IF(NOT PREV_LINK_FLAGS)
+  SET(PREV_LINK_FLAGS)
+ENDIF()
+SET_TARGET_PROPERTIES(mysql_client_test_embedded PROPERTIES LINK_FLAGS
+"${PREV_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}")

storage/rocksdb/rdb_utils.h

@@ -24,6 +24,10 @@
 /* RocksDB header files */
 #include "rocksdb/slice.h"
 
+#ifdef HAVE_JEMALLOC
+  #include <jemalloc/jemalloc.h>
+#endif
+
 namespace myrocks {
 
 /*
@@ -127,6 +131,40 @@ inline const uchar* rdb_slice_to_uchar_ptr(const rocksdb::Slice *item)
   return reinterpret_cast<const uchar*>(item->data());
 }
 
+/*
+  Call this function in cases when you can't rely on garbage collector and need
+  to explicitly purge all unused dirty pages. This should be a relatively rare
+  scenario for cases where it has been verified that this intervention has
+  noticeable benefits.
+*/
+inline int purge_all_jemalloc_arenas()
+{
+#ifdef HAVE_JEMALLOC
+  unsigned narenas = 0;
+  size_t sz = sizeof(unsigned);
+  char name[25] = { 0 };
+
+  // Get the number of arenas first. Please see `jemalloc` documentation for
+  // all the various options.
+  int result = mallctl("arenas.narenas", &narenas, &sz, nullptr, 0);
+
+  // `mallctl` returns 0 on success and we really want caller to know if all the
+  // trickery actually works.
+  if (result) {
+    return result;
+  }
+
+  // Form the command to be passed to `mallctl` and purge all the unused dirty
+  // pages.
+  snprintf(name, sizeof(name) / sizeof(char), "arena.%d.purge", narenas);
+  result = mallctl(name, nullptr, nullptr, nullptr, 0);
+
+  return result;
+#else
+  return EXIT_SUCCESS;
+#endif
+}
+
 /*
   Helper functions to parse strings.
 */

storage/rocksdb/unittest/CMakeLists.txt

@@ -11,4 +11,12 @@ IF (WITH_ROCKSDB_SE_STORAGE_ENGINE)
           test_properties_collector.cc
           )
   TARGET_LINK_LIBRARIES(test_properties_collector mysqlserver)
+
+  # Necessary to make sure that we can use the jemalloc API calls.
+  GET_TARGET_PROPERTY(mysql_embedded LINK_FLAGS PREV_LINK_FLAGS)
+  IF(NOT PREV_LINK_FLAGS)
+    SET(PREV_LINK_FLAGS)
+  ENDIF()
+  SET_TARGET_PROPERTIES(test_properties_collector PROPERTIES LINK_FLAGS
+  "${PREV_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}")
 ENDIF()