tst-rec-dlopen: Use interposed malloc instead of hooks

This avoids use of the deprecated hook variables.

ChangeLog

@@ -1,3 +1,12 @@
+2016-06-05  Florian Weimer  <fweimer@redhat.com>
+
+	* dlfcn/tst-rec-dlopen.c (custom_malloc_hook, old_malloc_hook):
+	Remove.
+	(call_function, interposed_malloc_called): New variables.
+	(malloc): New function.
+	(do_test): Do not change malloc hooks.  Initialize malloc.  Set
+	and clear call_function as needed.
+
 2016-06-04  Andreas Schwab  <schwab@linux-m68k.org>
 
 	* include/wchar.h (__wmemset): Use __typeof.

dlfcn/tst-rec-dlopen.c

@@ -18,8 +18,11 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <malloc.h>
 #include <dlfcn.h>
+#include <stdbool.h>
+#include <stdalign.h>
+#include <sys/mman.h>
+#include <unistd.h>
 
 #define DSO "moddummy1.so"
 #define FUNC "dummy1"
@@ -30,12 +33,6 @@
 /* Result of the called function.  */
 int func_result;
 
-/* Prototype for my hook.  */
-void *custom_malloc_hook (size_t, const void *);
-
-/* Pointer to old malloc hooks.  */
-void *(*old_malloc_hook) (size_t, const void *);
-
 /* Call function func_name in DSO dso_name via dlopen.  */
 void
 call_func (const char *dso_name, const char *func_name)
@@ -81,29 +78,67 @@ call_func (const char *dso_name, const char *func_name)
 
 }
 
-/* Empty hook that does nothing.  */
+/* If true, call another function from malloc.  */
+static bool call_function;
+
+/* Set to true to indicate that the interposed malloc was called.  */
+static bool interposed_malloc_called;
+
+/* Interposed malloc which optionally calls another function.  */
 void *
-custom_malloc_hook (size_t size, const void *caller)
+malloc (size_t size)
 {
-  void *result;
-  /* Restore old hooks.  */
-  __malloc_hook = old_malloc_hook;
-  /* First call a function in another library via dlopen.  */
+  interposed_malloc_called = true;
+  static void *(*original_malloc) (size_t);
+
+  if (original_malloc == NULL)
+    {
+      static bool in_initialization;
+      if (in_initialization)
+	{
+	  const char *message
+	    = "error: malloc called recursively during initialization\n";
+	  (void) write (STDOUT_FILENO, message, strlen (message));
+	  _exit (2);
+	}
+      in_initialization = true;
+
+      original_malloc
+	= (__typeof (original_malloc)) dlsym (RTLD_NEXT, "malloc");
+      if (original_malloc == NULL)
+	{
+	  const char *message
+	    = "error: dlsym for malloc failed\n";
+	  (void) write (STDOUT_FILENO, message, strlen (message));
+	  _exit (2);
+	}
+    }
+
+  if (call_function)
+    {
+      call_function = false;
       call_func (DSO1, FUNC1);
-  /* Called recursively.  */
-  result = malloc (size);
-  /* Restore new hooks.  */
-  __malloc_hook = custom_malloc_hook;
-  return result;
+      call_function = true;
+    }
+  return original_malloc (size);
 }
 
 static int
 do_test (void)
 {
-  /* Save old hook.  */
-  old_malloc_hook = __malloc_hook;
-  /* Install new hook.  */
-  __malloc_hook = custom_malloc_hook;
+  /* Ensure initialization.  */
+  {
+    void *volatile ptr = malloc (1);
+    free (ptr);
+  }
+
+  if (!interposed_malloc_called)
+    {
+      printf ("error: interposed malloc not called during initialization\n");
+      return 1;
+    }
+
+  call_function = true;
 
   /* Bug 17702 fixes two things:
        * A recursive dlopen unmapping the ld.so.cache.
@@ -116,8 +151,7 @@ do_test (void)
      will abort because of the assert described in detail below.  */
   call_func (DSO, FUNC);
 
-  /* Restore old hook.  */
-  __malloc_hook = old_malloc_hook;
+  call_function = false;
 
   /* The function dummy2() is called by the malloc hook. Check to
      see that it was called. This ensures the second recursive