diff --git a/elf/dl-sym.c b/elf/dl-sym.c
index 286cf7e..b133850 100644
--- a/elf/dl-sym.c
+++ b/elf/dl-sym.c
@@ -80,6 +80,18 @@ call_dl_lookup (void *ptr)
 					args->flags, NULL);
 }
 
+/* Return the link map containing the caller address.  */
+static inline struct link_map *
+find_caller_link_map (ElfW(Addr) caller)
+{
+  struct link_map *l = _dl_find_dso_for_object (caller);
+  if (l != NULL)
+    return l;
+  else
+    /* If the address is not recognized the call comes from the main
+       program (we hope).  */
+    return GL(dl_ns)[LM_ID_BASE]._ns_loaded;
+}
 
 static void *
 do_sym (void *handle, const char *name, void *who,
@@ -89,13 +101,13 @@ do_sym (void *handle, const char *name, void *who,
   lookup_t result;
   ElfW(Addr) caller = (ElfW(Addr)) who;
 
-  struct link_map *l = _dl_find_dso_for_object (caller);
-  /* If the address is not recognized the call comes from the main
-     program (we hope).  */
-  struct link_map *match = l ? l : GL(dl_ns)[LM_ID_BASE]._ns_loaded;
+  /* Link map of the caller if needed.  */
+  struct link_map *match = NULL;
 
   if (handle == RTLD_DEFAULT)
     {
+      match = find_caller_link_map (caller);
+
       /* Search the global scope.  We have the simple case where
 	 we look up in the scope of an object which was part of
 	 the initial binary.  And then the more complex part
@@ -128,6 +140,8 @@ do_sym (void *handle, const char *name, void *who,
     }
   else if (handle == RTLD_NEXT)
     {
+      match = find_caller_link_map (caller);
+
       if (__glibc_unlikely (match == GL(dl_ns)[LM_ID_BASE]._ns_loaded))
 	{
 	  if (match == NULL
@@ -187,6 +201,9 @@ RTLD_NEXT used in code not dynamically loaded"));
 	  unsigned int ndx = (ref - (ElfW(Sym) *) D_PTR (result,
 							 l_info[DT_SYMTAB]));
 
+	  if (match == NULL)
+	    match = find_caller_link_map (caller);
+
 	  if ((match->l_audit_any_plt | result->l_audit_any_plt) != 0)
 	    {
 	      unsigned int altvalue = 0;