qemu/roms/ipxe/src/arch/x86/include/valgrind/valgrind.h

   1 /* -*- c -*-
   2    ----------------------------------------------------------------
   3
   4    Notice that the following BSD-style license applies to this one
   5    file (valgrind.h) only.  The rest of Valgrind is licensed under the
   6    terms of the GNU General Public License, version 2, unless
   7    otherwise indicated.  See the COPYING file in the source
   8    distribution for details.
   9
  10    ----------------------------------------------------------------
  11
  12    This file is part of Valgrind, a dynamic binary instrumentation
  13    framework.
  14
  15    Copyright (C) 2000-2010 Julian Seward.  All rights reserved.
  16
  17    Redistribution and use in source and binary forms, with or without
  18    modification, are permitted provided that the following conditions
  19    are met:
  20
  21    1. Redistributions of source code must retain the above copyright
  22       notice, this list of conditions and the following disclaimer.
  23
  24    2. The origin of this software must not be misrepresented; you must
  25       not claim that you wrote the original software.  If you use this
  26       software in a product, an acknowledgment in the product
  27       documentation would be appreciated but is not required.
  28
  29    3. Altered source versions must be plainly marked as such, and must
  30       not be misrepresented as being the original software.
  31
  32    4. The name of the author may not be used to endorse or promote
  33       products derived from this software without specific prior written
  34       permission.
  35
  36    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  37    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  38    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  39    ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  40    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  41    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  42    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  43    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  44    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  45    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  46    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47
  48    ----------------------------------------------------------------
  49
  50    Notice that the above BSD-style license applies to this one file
  51    (valgrind.h) only.  The entire rest of Valgrind is licensed under
  52    the terms of the GNU General Public License, version 2.  See the
  53    COPYING file in the source distribution for details.
  54
  55    ----------------------------------------------------------------
  56 */
  57
  58
  59 /* This file is for inclusion into client (your!) code.
  60
  61    You can use these macros to manipulate and query Valgrind's
  62    execution inside your own programs.
  63
  64    The resulting executables will still run without Valgrind, just a
  65    little bit more slowly than they otherwise would, but otherwise
  66    unchanged.  When not running on valgrind, each client request
  67    consumes very few (eg. 7) instructions, so the resulting performance
  68    loss is negligible unless you plan to execute client requests
  69    millions of times per second.  Nevertheless, if that is still a
  70    problem, you can compile with the NVALGRIND symbol defined (gcc
  71    -DNVALGRIND) so that client requests are not even compiled in.  */
  72
  73 #ifndef __VALGRIND_H
  74 #define __VALGRIND_H
  75
  76 FILE_LICENCE ( BSD3 );
  77
  78
  79 /* ------------------------------------------------------------------ */
  80 /* VERSION NUMBER OF VALGRIND                                         */
  81 /* ------------------------------------------------------------------ */
  82
  83 /* Specify Valgrind's version number, so that user code can
  84    conditionally compile based on our version number.  Note that these
  85    were introduced at version 3.6 and so do not exist in version 3.5
  86    or earlier.  The recommended way to use them to check for "version
  87    X.Y or later" is (eg)
  88
  89 #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
  90     && (__VALGRIND_MAJOR__ > 3                                   \
  91         || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
  92 */
  93 #define __VALGRIND_MAJOR__    3
  94 #define __VALGRIND_MINOR__    6
  95
  96
  97 #include <stdarg.h>
  98
  99 /* Nb: this file might be included in a file compiled with -ansi.  So
 100    we can't use C++ style "//" comments nor the "asm" keyword (instead
 101    use "__asm__"). */
 102
 103 /* Derive some tags indicating what the target platform is.  Note
 104    that in this file we're using the compiler's CPP symbols for
 105    identifying architectures, which are different to the ones we use
 106    within the rest of Valgrind.  Note, __powerpc__ is active for both
 107    32 and 64-bit PPC, whereas __powerpc64__ is only active for the
 108    latter (on Linux, that is).
 109
 110    Misc note: how to find out what's predefined in gcc by default:
 111    gcc -Wp,-dM somefile.c
 112 */
 113 #undef PLAT_ppc64_aix5
 114 #undef PLAT_ppc32_aix5
 115 #undef PLAT_x86_darwin
 116 #undef PLAT_amd64_darwin
 117 #undef PLAT_x86_linux
 118 #undef PLAT_amd64_linux
 119 #undef PLAT_ppc32_linux
 120 #undef PLAT_ppc64_linux
 121 #undef PLAT_arm_linux
 122
 123 #if defined(_AIX) && defined(__64BIT__)
 124 #  define PLAT_ppc64_aix5 1
 125 #elif defined(_AIX) && !defined(__64BIT__)
 126 #  define PLAT_ppc32_aix5 1
 127 #elif defined(__APPLE__) && defined(__i386__)
 128 #  define PLAT_x86_darwin 1
 129 #elif defined(__APPLE__) && defined(__x86_64__)
 130 #  define PLAT_amd64_darwin 1
 131 #elif defined(__linux__) && defined(__i386__)
 132 #  define PLAT_x86_linux 1
 133 #elif defined(__linux__) && defined(__x86_64__)
 134 #  define PLAT_amd64_linux 1
 135 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
 136 #  define PLAT_ppc32_linux 1
 137 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__)
 138 #  define PLAT_ppc64_linux 1
 139 #elif defined(__linux__) && defined(__arm__)
 140 #  define PLAT_arm_linux 1
 141 #else
 142 /* If we're not compiling for our target platform, don't generate
 143    any inline asms.  */
 144 #  if !defined(NVALGRIND)
 145 #    define NVALGRIND 1
 146 #  endif
 147 #endif
 148
 149
 150 /* ------------------------------------------------------------------ */
 151 /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
 152 /* in here of use to end-users -- skip to the next section.           */
 153 /* ------------------------------------------------------------------ */
 154
 155 #if defined(NVALGRIND)
 156
 157 /* Define NVALGRIND to completely remove the Valgrind magic sequence
 158    from the compiled code (analogous to NDEBUG's effects on
 159    assert()) */
 160 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 161         _zzq_rlval, _zzq_default, _zzq_request,                   \
 162         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 163    {                                                              \
 164       (_zzq_rlval) = (_zzq_default);                              \
 165    }
 166
 167 #else  /* ! NVALGRIND */
 168
 169 /* The following defines the magic code sequences which the JITter
 170    spots and handles magically.  Don't look too closely at them as
 171    they will rot your brain.
 172
 173    The assembly code sequences for all architectures is in this one
 174    file.  This is because this file must be stand-alone, and we don't
 175    want to have multiple files.
 176
 177    For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
 178    value gets put in the return slot, so that everything works when
 179    this is executed not under Valgrind.  Args are passed in a memory
 180    block, and so there's no intrinsic limit to the number that could
 181    be passed, but it's currently five.
 182
 183    The macro args are:
 184       _zzq_rlval    result lvalue
 185       _zzq_default  default value (result returned when running on real CPU)
 186       _zzq_request  request code
 187       _zzq_arg1..5  request params
 188
 189    The other two macros are used to support function wrapping, and are
 190    a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
 191    guest's NRADDR pseudo-register and whatever other information is
 192    needed to safely run the call original from the wrapper: on
 193    ppc64-linux, the R2 value at the divert point is also needed.  This
 194    information is abstracted into a user-visible type, OrigFn.
 195
 196    VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
 197    guest, but guarantees that the branch instruction will not be
 198    redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
 199    branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
 200    complete inline asm, since it needs to be combined with more magic
 201    inline asm stuff to be useful.
 202 */
 203
 204 /* ------------------------- x86-{linux,darwin} ---------------- */
 205
 206 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 207
 208 typedef
 209    struct {
 210       unsigned int nraddr; /* where's the code? */
 211    }
 212    OrigFn;
 213
 214 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 215                      "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
 216                      "roll $29, %%edi ; roll $19, %%edi\n\t"
 217
 218 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 219         _zzq_rlval, _zzq_default, _zzq_request,                   \
 220         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 221   { volatile unsigned int _zzq_args[6];                           \
 222     volatile unsigned int _zzq_result;                            \
 223     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 224     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 225     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 226     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 227     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 228     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 229     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 230                      /* %EDX = client_request ( %EAX ) */         \
 231                      "xchgl %%ebx,%%ebx"                          \
 232                      : "=d" (_zzq_result)                         \
 233                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 234                      : "cc", "memory"                             \
 235                     );                                            \
 236     _zzq_rlval = _zzq_result;                                     \
 237   }
 238
 239 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 240   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 241     volatile unsigned int __addr;                                 \
 242     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 243                      /* %EAX = guest_NRADDR */                    \
 244                      "xchgl %%ecx,%%ecx"                          \
 245                      : "=a" (__addr)                              \
 246                      :                                            \
 247                      : "cc", "memory"                             \
 248                     );                                            \
 249     _zzq_orig->nraddr = __addr;                                   \
 250   }
 251
 252 #define VALGRIND_CALL_NOREDIR_EAX                                 \
 253                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 254                      /* call-noredir *%EAX */                     \
 255                      "xchgl %%edx,%%edx\n\t"
 256 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
 257
 258 /* ------------------------ amd64-{linux,darwin} --------------- */
 259
 260 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
 261
 262 typedef
 263    struct {
 264       unsigned long long int nraddr; /* where's the code? */
 265    }
 266    OrigFn;
 267
 268 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 269                      "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
 270                      "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
 271
 272 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 273         _zzq_rlval, _zzq_default, _zzq_request,                   \
 274         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 275   { volatile unsigned long long int _zzq_args[6];                 \
 276     volatile unsigned long long int _zzq_result;                  \
 277     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 278     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 279     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 280     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 281     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 282     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 283     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 284                      /* %RDX = client_request ( %RAX ) */         \
 285                      "xchgq %%rbx,%%rbx"                          \
 286                      : "=d" (_zzq_result)                         \
 287                      : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
 288                      : "cc", "memory"                             \
 289                     );                                            \
 290     _zzq_rlval = _zzq_result;                                     \
 291   }
 292
 293 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 294   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 295     volatile unsigned long long int __addr;                       \
 296     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 297                      /* %RAX = guest_NRADDR */                    \
 298                      "xchgq %%rcx,%%rcx"                          \
 299                      : "=a" (__addr)                              \
 300                      :                                            \
 301                      : "cc", "memory"                             \
 302                     );                                            \
 303     _zzq_orig->nraddr = __addr;                                   \
 304   }
 305
 306 #define VALGRIND_CALL_NOREDIR_RAX                                 \
 307                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 308                      /* call-noredir *%RAX */                     \
 309                      "xchgq %%rdx,%%rdx\n\t"
 310 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
 311
 312 /* ------------------------ ppc32-linux ------------------------ */
 313
 314 #if defined(PLAT_ppc32_linux)
 315
 316 typedef
 317    struct {
 318       unsigned int nraddr; /* where's the code? */
 319    }
 320    OrigFn;
 321
 322 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 323                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 324                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 325
 326 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 327         _zzq_rlval, _zzq_default, _zzq_request,                   \
 328         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 329                                                                   \
 330   {          unsigned int  _zzq_args[6];                          \
 331              unsigned int  _zzq_result;                           \
 332              unsigned int* _zzq_ptr;                              \
 333     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 334     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 335     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 336     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 337     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 338     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 339     _zzq_ptr = _zzq_args;                                         \
 340     __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
 341                      "mr 4,%2\n\t" /*ptr*/                        \
 342                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 343                      /* %R3 = client_request ( %R4 ) */           \
 344                      "or 1,1,1\n\t"                               \
 345                      "mr %0,3"     /*result*/                     \
 346                      : "=b" (_zzq_result)                         \
 347                      : "b" (_zzq_default), "b" (_zzq_ptr)         \
 348                      : "cc", "memory", "r3", "r4");               \
 349     _zzq_rlval = _zzq_result;                                     \
 350   }
 351
 352 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 353   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 354     unsigned int __addr;                                          \
 355     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 356                      /* %R3 = guest_NRADDR */                     \
 357                      "or 2,2,2\n\t"                               \
 358                      "mr %0,3"                                    \
 359                      : "=b" (__addr)                              \
 360                      :                                            \
 361                      : "cc", "memory", "r3"                       \
 362                     );                                            \
 363     _zzq_orig->nraddr = __addr;                                   \
 364   }
 365
 366 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 367                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 368                      /* branch-and-link-to-noredir *%R11 */       \
 369                      "or 3,3,3\n\t"
 370 #endif /* PLAT_ppc32_linux */
 371
 372 /* ------------------------ ppc64-linux ------------------------ */
 373
 374 #if defined(PLAT_ppc64_linux)
 375
 376 typedef
 377    struct {
 378       unsigned long long int nraddr; /* where's the code? */
 379       unsigned long long int r2;  /* what tocptr do we need? */
 380    }
 381    OrigFn;
 382
 383 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 384                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 385                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 386
 387 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 388         _zzq_rlval, _zzq_default, _zzq_request,                   \
 389         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 390                                                                   \
 391   {          unsigned long long int  _zzq_args[6];                \
 392     register unsigned long long int  _zzq_result __asm__("r3");   \
 393     register unsigned long long int* _zzq_ptr __asm__("r4");      \
 394     _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
 395     _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
 396     _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
 397     _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
 398     _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
 399     _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
 400     _zzq_ptr = _zzq_args;                                         \
 401     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 402                      /* %R3 = client_request ( %R4 ) */           \
 403                      "or 1,1,1"                                   \
 404                      : "=r" (_zzq_result)                         \
 405                      : "0" (_zzq_default), "r" (_zzq_ptr)         \
 406                      : "cc", "memory");                           \
 407     _zzq_rlval = _zzq_result;                                     \
 408   }
 409
 410 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 411   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 412     register unsigned long long int __addr __asm__("r3");         \
 413     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 414                      /* %R3 = guest_NRADDR */                     \
 415                      "or 2,2,2"                                   \
 416                      : "=r" (__addr)                              \
 417                      :                                            \
 418                      : "cc", "memory"                             \
 419                     );                                            \
 420     _zzq_orig->nraddr = __addr;                                   \
 421     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 422                      /* %R3 = guest_NRADDR_GPR2 */                \
 423                      "or 4,4,4"                                   \
 424                      : "=r" (__addr)                              \
 425                      :                                            \
 426                      : "cc", "memory"                             \
 427                     );                                            \
 428     _zzq_orig->r2 = __addr;                                       \
 429   }
 430
 431 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 432                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 433                      /* branch-and-link-to-noredir *%R11 */       \
 434                      "or 3,3,3\n\t"
 435
 436 #endif /* PLAT_ppc64_linux */
 437
 438 /* ------------------------- arm-linux ------------------------- */
 439
 440 #if defined(PLAT_arm_linux)
 441
 442 typedef
 443    struct {
 444       unsigned int nraddr; /* where's the code? */
 445    }
 446    OrigFn;
 447
 448 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 449             "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
 450             "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
 451
 452 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 453         _zzq_rlval, _zzq_default, _zzq_request,                   \
 454         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 455                                                                   \
 456   { volatile unsigned int  _zzq_args[6];                          \
 457     volatile unsigned int  _zzq_result;                           \
 458     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 459     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 460     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 461     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 462     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 463     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 464     __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
 465                      "mov r4, %2\n\t" /*ptr*/                     \
 466                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 467                      /* R3 = client_request ( R4 ) */             \
 468                      "orr r10, r10, r10\n\t"                      \
 469                      "mov %0, r3"     /*result*/                  \
 470                      : "=r" (_zzq_result)                         \
 471                      : "r" (_zzq_default), "r" (&_zzq_args[0])    \
 472                      : "cc","memory", "r3", "r4");                \
 473     _zzq_rlval = _zzq_result;                                     \
 474   }
 475
 476 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 477   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 478     unsigned int __addr;                                          \
 479     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 480                      /* R3 = guest_NRADDR */                      \
 481                      "orr r11, r11, r11\n\t"                      \
 482                      "mov %0, r3"                                 \
 483                      : "=r" (__addr)                              \
 484                      :                                            \
 485                      : "cc", "memory", "r3"                       \
 486                     );                                            \
 487     _zzq_orig->nraddr = __addr;                                   \
 488   }
 489
 490 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
 491                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 492                      /* branch-and-link-to-noredir *%R4 */        \
 493                      "orr r12, r12, r12\n\t"
 494
 495 #endif /* PLAT_arm_linux */
 496
 497 /* ------------------------ ppc32-aix5 ------------------------- */
 498
 499 #if defined(PLAT_ppc32_aix5)
 500
 501 typedef
 502    struct {
 503       unsigned int nraddr; /* where's the code? */
 504       unsigned int r2;  /* what tocptr do we need? */
 505    }
 506    OrigFn;
 507
 508 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 509                      "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
 510                      "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
 511
 512 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 513         _zzq_rlval, _zzq_default, _zzq_request,                   \
 514         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 515                                                                   \
 516   {          unsigned int  _zzq_args[7];                          \
 517     register unsigned int  _zzq_result;                           \
 518     register unsigned int* _zzq_ptr;                              \
 519     _zzq_args[0] = (unsigned int)(_zzq_request);                  \
 520     _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
 521     _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
 522     _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
 523     _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
 524     _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
 525     _zzq_args[6] = (unsigned int)(_zzq_default);                  \
 526     _zzq_ptr = _zzq_args;                                         \
 527     __asm__ volatile("mr 4,%1\n\t"                                \
 528                      "lwz 3, 24(4)\n\t"                           \
 529                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 530                      /* %R3 = client_request ( %R4 ) */           \
 531                      "or 1,1,1\n\t"                               \
 532                      "mr %0,3"                                    \
 533                      : "=b" (_zzq_result)                         \
 534                      : "b" (_zzq_ptr)                             \
 535                      : "r3", "r4", "cc", "memory");               \
 536     _zzq_rlval = _zzq_result;                                     \
 537   }
 538
 539 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 540   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 541     register unsigned int __addr;                                 \
 542     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 543                      /* %R3 = guest_NRADDR */                     \
 544                      "or 2,2,2\n\t"                               \
 545                      "mr %0,3"                                    \
 546                      : "=b" (__addr)                              \
 547                      :                                            \
 548                      : "r3", "cc", "memory"                       \
 549                     );                                            \
 550     _zzq_orig->nraddr = __addr;                                   \
 551     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 552                      /* %R3 = guest_NRADDR_GPR2 */                \
 553                      "or 4,4,4\n\t"                               \
 554                      "mr %0,3"                                    \
 555                      : "=b" (__addr)                              \
 556                      :                                            \
 557                      : "r3", "cc", "memory"                       \
 558                     );                                            \
 559     _zzq_orig->r2 = __addr;                                       \
 560   }
 561
 562 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 563                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 564                      /* branch-and-link-to-noredir *%R11 */       \
 565                      "or 3,3,3\n\t"
 566
 567 #endif /* PLAT_ppc32_aix5 */
 568
 569 /* ------------------------ ppc64-aix5 ------------------------- */
 570
 571 #if defined(PLAT_ppc64_aix5)
 572
 573 typedef
 574    struct {
 575       unsigned long long int nraddr; /* where's the code? */
 576       unsigned long long int r2;  /* what tocptr do we need? */
 577    }
 578    OrigFn;
 579
 580 #define __SPECIAL_INSTRUCTION_PREAMBLE                            \
 581                      "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
 582                      "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
 583
 584 #define VALGRIND_DO_CLIENT_REQUEST(                               \
 585         _zzq_rlval, _zzq_default, _zzq_request,                   \
 586         _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
 587                                                                   \
 588   {          unsigned long long int  _zzq_args[7];                \
 589     register unsigned long long int  _zzq_result;                 \
 590     register unsigned long long int* _zzq_ptr;                    \
 591     _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
 592     _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
 593     _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
 594     _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
 595     _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
 596     _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
 597     _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
 598     _zzq_ptr = _zzq_args;                                         \
 599     __asm__ volatile("mr 4,%1\n\t"                                \
 600                      "ld 3, 48(4)\n\t"                            \
 601                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 602                      /* %R3 = client_request ( %R4 ) */           \
 603                      "or 1,1,1\n\t"                               \
 604                      "mr %0,3"                                    \
 605                      : "=b" (_zzq_result)                         \
 606                      : "b" (_zzq_ptr)                             \
 607                      : "r3", "r4", "cc", "memory");               \
 608     _zzq_rlval = _zzq_result;                                     \
 609   }
 610
 611 #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
 612   { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
 613     register unsigned long long int __addr;                       \
 614     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 615                      /* %R3 = guest_NRADDR */                     \
 616                      "or 2,2,2\n\t"                               \
 617                      "mr %0,3"                                    \
 618                      : "=b" (__addr)                              \
 619                      :                                            \
 620                      : "r3", "cc", "memory"                       \
 621                     );                                            \
 622     _zzq_orig->nraddr = __addr;                                   \
 623     __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
 624                      /* %R3 = guest_NRADDR_GPR2 */                \
 625                      "or 4,4,4\n\t"                               \
 626                      "mr %0,3"                                    \
 627                      : "=b" (__addr)                              \
 628                      :                                            \
 629                      : "r3", "cc", "memory"                       \
 630                     );                                            \
 631     _zzq_orig->r2 = __addr;                                       \
 632   }
 633
 634 #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
 635                      __SPECIAL_INSTRUCTION_PREAMBLE               \
 636                      /* branch-and-link-to-noredir *%R11 */       \
 637                      "or 3,3,3\n\t"
 638
 639 #endif /* PLAT_ppc64_aix5 */
 640
 641 /* Insert assembly code for other platforms here... */
 642
 643 #endif /* NVALGRIND */
 644
 645
 646 /* ------------------------------------------------------------------ */
 647 /* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
 648 /* ugly.  It's the least-worst tradeoff I can think of.               */
 649 /* ------------------------------------------------------------------ */
 650
 651 /* This section defines magic (a.k.a appalling-hack) macros for doing
 652    guaranteed-no-redirection macros, so as to get from function
 653    wrappers to the functions they are wrapping.  The whole point is to
 654    construct standard call sequences, but to do the call itself with a
 655    special no-redirect call pseudo-instruction that the JIT
 656    understands and handles specially.  This section is long and
 657    repetitious, and I can't see a way to make it shorter.
 658
 659    The naming scheme is as follows:
 660
 661       CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
 662
 663    'W' stands for "word" and 'v' for "void".  Hence there are
 664    different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
 665    and for each, the possibility of returning a word-typed result, or
 666    no result.
 667 */
 668
 669 /* Use these to write the name of your wrapper.  NOTE: duplicates
 670    VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
 671
 672 /* Use an extra level of macroisation so as to ensure the soname/fnname
 673    args are fully macro-expanded before pasting them together. */
 674 #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
 675
 676 #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
 677    VG_CONCAT4(_vgwZU_,soname,_,fnname)
 678
 679 #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
 680    VG_CONCAT4(_vgwZZ_,soname,_,fnname)
 681
 682 /* Use this macro from within a wrapper function to collect the
 683    context (address and possibly other info) of the original function.
 684    Once you have that you can then use it in one of the CALL_FN_
 685    macros.  The type of the argument _lval is OrigFn. */
 686 #define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
 687
 688 /* Derivatives of the main macros below, for calling functions
 689    returning void. */
 690
 691 #define CALL_FN_v_v(fnptr)                                        \
 692    do { volatile unsigned long _junk;                             \
 693         CALL_FN_W_v(_junk,fnptr); } while (0)
 694
 695 #define CALL_FN_v_W(fnptr, arg1)                                  \
 696    do { volatile unsigned long _junk;                             \
 697         CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
 698
 699 #define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
 700    do { volatile unsigned long _junk;                             \
 701         CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
 702
 703 #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
 704    do { volatile unsigned long _junk;                             \
 705         CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
 706
 707 #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
 708    do { volatile unsigned long _junk;                             \
 709         CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
 710
 711 #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
 712    do { volatile unsigned long _junk;                             \
 713         CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
 714
 715 #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
 716    do { volatile unsigned long _junk;                             \
 717         CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
 718
 719 #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
 720    do { volatile unsigned long _junk;                             \
 721         CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
 722
 723 /* ------------------------- x86-{linux,darwin} ---------------- */
 724
 725 #if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)
 726
 727 /* These regs are trashed by the hidden call.  No need to mention eax
 728    as gcc can already see that, plus causes gcc to bomb. */
 729 #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
 730
 731 /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
 732    long) == 4. */
 733
 734 #define CALL_FN_W_v(lval, orig)                                   \
 735    do {                                                           \
 736       volatile OrigFn        _orig = (orig);                      \
 737       volatile unsigned long _argvec[1];                          \
 738       volatile unsigned long _res;                                \
 739       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 740       __asm__ volatile(                                           \
 741          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 742          VALGRIND_CALL_NOREDIR_EAX                                \
 743          : /*out*/   "=a" (_res)                                  \
 744          : /*in*/    "a" (&_argvec[0])                            \
 745          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 746       );                                                          \
 747       lval = (__typeof__(lval)) _res;                             \
 748    } while (0)
 749
 750 #define CALL_FN_W_W(lval, orig, arg1)                             \
 751    do {                                                           \
 752       volatile OrigFn        _orig = (orig);                      \
 753       volatile unsigned long _argvec[2];                          \
 754       volatile unsigned long _res;                                \
 755       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 756       _argvec[1] = (unsigned long)(arg1);                         \
 757       __asm__ volatile(                                           \
 758          "pushl 4(%%eax)\n\t"                                     \
 759          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 760          VALGRIND_CALL_NOREDIR_EAX                                \
 761          "addl $4, %%esp\n"                                       \
 762          : /*out*/   "=a" (_res)                                  \
 763          : /*in*/    "a" (&_argvec[0])                            \
 764          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 765       );                                                          \
 766       lval = (__typeof__(lval)) _res;                             \
 767    } while (0)
 768
 769 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
 770    do {                                                           \
 771       volatile OrigFn        _orig = (orig);                      \
 772       volatile unsigned long _argvec[3];                          \
 773       volatile unsigned long _res;                                \
 774       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 775       _argvec[1] = (unsigned long)(arg1);                         \
 776       _argvec[2] = (unsigned long)(arg2);                         \
 777       __asm__ volatile(                                           \
 778          "pushl 8(%%eax)\n\t"                                     \
 779          "pushl 4(%%eax)\n\t"                                     \
 780          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 781          VALGRIND_CALL_NOREDIR_EAX                                \
 782          "addl $8, %%esp\n"                                       \
 783          : /*out*/   "=a" (_res)                                  \
 784          : /*in*/    "a" (&_argvec[0])                            \
 785          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 786       );                                                          \
 787       lval = (__typeof__(lval)) _res;                             \
 788    } while (0)
 789
 790 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
 791    do {                                                           \
 792       volatile OrigFn        _orig = (orig);                      \
 793       volatile unsigned long _argvec[4];                          \
 794       volatile unsigned long _res;                                \
 795       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 796       _argvec[1] = (unsigned long)(arg1);                         \
 797       _argvec[2] = (unsigned long)(arg2);                         \
 798       _argvec[3] = (unsigned long)(arg3);                         \
 799       __asm__ volatile(                                           \
 800          "pushl 12(%%eax)\n\t"                                    \
 801          "pushl 8(%%eax)\n\t"                                     \
 802          "pushl 4(%%eax)\n\t"                                     \
 803          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 804          VALGRIND_CALL_NOREDIR_EAX                                \
 805          "addl $12, %%esp\n"                                      \
 806          : /*out*/   "=a" (_res)                                  \
 807          : /*in*/    "a" (&_argvec[0])                            \
 808          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 809       );                                                          \
 810       lval = (__typeof__(lval)) _res;                             \
 811    } while (0)
 812
 813 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
 814    do {                                                           \
 815       volatile OrigFn        _orig = (orig);                      \
 816       volatile unsigned long _argvec[5];                          \
 817       volatile unsigned long _res;                                \
 818       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 819       _argvec[1] = (unsigned long)(arg1);                         \
 820       _argvec[2] = (unsigned long)(arg2);                         \
 821       _argvec[3] = (unsigned long)(arg3);                         \
 822       _argvec[4] = (unsigned long)(arg4);                         \
 823       __asm__ volatile(                                           \
 824          "pushl 16(%%eax)\n\t"                                    \
 825          "pushl 12(%%eax)\n\t"                                    \
 826          "pushl 8(%%eax)\n\t"                                     \
 827          "pushl 4(%%eax)\n\t"                                     \
 828          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 829          VALGRIND_CALL_NOREDIR_EAX                                \
 830          "addl $16, %%esp\n"                                      \
 831          : /*out*/   "=a" (_res)                                  \
 832          : /*in*/    "a" (&_argvec[0])                            \
 833          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 834       );                                                          \
 835       lval = (__typeof__(lval)) _res;                             \
 836    } while (0)
 837
 838 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
 839    do {                                                           \
 840       volatile OrigFn        _orig = (orig);                      \
 841       volatile unsigned long _argvec[6];                          \
 842       volatile unsigned long _res;                                \
 843       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 844       _argvec[1] = (unsigned long)(arg1);                         \
 845       _argvec[2] = (unsigned long)(arg2);                         \
 846       _argvec[3] = (unsigned long)(arg3);                         \
 847       _argvec[4] = (unsigned long)(arg4);                         \
 848       _argvec[5] = (unsigned long)(arg5);                         \
 849       __asm__ volatile(                                           \
 850          "pushl 20(%%eax)\n\t"                                    \
 851          "pushl 16(%%eax)\n\t"                                    \
 852          "pushl 12(%%eax)\n\t"                                    \
 853          "pushl 8(%%eax)\n\t"                                     \
 854          "pushl 4(%%eax)\n\t"                                     \
 855          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 856          VALGRIND_CALL_NOREDIR_EAX                                \
 857          "addl $20, %%esp\n"                                      \
 858          : /*out*/   "=a" (_res)                                  \
 859          : /*in*/    "a" (&_argvec[0])                            \
 860          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 861       );                                                          \
 862       lval = (__typeof__(lval)) _res;                             \
 863    } while (0)
 864
 865 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
 866    do {                                                           \
 867       volatile OrigFn        _orig = (orig);                      \
 868       volatile unsigned long _argvec[7];                          \
 869       volatile unsigned long _res;                                \
 870       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 871       _argvec[1] = (unsigned long)(arg1);                         \
 872       _argvec[2] = (unsigned long)(arg2);                         \
 873       _argvec[3] = (unsigned long)(arg3);                         \
 874       _argvec[4] = (unsigned long)(arg4);                         \
 875       _argvec[5] = (unsigned long)(arg5);                         \
 876       _argvec[6] = (unsigned long)(arg6);                         \
 877       __asm__ volatile(                                           \
 878          "pushl 24(%%eax)\n\t"                                    \
 879          "pushl 20(%%eax)\n\t"                                    \
 880          "pushl 16(%%eax)\n\t"                                    \
 881          "pushl 12(%%eax)\n\t"                                    \
 882          "pushl 8(%%eax)\n\t"                                     \
 883          "pushl 4(%%eax)\n\t"                                     \
 884          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 885          VALGRIND_CALL_NOREDIR_EAX                                \
 886          "addl $24, %%esp\n"                                      \
 887          : /*out*/   "=a" (_res)                                  \
 888          : /*in*/    "a" (&_argvec[0])                            \
 889          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 890       );                                                          \
 891       lval = (__typeof__(lval)) _res;                             \
 892    } while (0)
 893
 894 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 895                                  arg7)                            \
 896    do {                                                           \
 897       volatile OrigFn        _orig = (orig);                      \
 898       volatile unsigned long _argvec[8];                          \
 899       volatile unsigned long _res;                                \
 900       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 901       _argvec[1] = (unsigned long)(arg1);                         \
 902       _argvec[2] = (unsigned long)(arg2);                         \
 903       _argvec[3] = (unsigned long)(arg3);                         \
 904       _argvec[4] = (unsigned long)(arg4);                         \
 905       _argvec[5] = (unsigned long)(arg5);                         \
 906       _argvec[6] = (unsigned long)(arg6);                         \
 907       _argvec[7] = (unsigned long)(arg7);                         \
 908       __asm__ volatile(                                           \
 909          "pushl 28(%%eax)\n\t"                                    \
 910          "pushl 24(%%eax)\n\t"                                    \
 911          "pushl 20(%%eax)\n\t"                                    \
 912          "pushl 16(%%eax)\n\t"                                    \
 913          "pushl 12(%%eax)\n\t"                                    \
 914          "pushl 8(%%eax)\n\t"                                     \
 915          "pushl 4(%%eax)\n\t"                                     \
 916          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 917          VALGRIND_CALL_NOREDIR_EAX                                \
 918          "addl $28, %%esp\n"                                      \
 919          : /*out*/   "=a" (_res)                                  \
 920          : /*in*/    "a" (&_argvec[0])                            \
 921          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 922       );                                                          \
 923       lval = (__typeof__(lval)) _res;                             \
 924    } while (0)
 925
 926 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 927                                  arg7,arg8)                       \
 928    do {                                                           \
 929       volatile OrigFn        _orig = (orig);                      \
 930       volatile unsigned long _argvec[9];                          \
 931       volatile unsigned long _res;                                \
 932       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 933       _argvec[1] = (unsigned long)(arg1);                         \
 934       _argvec[2] = (unsigned long)(arg2);                         \
 935       _argvec[3] = (unsigned long)(arg3);                         \
 936       _argvec[4] = (unsigned long)(arg4);                         \
 937       _argvec[5] = (unsigned long)(arg5);                         \
 938       _argvec[6] = (unsigned long)(arg6);                         \
 939       _argvec[7] = (unsigned long)(arg7);                         \
 940       _argvec[8] = (unsigned long)(arg8);                         \
 941       __asm__ volatile(                                           \
 942          "pushl 32(%%eax)\n\t"                                    \
 943          "pushl 28(%%eax)\n\t"                                    \
 944          "pushl 24(%%eax)\n\t"                                    \
 945          "pushl 20(%%eax)\n\t"                                    \
 946          "pushl 16(%%eax)\n\t"                                    \
 947          "pushl 12(%%eax)\n\t"                                    \
 948          "pushl 8(%%eax)\n\t"                                     \
 949          "pushl 4(%%eax)\n\t"                                     \
 950          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 951          VALGRIND_CALL_NOREDIR_EAX                                \
 952          "addl $32, %%esp\n"                                      \
 953          : /*out*/   "=a" (_res)                                  \
 954          : /*in*/    "a" (&_argvec[0])                            \
 955          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 956       );                                                          \
 957       lval = (__typeof__(lval)) _res;                             \
 958    } while (0)
 959
 960 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
 961                                  arg7,arg8,arg9)                  \
 962    do {                                                           \
 963       volatile OrigFn        _orig = (orig);                      \
 964       volatile unsigned long _argvec[10];                         \
 965       volatile unsigned long _res;                                \
 966       _argvec[0] = (unsigned long)_orig.nraddr;                   \
 967       _argvec[1] = (unsigned long)(arg1);                         \
 968       _argvec[2] = (unsigned long)(arg2);                         \
 969       _argvec[3] = (unsigned long)(arg3);                         \
 970       _argvec[4] = (unsigned long)(arg4);                         \
 971       _argvec[5] = (unsigned long)(arg5);                         \
 972       _argvec[6] = (unsigned long)(arg6);                         \
 973       _argvec[7] = (unsigned long)(arg7);                         \
 974       _argvec[8] = (unsigned long)(arg8);                         \
 975       _argvec[9] = (unsigned long)(arg9);                         \
 976       __asm__ volatile(                                           \
 977          "pushl 36(%%eax)\n\t"                                    \
 978          "pushl 32(%%eax)\n\t"                                    \
 979          "pushl 28(%%eax)\n\t"                                    \
 980          "pushl 24(%%eax)\n\t"                                    \
 981          "pushl 20(%%eax)\n\t"                                    \
 982          "pushl 16(%%eax)\n\t"                                    \
 983          "pushl 12(%%eax)\n\t"                                    \
 984          "pushl 8(%%eax)\n\t"                                     \
 985          "pushl 4(%%eax)\n\t"                                     \
 986          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
 987          VALGRIND_CALL_NOREDIR_EAX                                \
 988          "addl $36, %%esp\n"                                      \
 989          : /*out*/   "=a" (_res)                                  \
 990          : /*in*/    "a" (&_argvec[0])                            \
 991          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
 992       );                                                          \
 993       lval = (__typeof__(lval)) _res;                             \
 994    } while (0)
 995
 996 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
 997                                   arg7,arg8,arg9,arg10)           \
 998    do {                                                           \
 999       volatile OrigFn        _orig = (orig);                      \
1000       volatile unsigned long _argvec[11];                         \
1001       volatile unsigned long _res;                                \
1002       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1003       _argvec[1] = (unsigned long)(arg1);                         \
1004       _argvec[2] = (unsigned long)(arg2);                         \
1005       _argvec[3] = (unsigned long)(arg3);                         \
1006       _argvec[4] = (unsigned long)(arg4);                         \
1007       _argvec[5] = (unsigned long)(arg5);                         \
1008       _argvec[6] = (unsigned long)(arg6);                         \
1009       _argvec[7] = (unsigned long)(arg7);                         \
1010       _argvec[8] = (unsigned long)(arg8);                         \
1011       _argvec[9] = (unsigned long)(arg9);                         \
1012       _argvec[10] = (unsigned long)(arg10);                       \
1013       __asm__ volatile(                                           \
1014          "pushl 40(%%eax)\n\t"                                    \
1015          "pushl 36(%%eax)\n\t"                                    \
1016          "pushl 32(%%eax)\n\t"                                    \
1017          "pushl 28(%%eax)\n\t"                                    \
1018          "pushl 24(%%eax)\n\t"                                    \
1019          "pushl 20(%%eax)\n\t"                                    \
1020          "pushl 16(%%eax)\n\t"                                    \
1021          "pushl 12(%%eax)\n\t"                                    \
1022          "pushl 8(%%eax)\n\t"                                     \
1023          "pushl 4(%%eax)\n\t"                                     \
1024          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1025          VALGRIND_CALL_NOREDIR_EAX                                \
1026          "addl $40, %%esp\n"                                      \
1027          : /*out*/   "=a" (_res)                                  \
1028          : /*in*/    "a" (&_argvec[0])                            \
1029          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1030       );                                                          \
1031       lval = (__typeof__(lval)) _res;                             \
1032    } while (0)
1033
1034 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1035                                   arg6,arg7,arg8,arg9,arg10,      \
1036                                   arg11)                          \
1037    do {                                                           \
1038       volatile OrigFn        _orig = (orig);                      \
1039       volatile unsigned long _argvec[12];                         \
1040       volatile unsigned long _res;                                \
1041       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1042       _argvec[1] = (unsigned long)(arg1);                         \
1043       _argvec[2] = (unsigned long)(arg2);                         \
1044       _argvec[3] = (unsigned long)(arg3);                         \
1045       _argvec[4] = (unsigned long)(arg4);                         \
1046       _argvec[5] = (unsigned long)(arg5);                         \
1047       _argvec[6] = (unsigned long)(arg6);                         \
1048       _argvec[7] = (unsigned long)(arg7);                         \
1049       _argvec[8] = (unsigned long)(arg8);                         \
1050       _argvec[9] = (unsigned long)(arg9);                         \
1051       _argvec[10] = (unsigned long)(arg10);                       \
1052       _argvec[11] = (unsigned long)(arg11);                       \
1053       __asm__ volatile(                                           \
1054          "pushl 44(%%eax)\n\t"                                    \
1055          "pushl 40(%%eax)\n\t"                                    \
1056          "pushl 36(%%eax)\n\t"                                    \
1057          "pushl 32(%%eax)\n\t"                                    \
1058          "pushl 28(%%eax)\n\t"                                    \
1059          "pushl 24(%%eax)\n\t"                                    \
1060          "pushl 20(%%eax)\n\t"                                    \
1061          "pushl 16(%%eax)\n\t"                                    \
1062          "pushl 12(%%eax)\n\t"                                    \
1063          "pushl 8(%%eax)\n\t"                                     \
1064          "pushl 4(%%eax)\n\t"                                     \
1065          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1066          VALGRIND_CALL_NOREDIR_EAX                                \
1067          "addl $44, %%esp\n"                                      \
1068          : /*out*/   "=a" (_res)                                  \
1069          : /*in*/    "a" (&_argvec[0])                            \
1070          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1071       );                                                          \
1072       lval = (__typeof__(lval)) _res;                             \
1073    } while (0)
1074
1075 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1076                                   arg6,arg7,arg8,arg9,arg10,      \
1077                                   arg11,arg12)                    \
1078    do {                                                           \
1079       volatile OrigFn        _orig = (orig);                      \
1080       volatile unsigned long _argvec[13];                         \
1081       volatile unsigned long _res;                                \
1082       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1083       _argvec[1] = (unsigned long)(arg1);                         \
1084       _argvec[2] = (unsigned long)(arg2);                         \
1085       _argvec[3] = (unsigned long)(arg3);                         \
1086       _argvec[4] = (unsigned long)(arg4);                         \
1087       _argvec[5] = (unsigned long)(arg5);                         \
1088       _argvec[6] = (unsigned long)(arg6);                         \
1089       _argvec[7] = (unsigned long)(arg7);                         \
1090       _argvec[8] = (unsigned long)(arg8);                         \
1091       _argvec[9] = (unsigned long)(arg9);                         \
1092       _argvec[10] = (unsigned long)(arg10);                       \
1093       _argvec[11] = (unsigned long)(arg11);                       \
1094       _argvec[12] = (unsigned long)(arg12);                       \
1095       __asm__ volatile(                                           \
1096          "pushl 48(%%eax)\n\t"                                    \
1097          "pushl 44(%%eax)\n\t"                                    \
1098          "pushl 40(%%eax)\n\t"                                    \
1099          "pushl 36(%%eax)\n\t"                                    \
1100          "pushl 32(%%eax)\n\t"                                    \
1101          "pushl 28(%%eax)\n\t"                                    \
1102          "pushl 24(%%eax)\n\t"                                    \
1103          "pushl 20(%%eax)\n\t"                                    \
1104          "pushl 16(%%eax)\n\t"                                    \
1105          "pushl 12(%%eax)\n\t"                                    \
1106          "pushl 8(%%eax)\n\t"                                     \
1107          "pushl 4(%%eax)\n\t"                                     \
1108          "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1109          VALGRIND_CALL_NOREDIR_EAX                                \
1110          "addl $48, %%esp\n"                                      \
1111          : /*out*/   "=a" (_res)                                  \
1112          : /*in*/    "a" (&_argvec[0])                            \
1113          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1114       );                                                          \
1115       lval = (__typeof__(lval)) _res;                             \
1116    } while (0)
1117
1118 #endif /* PLAT_x86_linux || PLAT_x86_darwin */
1119
1120 /* ------------------------ amd64-{linux,darwin} --------------- */
1121
1122 #if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin)
1123
1124 /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1125
1126 /* These regs are trashed by the hidden call. */
1127 #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1128                             "rdi", "r8", "r9", "r10", "r11"
1129
1130 /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1131    long) == 8. */
1132
1133 /* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1134    macros.  In order not to trash the stack redzone, we need to drop
1135    %rsp by 128 before the hidden call, and restore afterwards.  The
1136    nastyness is that it is only by luck that the stack still appears
1137    to be unwindable during the hidden call - since then the behaviour
1138    of any routine using this macro does not match what the CFI data
1139    says.  Sigh.
1140
1141    Why is this important?  Imagine that a wrapper has a stack
1142    allocated local, and passes to the hidden call, a pointer to it.
1143    Because gcc does not know about the hidden call, it may allocate
1144    that local in the redzone.  Unfortunately the hidden call may then
1145    trash it before it comes to use it.  So we must step clear of the
1146    redzone, for the duration of the hidden call, to make it safe.
1147
1148    Probably the same problem afflicts the other redzone-style ABIs too
1149    (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
1150    self describing (none of this CFI nonsense) so at least messing
1151    with the stack pointer doesn't give a danger of non-unwindable
1152    stack. */
1153
1154 #define CALL_FN_W_v(lval, orig)                                   \
1155    do {                                                           \
1156       volatile OrigFn        _orig = (orig);                      \
1157       volatile unsigned long _argvec[1];                          \
1158       volatile unsigned long _res;                                \
1159       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1160       __asm__ volatile(                                           \
1161          "subq $128,%%rsp\n\t"                                    \
1162          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1163          VALGRIND_CALL_NOREDIR_RAX                                \
1164          "addq $128,%%rsp\n\t"                                    \
1165          : /*out*/   "=a" (_res)                                  \
1166          : /*in*/    "a" (&_argvec[0])                            \
1167          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1168       );                                                          \
1169       lval = (__typeof__(lval)) _res;                             \
1170    } while (0)
1171
1172 #define CALL_FN_W_W(lval, orig, arg1)                             \
1173    do {                                                           \
1174       volatile OrigFn        _orig = (orig);                      \
1175       volatile unsigned long _argvec[2];                          \
1176       volatile unsigned long _res;                                \
1177       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1178       _argvec[1] = (unsigned long)(arg1);                         \
1179       __asm__ volatile(                                           \
1180          "subq $128,%%rsp\n\t"                                    \
1181          "movq 8(%%rax), %%rdi\n\t"                               \
1182          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1183          VALGRIND_CALL_NOREDIR_RAX                                \
1184          "addq $128,%%rsp\n\t"                                    \
1185          : /*out*/   "=a" (_res)                                  \
1186          : /*in*/    "a" (&_argvec[0])                            \
1187          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1188       );                                                          \
1189       lval = (__typeof__(lval)) _res;                             \
1190    } while (0)
1191
1192 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1193    do {                                                           \
1194       volatile OrigFn        _orig = (orig);                      \
1195       volatile unsigned long _argvec[3];                          \
1196       volatile unsigned long _res;                                \
1197       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1198       _argvec[1] = (unsigned long)(arg1);                         \
1199       _argvec[2] = (unsigned long)(arg2);                         \
1200       __asm__ volatile(                                           \
1201          "subq $128,%%rsp\n\t"                                    \
1202          "movq 16(%%rax), %%rsi\n\t"                              \
1203          "movq 8(%%rax), %%rdi\n\t"                               \
1204          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1205          VALGRIND_CALL_NOREDIR_RAX                                \
1206          "addq $128,%%rsp\n\t"                                    \
1207          : /*out*/   "=a" (_res)                                  \
1208          : /*in*/    "a" (&_argvec[0])                            \
1209          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1210       );                                                          \
1211       lval = (__typeof__(lval)) _res;                             \
1212    } while (0)
1213
1214 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1215    do {                                                           \
1216       volatile OrigFn        _orig = (orig);                      \
1217       volatile unsigned long _argvec[4];                          \
1218       volatile unsigned long _res;                                \
1219       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1220       _argvec[1] = (unsigned long)(arg1);                         \
1221       _argvec[2] = (unsigned long)(arg2);                         \
1222       _argvec[3] = (unsigned long)(arg3);                         \
1223       __asm__ volatile(                                           \
1224          "subq $128,%%rsp\n\t"                                    \
1225          "movq 24(%%rax), %%rdx\n\t"                              \
1226          "movq 16(%%rax), %%rsi\n\t"                              \
1227          "movq 8(%%rax), %%rdi\n\t"                               \
1228          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1229          VALGRIND_CALL_NOREDIR_RAX                                \
1230          "addq $128,%%rsp\n\t"                                    \
1231          : /*out*/   "=a" (_res)                                  \
1232          : /*in*/    "a" (&_argvec[0])                            \
1233          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1234       );                                                          \
1235       lval = (__typeof__(lval)) _res;                             \
1236    } while (0)
1237
1238 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1239    do {                                                           \
1240       volatile OrigFn        _orig = (orig);                      \
1241       volatile unsigned long _argvec[5];                          \
1242       volatile unsigned long _res;                                \
1243       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1244       _argvec[1] = (unsigned long)(arg1);                         \
1245       _argvec[2] = (unsigned long)(arg2);                         \
1246       _argvec[3] = (unsigned long)(arg3);                         \
1247       _argvec[4] = (unsigned long)(arg4);                         \
1248       __asm__ volatile(                                           \
1249          "subq $128,%%rsp\n\t"                                    \
1250          "movq 32(%%rax), %%rcx\n\t"                              \
1251          "movq 24(%%rax), %%rdx\n\t"                              \
1252          "movq 16(%%rax), %%rsi\n\t"                              \
1253          "movq 8(%%rax), %%rdi\n\t"                               \
1254          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1255          VALGRIND_CALL_NOREDIR_RAX                                \
1256          "addq $128,%%rsp\n\t"                                    \
1257          : /*out*/   "=a" (_res)                                  \
1258          : /*in*/    "a" (&_argvec[0])                            \
1259          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1260       );                                                          \
1261       lval = (__typeof__(lval)) _res;                             \
1262    } while (0)
1263
1264 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1265    do {                                                           \
1266       volatile OrigFn        _orig = (orig);                      \
1267       volatile unsigned long _argvec[6];                          \
1268       volatile unsigned long _res;                                \
1269       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1270       _argvec[1] = (unsigned long)(arg1);                         \
1271       _argvec[2] = (unsigned long)(arg2);                         \
1272       _argvec[3] = (unsigned long)(arg3);                         \
1273       _argvec[4] = (unsigned long)(arg4);                         \
1274       _argvec[5] = (unsigned long)(arg5);                         \
1275       __asm__ volatile(                                           \
1276          "subq $128,%%rsp\n\t"                                    \
1277          "movq 40(%%rax), %%r8\n\t"                               \
1278          "movq 32(%%rax), %%rcx\n\t"                              \
1279          "movq 24(%%rax), %%rdx\n\t"                              \
1280          "movq 16(%%rax), %%rsi\n\t"                              \
1281          "movq 8(%%rax), %%rdi\n\t"                               \
1282          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1283          VALGRIND_CALL_NOREDIR_RAX                                \
1284          "addq $128,%%rsp\n\t"                                    \
1285          : /*out*/   "=a" (_res)                                  \
1286          : /*in*/    "a" (&_argvec[0])                            \
1287          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1288       );                                                          \
1289       lval = (__typeof__(lval)) _res;                             \
1290    } while (0)
1291
1292 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1293    do {                                                           \
1294       volatile OrigFn        _orig = (orig);                      \
1295       volatile unsigned long _argvec[7];                          \
1296       volatile unsigned long _res;                                \
1297       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1298       _argvec[1] = (unsigned long)(arg1);                         \
1299       _argvec[2] = (unsigned long)(arg2);                         \
1300       _argvec[3] = (unsigned long)(arg3);                         \
1301       _argvec[4] = (unsigned long)(arg4);                         \
1302       _argvec[5] = (unsigned long)(arg5);                         \
1303       _argvec[6] = (unsigned long)(arg6);                         \
1304       __asm__ volatile(                                           \
1305          "subq $128,%%rsp\n\t"                                    \
1306          "movq 48(%%rax), %%r9\n\t"                               \
1307          "movq 40(%%rax), %%r8\n\t"                               \
1308          "movq 32(%%rax), %%rcx\n\t"                              \
1309          "movq 24(%%rax), %%rdx\n\t"                              \
1310          "movq 16(%%rax), %%rsi\n\t"                              \
1311          "movq 8(%%rax), %%rdi\n\t"                               \
1312          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1313          "addq $128,%%rsp\n\t"                                    \
1314          VALGRIND_CALL_NOREDIR_RAX                                \
1315          : /*out*/   "=a" (_res)                                  \
1316          : /*in*/    "a" (&_argvec[0])                            \
1317          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1318       );                                                          \
1319       lval = (__typeof__(lval)) _res;                             \
1320    } while (0)
1321
1322 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1323                                  arg7)                            \
1324    do {                                                           \
1325       volatile OrigFn        _orig = (orig);                      \
1326       volatile unsigned long _argvec[8];                          \
1327       volatile unsigned long _res;                                \
1328       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1329       _argvec[1] = (unsigned long)(arg1);                         \
1330       _argvec[2] = (unsigned long)(arg2);                         \
1331       _argvec[3] = (unsigned long)(arg3);                         \
1332       _argvec[4] = (unsigned long)(arg4);                         \
1333       _argvec[5] = (unsigned long)(arg5);                         \
1334       _argvec[6] = (unsigned long)(arg6);                         \
1335       _argvec[7] = (unsigned long)(arg7);                         \
1336       __asm__ volatile(                                           \
1337          "subq $128,%%rsp\n\t"                                    \
1338          "pushq 56(%%rax)\n\t"                                    \
1339          "movq 48(%%rax), %%r9\n\t"                               \
1340          "movq 40(%%rax), %%r8\n\t"                               \
1341          "movq 32(%%rax), %%rcx\n\t"                              \
1342          "movq 24(%%rax), %%rdx\n\t"                              \
1343          "movq 16(%%rax), %%rsi\n\t"                              \
1344          "movq 8(%%rax), %%rdi\n\t"                               \
1345          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1346          VALGRIND_CALL_NOREDIR_RAX                                \
1347          "addq $8, %%rsp\n"                                       \
1348          "addq $128,%%rsp\n\t"                                    \
1349          : /*out*/   "=a" (_res)                                  \
1350          : /*in*/    "a" (&_argvec[0])                            \
1351          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1352       );                                                          \
1353       lval = (__typeof__(lval)) _res;                             \
1354    } while (0)
1355
1356 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1357                                  arg7,arg8)                       \
1358    do {                                                           \
1359       volatile OrigFn        _orig = (orig);                      \
1360       volatile unsigned long _argvec[9];                          \
1361       volatile unsigned long _res;                                \
1362       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1363       _argvec[1] = (unsigned long)(arg1);                         \
1364       _argvec[2] = (unsigned long)(arg2);                         \
1365       _argvec[3] = (unsigned long)(arg3);                         \
1366       _argvec[4] = (unsigned long)(arg4);                         \
1367       _argvec[5] = (unsigned long)(arg5);                         \
1368       _argvec[6] = (unsigned long)(arg6);                         \
1369       _argvec[7] = (unsigned long)(arg7);                         \
1370       _argvec[8] = (unsigned long)(arg8);                         \
1371       __asm__ volatile(                                           \
1372          "subq $128,%%rsp\n\t"                                    \
1373          "pushq 64(%%rax)\n\t"                                    \
1374          "pushq 56(%%rax)\n\t"                                    \
1375          "movq 48(%%rax), %%r9\n\t"                               \
1376          "movq 40(%%rax), %%r8\n\t"                               \
1377          "movq 32(%%rax), %%rcx\n\t"                              \
1378          "movq 24(%%rax), %%rdx\n\t"                              \
1379          "movq 16(%%rax), %%rsi\n\t"                              \
1380          "movq 8(%%rax), %%rdi\n\t"                               \
1381          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1382          VALGRIND_CALL_NOREDIR_RAX                                \
1383          "addq $16, %%rsp\n"                                      \
1384          "addq $128,%%rsp\n\t"                                    \
1385          : /*out*/   "=a" (_res)                                  \
1386          : /*in*/    "a" (&_argvec[0])                            \
1387          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1388       );                                                          \
1389       lval = (__typeof__(lval)) _res;                             \
1390    } while (0)
1391
1392 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1393                                  arg7,arg8,arg9)                  \
1394    do {                                                           \
1395       volatile OrigFn        _orig = (orig);                      \
1396       volatile unsigned long _argvec[10];                         \
1397       volatile unsigned long _res;                                \
1398       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1399       _argvec[1] = (unsigned long)(arg1);                         \
1400       _argvec[2] = (unsigned long)(arg2);                         \
1401       _argvec[3] = (unsigned long)(arg3);                         \
1402       _argvec[4] = (unsigned long)(arg4);                         \
1403       _argvec[5] = (unsigned long)(arg5);                         \
1404       _argvec[6] = (unsigned long)(arg6);                         \
1405       _argvec[7] = (unsigned long)(arg7);                         \
1406       _argvec[8] = (unsigned long)(arg8);                         \
1407       _argvec[9] = (unsigned long)(arg9);                         \
1408       __asm__ volatile(                                           \
1409          "subq $128,%%rsp\n\t"                                    \
1410          "pushq 72(%%rax)\n\t"                                    \
1411          "pushq 64(%%rax)\n\t"                                    \
1412          "pushq 56(%%rax)\n\t"                                    \
1413          "movq 48(%%rax), %%r9\n\t"                               \
1414          "movq 40(%%rax), %%r8\n\t"                               \
1415          "movq 32(%%rax), %%rcx\n\t"                              \
1416          "movq 24(%%rax), %%rdx\n\t"                              \
1417          "movq 16(%%rax), %%rsi\n\t"                              \
1418          "movq 8(%%rax), %%rdi\n\t"                               \
1419          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1420          VALGRIND_CALL_NOREDIR_RAX                                \
1421          "addq $24, %%rsp\n"                                      \
1422          "addq $128,%%rsp\n\t"                                    \
1423          : /*out*/   "=a" (_res)                                  \
1424          : /*in*/    "a" (&_argvec[0])                            \
1425          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1426       );                                                          \
1427       lval = (__typeof__(lval)) _res;                             \
1428    } while (0)
1429
1430 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1431                                   arg7,arg8,arg9,arg10)           \
1432    do {                                                           \
1433       volatile OrigFn        _orig = (orig);                      \
1434       volatile unsigned long _argvec[11];                         \
1435       volatile unsigned long _res;                                \
1436       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1437       _argvec[1] = (unsigned long)(arg1);                         \
1438       _argvec[2] = (unsigned long)(arg2);                         \
1439       _argvec[3] = (unsigned long)(arg3);                         \
1440       _argvec[4] = (unsigned long)(arg4);                         \
1441       _argvec[5] = (unsigned long)(arg5);                         \
1442       _argvec[6] = (unsigned long)(arg6);                         \
1443       _argvec[7] = (unsigned long)(arg7);                         \
1444       _argvec[8] = (unsigned long)(arg8);                         \
1445       _argvec[9] = (unsigned long)(arg9);                         \
1446       _argvec[10] = (unsigned long)(arg10);                       \
1447       __asm__ volatile(                                           \
1448          "subq $128,%%rsp\n\t"                                    \
1449          "pushq 80(%%rax)\n\t"                                    \
1450          "pushq 72(%%rax)\n\t"                                    \
1451          "pushq 64(%%rax)\n\t"                                    \
1452          "pushq 56(%%rax)\n\t"                                    \
1453          "movq 48(%%rax), %%r9\n\t"                               \
1454          "movq 40(%%rax), %%r8\n\t"                               \
1455          "movq 32(%%rax), %%rcx\n\t"                              \
1456          "movq 24(%%rax), %%rdx\n\t"                              \
1457          "movq 16(%%rax), %%rsi\n\t"                              \
1458          "movq 8(%%rax), %%rdi\n\t"                               \
1459          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1460          VALGRIND_CALL_NOREDIR_RAX                                \
1461          "addq $32, %%rsp\n"                                      \
1462          "addq $128,%%rsp\n\t"                                    \
1463          : /*out*/   "=a" (_res)                                  \
1464          : /*in*/    "a" (&_argvec[0])                            \
1465          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1466       );                                                          \
1467       lval = (__typeof__(lval)) _res;                             \
1468    } while (0)
1469
1470 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1471                                   arg7,arg8,arg9,arg10,arg11)     \
1472    do {                                                           \
1473       volatile OrigFn        _orig = (orig);                      \
1474       volatile unsigned long _argvec[12];                         \
1475       volatile unsigned long _res;                                \
1476       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1477       _argvec[1] = (unsigned long)(arg1);                         \
1478       _argvec[2] = (unsigned long)(arg2);                         \
1479       _argvec[3] = (unsigned long)(arg3);                         \
1480       _argvec[4] = (unsigned long)(arg4);                         \
1481       _argvec[5] = (unsigned long)(arg5);                         \
1482       _argvec[6] = (unsigned long)(arg6);                         \
1483       _argvec[7] = (unsigned long)(arg7);                         \
1484       _argvec[8] = (unsigned long)(arg8);                         \
1485       _argvec[9] = (unsigned long)(arg9);                         \
1486       _argvec[10] = (unsigned long)(arg10);                       \
1487       _argvec[11] = (unsigned long)(arg11);                       \
1488       __asm__ volatile(                                           \
1489          "subq $128,%%rsp\n\t"                                    \
1490          "pushq 88(%%rax)\n\t"                                    \
1491          "pushq 80(%%rax)\n\t"                                    \
1492          "pushq 72(%%rax)\n\t"                                    \
1493          "pushq 64(%%rax)\n\t"                                    \
1494          "pushq 56(%%rax)\n\t"                                    \
1495          "movq 48(%%rax), %%r9\n\t"                               \
1496          "movq 40(%%rax), %%r8\n\t"                               \
1497          "movq 32(%%rax), %%rcx\n\t"                              \
1498          "movq 24(%%rax), %%rdx\n\t"                              \
1499          "movq 16(%%rax), %%rsi\n\t"                              \
1500          "movq 8(%%rax), %%rdi\n\t"                               \
1501          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1502          VALGRIND_CALL_NOREDIR_RAX                                \
1503          "addq $40, %%rsp\n"                                      \
1504          "addq $128,%%rsp\n\t"                                    \
1505          : /*out*/   "=a" (_res)                                  \
1506          : /*in*/    "a" (&_argvec[0])                            \
1507          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1508       );                                                          \
1509       lval = (__typeof__(lval)) _res;                             \
1510    } while (0)
1511
1512 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1513                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1514    do {                                                           \
1515       volatile OrigFn        _orig = (orig);                      \
1516       volatile unsigned long _argvec[13];                         \
1517       volatile unsigned long _res;                                \
1518       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1519       _argvec[1] = (unsigned long)(arg1);                         \
1520       _argvec[2] = (unsigned long)(arg2);                         \
1521       _argvec[3] = (unsigned long)(arg3);                         \
1522       _argvec[4] = (unsigned long)(arg4);                         \
1523       _argvec[5] = (unsigned long)(arg5);                         \
1524       _argvec[6] = (unsigned long)(arg6);                         \
1525       _argvec[7] = (unsigned long)(arg7);                         \
1526       _argvec[8] = (unsigned long)(arg8);                         \
1527       _argvec[9] = (unsigned long)(arg9);                         \
1528       _argvec[10] = (unsigned long)(arg10);                       \
1529       _argvec[11] = (unsigned long)(arg11);                       \
1530       _argvec[12] = (unsigned long)(arg12);                       \
1531       __asm__ volatile(                                           \
1532          "subq $128,%%rsp\n\t"                                    \
1533          "pushq 96(%%rax)\n\t"                                    \
1534          "pushq 88(%%rax)\n\t"                                    \
1535          "pushq 80(%%rax)\n\t"                                    \
1536          "pushq 72(%%rax)\n\t"                                    \
1537          "pushq 64(%%rax)\n\t"                                    \
1538          "pushq 56(%%rax)\n\t"                                    \
1539          "movq 48(%%rax), %%r9\n\t"                               \
1540          "movq 40(%%rax), %%r8\n\t"                               \
1541          "movq 32(%%rax), %%rcx\n\t"                              \
1542          "movq 24(%%rax), %%rdx\n\t"                              \
1543          "movq 16(%%rax), %%rsi\n\t"                              \
1544          "movq 8(%%rax), %%rdi\n\t"                               \
1545          "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
1546          VALGRIND_CALL_NOREDIR_RAX                                \
1547          "addq $48, %%rsp\n"                                      \
1548          "addq $128,%%rsp\n\t"                                    \
1549          : /*out*/   "=a" (_res)                                  \
1550          : /*in*/    "a" (&_argvec[0])                            \
1551          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1552       );                                                          \
1553       lval = (__typeof__(lval)) _res;                             \
1554    } while (0)
1555
1556 #endif /* PLAT_amd64_linux || PLAT_amd64_darwin */
1557
1558 /* ------------------------ ppc32-linux ------------------------ */
1559
1560 #if defined(PLAT_ppc32_linux)
1561
1562 /* This is useful for finding out about the on-stack stuff:
1563
1564    extern int f9  ( int,int,int,int,int,int,int,int,int );
1565    extern int f10 ( int,int,int,int,int,int,int,int,int,int );
1566    extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
1567    extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
1568
1569    int g9 ( void ) {
1570       return f9(11,22,33,44,55,66,77,88,99);
1571    }
1572    int g10 ( void ) {
1573       return f10(11,22,33,44,55,66,77,88,99,110);
1574    }
1575    int g11 ( void ) {
1576       return f11(11,22,33,44,55,66,77,88,99,110,121);
1577    }
1578    int g12 ( void ) {
1579       return f12(11,22,33,44,55,66,77,88,99,110,121,132);
1580    }
1581 */
1582
1583 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
1584
1585 /* These regs are trashed by the hidden call. */
1586 #define __CALLER_SAVED_REGS                                       \
1587    "lr", "ctr", "xer",                                            \
1588    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
1589    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
1590    "r11", "r12", "r13"
1591
1592 /* These CALL_FN_ macros assume that on ppc32-linux,
1593    sizeof(unsigned long) == 4. */
1594
1595 #define CALL_FN_W_v(lval, orig)                                   \
1596    do {                                                           \
1597       volatile OrigFn        _orig = (orig);                      \
1598       volatile unsigned long _argvec[1];                          \
1599       volatile unsigned long _res;                                \
1600       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1601       __asm__ volatile(                                           \
1602          "mr 11,%1\n\t"                                           \
1603          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1604          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1605          "mr %0,3"                                                \
1606          : /*out*/   "=r" (_res)                                  \
1607          : /*in*/    "r" (&_argvec[0])                            \
1608          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1609       );                                                          \
1610       lval = (__typeof__(lval)) _res;                             \
1611    } while (0)
1612
1613 #define CALL_FN_W_W(lval, orig, arg1)                             \
1614    do {                                                           \
1615       volatile OrigFn        _orig = (orig);                      \
1616       volatile unsigned long _argvec[2];                          \
1617       volatile unsigned long _res;                                \
1618       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1619       _argvec[1] = (unsigned long)arg1;                           \
1620       __asm__ volatile(                                           \
1621          "mr 11,%1\n\t"                                           \
1622          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1623          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1624          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1625          "mr %0,3"                                                \
1626          : /*out*/   "=r" (_res)                                  \
1627          : /*in*/    "r" (&_argvec[0])                            \
1628          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1629       );                                                          \
1630       lval = (__typeof__(lval)) _res;                             \
1631    } while (0)
1632
1633 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1634    do {                                                           \
1635       volatile OrigFn        _orig = (orig);                      \
1636       volatile unsigned long _argvec[3];                          \
1637       volatile unsigned long _res;                                \
1638       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1639       _argvec[1] = (unsigned long)arg1;                           \
1640       _argvec[2] = (unsigned long)arg2;                           \
1641       __asm__ volatile(                                           \
1642          "mr 11,%1\n\t"                                           \
1643          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1644          "lwz 4,8(11)\n\t"                                        \
1645          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1646          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1647          "mr %0,3"                                                \
1648          : /*out*/   "=r" (_res)                                  \
1649          : /*in*/    "r" (&_argvec[0])                            \
1650          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1651       );                                                          \
1652       lval = (__typeof__(lval)) _res;                             \
1653    } while (0)
1654
1655 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1656    do {                                                           \
1657       volatile OrigFn        _orig = (orig);                      \
1658       volatile unsigned long _argvec[4];                          \
1659       volatile unsigned long _res;                                \
1660       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1661       _argvec[1] = (unsigned long)arg1;                           \
1662       _argvec[2] = (unsigned long)arg2;                           \
1663       _argvec[3] = (unsigned long)arg3;                           \
1664       __asm__ volatile(                                           \
1665          "mr 11,%1\n\t"                                           \
1666          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1667          "lwz 4,8(11)\n\t"                                        \
1668          "lwz 5,12(11)\n\t"                                       \
1669          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1670          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1671          "mr %0,3"                                                \
1672          : /*out*/   "=r" (_res)                                  \
1673          : /*in*/    "r" (&_argvec[0])                            \
1674          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1675       );                                                          \
1676       lval = (__typeof__(lval)) _res;                             \
1677    } while (0)
1678
1679 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1680    do {                                                           \
1681       volatile OrigFn        _orig = (orig);                      \
1682       volatile unsigned long _argvec[5];                          \
1683       volatile unsigned long _res;                                \
1684       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1685       _argvec[1] = (unsigned long)arg1;                           \
1686       _argvec[2] = (unsigned long)arg2;                           \
1687       _argvec[3] = (unsigned long)arg3;                           \
1688       _argvec[4] = (unsigned long)arg4;                           \
1689       __asm__ volatile(                                           \
1690          "mr 11,%1\n\t"                                           \
1691          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1692          "lwz 4,8(11)\n\t"                                        \
1693          "lwz 5,12(11)\n\t"                                       \
1694          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1695          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1696          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1697          "mr %0,3"                                                \
1698          : /*out*/   "=r" (_res)                                  \
1699          : /*in*/    "r" (&_argvec[0])                            \
1700          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1701       );                                                          \
1702       lval = (__typeof__(lval)) _res;                             \
1703    } while (0)
1704
1705 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1706    do {                                                           \
1707       volatile OrigFn        _orig = (orig);                      \
1708       volatile unsigned long _argvec[6];                          \
1709       volatile unsigned long _res;                                \
1710       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1711       _argvec[1] = (unsigned long)arg1;                           \
1712       _argvec[2] = (unsigned long)arg2;                           \
1713       _argvec[3] = (unsigned long)arg3;                           \
1714       _argvec[4] = (unsigned long)arg4;                           \
1715       _argvec[5] = (unsigned long)arg5;                           \
1716       __asm__ volatile(                                           \
1717          "mr 11,%1\n\t"                                           \
1718          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1719          "lwz 4,8(11)\n\t"                                        \
1720          "lwz 5,12(11)\n\t"                                       \
1721          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1722          "lwz 7,20(11)\n\t"                                       \
1723          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1724          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1725          "mr %0,3"                                                \
1726          : /*out*/   "=r" (_res)                                  \
1727          : /*in*/    "r" (&_argvec[0])                            \
1728          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1729       );                                                          \
1730       lval = (__typeof__(lval)) _res;                             \
1731    } while (0)
1732
1733 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1734    do {                                                           \
1735       volatile OrigFn        _orig = (orig);                      \
1736       volatile unsigned long _argvec[7];                          \
1737       volatile unsigned long _res;                                \
1738       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1739       _argvec[1] = (unsigned long)arg1;                           \
1740       _argvec[2] = (unsigned long)arg2;                           \
1741       _argvec[3] = (unsigned long)arg3;                           \
1742       _argvec[4] = (unsigned long)arg4;                           \
1743       _argvec[5] = (unsigned long)arg5;                           \
1744       _argvec[6] = (unsigned long)arg6;                           \
1745       __asm__ volatile(                                           \
1746          "mr 11,%1\n\t"                                           \
1747          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1748          "lwz 4,8(11)\n\t"                                        \
1749          "lwz 5,12(11)\n\t"                                       \
1750          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1751          "lwz 7,20(11)\n\t"                                       \
1752          "lwz 8,24(11)\n\t"                                       \
1753          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1754          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1755          "mr %0,3"                                                \
1756          : /*out*/   "=r" (_res)                                  \
1757          : /*in*/    "r" (&_argvec[0])                            \
1758          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1759       );                                                          \
1760       lval = (__typeof__(lval)) _res;                             \
1761    } while (0)
1762
1763 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1764                                  arg7)                            \
1765    do {                                                           \
1766       volatile OrigFn        _orig = (orig);                      \
1767       volatile unsigned long _argvec[8];                          \
1768       volatile unsigned long _res;                                \
1769       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1770       _argvec[1] = (unsigned long)arg1;                           \
1771       _argvec[2] = (unsigned long)arg2;                           \
1772       _argvec[3] = (unsigned long)arg3;                           \
1773       _argvec[4] = (unsigned long)arg4;                           \
1774       _argvec[5] = (unsigned long)arg5;                           \
1775       _argvec[6] = (unsigned long)arg6;                           \
1776       _argvec[7] = (unsigned long)arg7;                           \
1777       __asm__ volatile(                                           \
1778          "mr 11,%1\n\t"                                           \
1779          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1780          "lwz 4,8(11)\n\t"                                        \
1781          "lwz 5,12(11)\n\t"                                       \
1782          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1783          "lwz 7,20(11)\n\t"                                       \
1784          "lwz 8,24(11)\n\t"                                       \
1785          "lwz 9,28(11)\n\t"                                       \
1786          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1787          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1788          "mr %0,3"                                                \
1789          : /*out*/   "=r" (_res)                                  \
1790          : /*in*/    "r" (&_argvec[0])                            \
1791          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1792       );                                                          \
1793       lval = (__typeof__(lval)) _res;                             \
1794    } while (0)
1795
1796 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1797                                  arg7,arg8)                       \
1798    do {                                                           \
1799       volatile OrigFn        _orig = (orig);                      \
1800       volatile unsigned long _argvec[9];                          \
1801       volatile unsigned long _res;                                \
1802       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1803       _argvec[1] = (unsigned long)arg1;                           \
1804       _argvec[2] = (unsigned long)arg2;                           \
1805       _argvec[3] = (unsigned long)arg3;                           \
1806       _argvec[4] = (unsigned long)arg4;                           \
1807       _argvec[5] = (unsigned long)arg5;                           \
1808       _argvec[6] = (unsigned long)arg6;                           \
1809       _argvec[7] = (unsigned long)arg7;                           \
1810       _argvec[8] = (unsigned long)arg8;                           \
1811       __asm__ volatile(                                           \
1812          "mr 11,%1\n\t"                                           \
1813          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1814          "lwz 4,8(11)\n\t"                                        \
1815          "lwz 5,12(11)\n\t"                                       \
1816          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1817          "lwz 7,20(11)\n\t"                                       \
1818          "lwz 8,24(11)\n\t"                                       \
1819          "lwz 9,28(11)\n\t"                                       \
1820          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1821          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1822          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1823          "mr %0,3"                                                \
1824          : /*out*/   "=r" (_res)                                  \
1825          : /*in*/    "r" (&_argvec[0])                            \
1826          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1827       );                                                          \
1828       lval = (__typeof__(lval)) _res;                             \
1829    } while (0)
1830
1831 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1832                                  arg7,arg8,arg9)                  \
1833    do {                                                           \
1834       volatile OrigFn        _orig = (orig);                      \
1835       volatile unsigned long _argvec[10];                         \
1836       volatile unsigned long _res;                                \
1837       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1838       _argvec[1] = (unsigned long)arg1;                           \
1839       _argvec[2] = (unsigned long)arg2;                           \
1840       _argvec[3] = (unsigned long)arg3;                           \
1841       _argvec[4] = (unsigned long)arg4;                           \
1842       _argvec[5] = (unsigned long)arg5;                           \
1843       _argvec[6] = (unsigned long)arg6;                           \
1844       _argvec[7] = (unsigned long)arg7;                           \
1845       _argvec[8] = (unsigned long)arg8;                           \
1846       _argvec[9] = (unsigned long)arg9;                           \
1847       __asm__ volatile(                                           \
1848          "mr 11,%1\n\t"                                           \
1849          "addi 1,1,-16\n\t"                                       \
1850          /* arg9 */                                               \
1851          "lwz 3,36(11)\n\t"                                       \
1852          "stw 3,8(1)\n\t"                                         \
1853          /* args1-8 */                                            \
1854          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1855          "lwz 4,8(11)\n\t"                                        \
1856          "lwz 5,12(11)\n\t"                                       \
1857          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1858          "lwz 7,20(11)\n\t"                                       \
1859          "lwz 8,24(11)\n\t"                                       \
1860          "lwz 9,28(11)\n\t"                                       \
1861          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1862          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1863          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1864          "addi 1,1,16\n\t"                                        \
1865          "mr %0,3"                                                \
1866          : /*out*/   "=r" (_res)                                  \
1867          : /*in*/    "r" (&_argvec[0])                            \
1868          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1869       );                                                          \
1870       lval = (__typeof__(lval)) _res;                             \
1871    } while (0)
1872
1873 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1874                                   arg7,arg8,arg9,arg10)           \
1875    do {                                                           \
1876       volatile OrigFn        _orig = (orig);                      \
1877       volatile unsigned long _argvec[11];                         \
1878       volatile unsigned long _res;                                \
1879       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1880       _argvec[1] = (unsigned long)arg1;                           \
1881       _argvec[2] = (unsigned long)arg2;                           \
1882       _argvec[3] = (unsigned long)arg3;                           \
1883       _argvec[4] = (unsigned long)arg4;                           \
1884       _argvec[5] = (unsigned long)arg5;                           \
1885       _argvec[6] = (unsigned long)arg6;                           \
1886       _argvec[7] = (unsigned long)arg7;                           \
1887       _argvec[8] = (unsigned long)arg8;                           \
1888       _argvec[9] = (unsigned long)arg9;                           \
1889       _argvec[10] = (unsigned long)arg10;                         \
1890       __asm__ volatile(                                           \
1891          "mr 11,%1\n\t"                                           \
1892          "addi 1,1,-16\n\t"                                       \
1893          /* arg10 */                                              \
1894          "lwz 3,40(11)\n\t"                                       \
1895          "stw 3,12(1)\n\t"                                        \
1896          /* arg9 */                                               \
1897          "lwz 3,36(11)\n\t"                                       \
1898          "stw 3,8(1)\n\t"                                         \
1899          /* args1-8 */                                            \
1900          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1901          "lwz 4,8(11)\n\t"                                        \
1902          "lwz 5,12(11)\n\t"                                       \
1903          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1904          "lwz 7,20(11)\n\t"                                       \
1905          "lwz 8,24(11)\n\t"                                       \
1906          "lwz 9,28(11)\n\t"                                       \
1907          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1908          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1909          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1910          "addi 1,1,16\n\t"                                        \
1911          "mr %0,3"                                                \
1912          : /*out*/   "=r" (_res)                                  \
1913          : /*in*/    "r" (&_argvec[0])                            \
1914          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1915       );                                                          \
1916       lval = (__typeof__(lval)) _res;                             \
1917    } while (0)
1918
1919 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1920                                   arg7,arg8,arg9,arg10,arg11)     \
1921    do {                                                           \
1922       volatile OrigFn        _orig = (orig);                      \
1923       volatile unsigned long _argvec[12];                         \
1924       volatile unsigned long _res;                                \
1925       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1926       _argvec[1] = (unsigned long)arg1;                           \
1927       _argvec[2] = (unsigned long)arg2;                           \
1928       _argvec[3] = (unsigned long)arg3;                           \
1929       _argvec[4] = (unsigned long)arg4;                           \
1930       _argvec[5] = (unsigned long)arg5;                           \
1931       _argvec[6] = (unsigned long)arg6;                           \
1932       _argvec[7] = (unsigned long)arg7;                           \
1933       _argvec[8] = (unsigned long)arg8;                           \
1934       _argvec[9] = (unsigned long)arg9;                           \
1935       _argvec[10] = (unsigned long)arg10;                         \
1936       _argvec[11] = (unsigned long)arg11;                         \
1937       __asm__ volatile(                                           \
1938          "mr 11,%1\n\t"                                           \
1939          "addi 1,1,-32\n\t"                                       \
1940          /* arg11 */                                              \
1941          "lwz 3,44(11)\n\t"                                       \
1942          "stw 3,16(1)\n\t"                                        \
1943          /* arg10 */                                              \
1944          "lwz 3,40(11)\n\t"                                       \
1945          "stw 3,12(1)\n\t"                                        \
1946          /* arg9 */                                               \
1947          "lwz 3,36(11)\n\t"                                       \
1948          "stw 3,8(1)\n\t"                                         \
1949          /* args1-8 */                                            \
1950          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
1951          "lwz 4,8(11)\n\t"                                        \
1952          "lwz 5,12(11)\n\t"                                       \
1953          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
1954          "lwz 7,20(11)\n\t"                                       \
1955          "lwz 8,24(11)\n\t"                                       \
1956          "lwz 9,28(11)\n\t"                                       \
1957          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
1958          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
1959          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
1960          "addi 1,1,32\n\t"                                        \
1961          "mr %0,3"                                                \
1962          : /*out*/   "=r" (_res)                                  \
1963          : /*in*/    "r" (&_argvec[0])                            \
1964          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
1965       );                                                          \
1966       lval = (__typeof__(lval)) _res;                             \
1967    } while (0)
1968
1969 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1970                                 arg7,arg8,arg9,arg10,arg11,arg12) \
1971    do {                                                           \
1972       volatile OrigFn        _orig = (orig);                      \
1973       volatile unsigned long _argvec[13];                         \
1974       volatile unsigned long _res;                                \
1975       _argvec[0] = (unsigned long)_orig.nraddr;                   \
1976       _argvec[1] = (unsigned long)arg1;                           \
1977       _argvec[2] = (unsigned long)arg2;                           \
1978       _argvec[3] = (unsigned long)arg3;                           \
1979       _argvec[4] = (unsigned long)arg4;                           \
1980       _argvec[5] = (unsigned long)arg5;                           \
1981       _argvec[6] = (unsigned long)arg6;                           \
1982       _argvec[7] = (unsigned long)arg7;                           \
1983       _argvec[8] = (unsigned long)arg8;                           \
1984       _argvec[9] = (unsigned long)arg9;                           \
1985       _argvec[10] = (unsigned long)arg10;                         \
1986       _argvec[11] = (unsigned long)arg11;                         \
1987       _argvec[12] = (unsigned long)arg12;                         \
1988       __asm__ volatile(                                           \
1989          "mr 11,%1\n\t"                                           \
1990          "addi 1,1,-32\n\t"                                       \
1991          /* arg12 */                                              \
1992          "lwz 3,48(11)\n\t"                                       \
1993          "stw 3,20(1)\n\t"                                        \
1994          /* arg11 */                                              \
1995          "lwz 3,44(11)\n\t"                                       \
1996          "stw 3,16(1)\n\t"                                        \
1997          /* arg10 */                                              \
1998          "lwz 3,40(11)\n\t"                                       \
1999          "stw 3,12(1)\n\t"                                        \
2000          /* arg9 */                                               \
2001          "lwz 3,36(11)\n\t"                                       \
2002          "stw 3,8(1)\n\t"                                         \
2003          /* args1-8 */                                            \
2004          "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2005          "lwz 4,8(11)\n\t"                                        \
2006          "lwz 5,12(11)\n\t"                                       \
2007          "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2008          "lwz 7,20(11)\n\t"                                       \
2009          "lwz 8,24(11)\n\t"                                       \
2010          "lwz 9,28(11)\n\t"                                       \
2011          "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2012          "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2013          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2014          "addi 1,1,32\n\t"                                        \
2015          "mr %0,3"                                                \
2016          : /*out*/   "=r" (_res)                                  \
2017          : /*in*/    "r" (&_argvec[0])                            \
2018          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2019       );                                                          \
2020       lval = (__typeof__(lval)) _res;                             \
2021    } while (0)
2022
2023 #endif /* PLAT_ppc32_linux */
2024
2025 /* ------------------------ ppc64-linux ------------------------ */
2026
2027 #if defined(PLAT_ppc64_linux)
2028
2029 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2030
2031 /* These regs are trashed by the hidden call. */
2032 #define __CALLER_SAVED_REGS                                       \
2033    "lr", "ctr", "xer",                                            \
2034    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2035    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2036    "r11", "r12", "r13"
2037
2038 /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2039    long) == 8. */
2040
2041 #define CALL_FN_W_v(lval, orig)                                   \
2042    do {                                                           \
2043       volatile OrigFn        _orig = (orig);                      \
2044       volatile unsigned long _argvec[3+0];                        \
2045       volatile unsigned long _res;                                \
2046       /* _argvec[0] holds current r2 across the call */           \
2047       _argvec[1] = (unsigned long)_orig.r2;                       \
2048       _argvec[2] = (unsigned long)_orig.nraddr;                   \
2049       __asm__ volatile(                                           \
2050          "mr 11,%1\n\t"                                           \
2051          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2052          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2053          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2054          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2055          "mr 11,%1\n\t"                                           \
2056          "mr %0,3\n\t"                                            \
2057          "ld 2,-16(11)" /* restore tocptr */                      \
2058          : /*out*/   "=r" (_res)                                  \
2059          : /*in*/    "r" (&_argvec[2])                            \
2060          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2061       );                                                          \
2062       lval = (__typeof__(lval)) _res;                             \
2063    } while (0)
2064
2065 #define CALL_FN_W_W(lval, orig, arg1)                             \
2066    do {                                                           \
2067       volatile OrigFn        _orig = (orig);                      \
2068       volatile unsigned long _argvec[3+1];                        \
2069       volatile unsigned long _res;                                \
2070       /* _argvec[0] holds current r2 across the call */           \
2071       _argvec[1]   = (unsigned long)_orig.r2;                     \
2072       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2073       _argvec[2+1] = (unsigned long)arg1;                         \
2074       __asm__ volatile(                                           \
2075          "mr 11,%1\n\t"                                           \
2076          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2077          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2078          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2079          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2080          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2081          "mr 11,%1\n\t"                                           \
2082          "mr %0,3\n\t"                                            \
2083          "ld 2,-16(11)" /* restore tocptr */                      \
2084          : /*out*/   "=r" (_res)                                  \
2085          : /*in*/    "r" (&_argvec[2])                            \
2086          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2087       );                                                          \
2088       lval = (__typeof__(lval)) _res;                             \
2089    } while (0)
2090
2091 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2092    do {                                                           \
2093       volatile OrigFn        _orig = (orig);                      \
2094       volatile unsigned long _argvec[3+2];                        \
2095       volatile unsigned long _res;                                \
2096       /* _argvec[0] holds current r2 across the call */           \
2097       _argvec[1]   = (unsigned long)_orig.r2;                     \
2098       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2099       _argvec[2+1] = (unsigned long)arg1;                         \
2100       _argvec[2+2] = (unsigned long)arg2;                         \
2101       __asm__ volatile(                                           \
2102          "mr 11,%1\n\t"                                           \
2103          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2104          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2105          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2106          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2107          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2108          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2109          "mr 11,%1\n\t"                                           \
2110          "mr %0,3\n\t"                                            \
2111          "ld 2,-16(11)" /* restore tocptr */                      \
2112          : /*out*/   "=r" (_res)                                  \
2113          : /*in*/    "r" (&_argvec[2])                            \
2114          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2115       );                                                          \
2116       lval = (__typeof__(lval)) _res;                             \
2117    } while (0)
2118
2119 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2120    do {                                                           \
2121       volatile OrigFn        _orig = (orig);                      \
2122       volatile unsigned long _argvec[3+3];                        \
2123       volatile unsigned long _res;                                \
2124       /* _argvec[0] holds current r2 across the call */           \
2125       _argvec[1]   = (unsigned long)_orig.r2;                     \
2126       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2127       _argvec[2+1] = (unsigned long)arg1;                         \
2128       _argvec[2+2] = (unsigned long)arg2;                         \
2129       _argvec[2+3] = (unsigned long)arg3;                         \
2130       __asm__ volatile(                                           \
2131          "mr 11,%1\n\t"                                           \
2132          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2133          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2134          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2135          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2136          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2137          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2138          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2139          "mr 11,%1\n\t"                                           \
2140          "mr %0,3\n\t"                                            \
2141          "ld 2,-16(11)" /* restore tocptr */                      \
2142          : /*out*/   "=r" (_res)                                  \
2143          : /*in*/    "r" (&_argvec[2])                            \
2144          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2145       );                                                          \
2146       lval = (__typeof__(lval)) _res;                             \
2147    } while (0)
2148
2149 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2150    do {                                                           \
2151       volatile OrigFn        _orig = (orig);                      \
2152       volatile unsigned long _argvec[3+4];                        \
2153       volatile unsigned long _res;                                \
2154       /* _argvec[0] holds current r2 across the call */           \
2155       _argvec[1]   = (unsigned long)_orig.r2;                     \
2156       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2157       _argvec[2+1] = (unsigned long)arg1;                         \
2158       _argvec[2+2] = (unsigned long)arg2;                         \
2159       _argvec[2+3] = (unsigned long)arg3;                         \
2160       _argvec[2+4] = (unsigned long)arg4;                         \
2161       __asm__ volatile(                                           \
2162          "mr 11,%1\n\t"                                           \
2163          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2164          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2165          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2166          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2167          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2168          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2169          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2170          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2171          "mr 11,%1\n\t"                                           \
2172          "mr %0,3\n\t"                                            \
2173          "ld 2,-16(11)" /* restore tocptr */                      \
2174          : /*out*/   "=r" (_res)                                  \
2175          : /*in*/    "r" (&_argvec[2])                            \
2176          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2177       );                                                          \
2178       lval = (__typeof__(lval)) _res;                             \
2179    } while (0)
2180
2181 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2182    do {                                                           \
2183       volatile OrigFn        _orig = (orig);                      \
2184       volatile unsigned long _argvec[3+5];                        \
2185       volatile unsigned long _res;                                \
2186       /* _argvec[0] holds current r2 across the call */           \
2187       _argvec[1]   = (unsigned long)_orig.r2;                     \
2188       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2189       _argvec[2+1] = (unsigned long)arg1;                         \
2190       _argvec[2+2] = (unsigned long)arg2;                         \
2191       _argvec[2+3] = (unsigned long)arg3;                         \
2192       _argvec[2+4] = (unsigned long)arg4;                         \
2193       _argvec[2+5] = (unsigned long)arg5;                         \
2194       __asm__ volatile(                                           \
2195          "mr 11,%1\n\t"                                           \
2196          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2197          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2198          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2199          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2200          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2201          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2202          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2203          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2204          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2205          "mr 11,%1\n\t"                                           \
2206          "mr %0,3\n\t"                                            \
2207          "ld 2,-16(11)" /* restore tocptr */                      \
2208          : /*out*/   "=r" (_res)                                  \
2209          : /*in*/    "r" (&_argvec[2])                            \
2210          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2211       );                                                          \
2212       lval = (__typeof__(lval)) _res;                             \
2213    } while (0)
2214
2215 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2216    do {                                                           \
2217       volatile OrigFn        _orig = (orig);                      \
2218       volatile unsigned long _argvec[3+6];                        \
2219       volatile unsigned long _res;                                \
2220       /* _argvec[0] holds current r2 across the call */           \
2221       _argvec[1]   = (unsigned long)_orig.r2;                     \
2222       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2223       _argvec[2+1] = (unsigned long)arg1;                         \
2224       _argvec[2+2] = (unsigned long)arg2;                         \
2225       _argvec[2+3] = (unsigned long)arg3;                         \
2226       _argvec[2+4] = (unsigned long)arg4;                         \
2227       _argvec[2+5] = (unsigned long)arg5;                         \
2228       _argvec[2+6] = (unsigned long)arg6;                         \
2229       __asm__ volatile(                                           \
2230          "mr 11,%1\n\t"                                           \
2231          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2232          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2233          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2234          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2235          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2236          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2237          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2238          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2239          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2240          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2241          "mr 11,%1\n\t"                                           \
2242          "mr %0,3\n\t"                                            \
2243          "ld 2,-16(11)" /* restore tocptr */                      \
2244          : /*out*/   "=r" (_res)                                  \
2245          : /*in*/    "r" (&_argvec[2])                            \
2246          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2247       );                                                          \
2248       lval = (__typeof__(lval)) _res;                             \
2249    } while (0)
2250
2251 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2252                                  arg7)                            \
2253    do {                                                           \
2254       volatile OrigFn        _orig = (orig);                      \
2255       volatile unsigned long _argvec[3+7];                        \
2256       volatile unsigned long _res;                                \
2257       /* _argvec[0] holds current r2 across the call */           \
2258       _argvec[1]   = (unsigned long)_orig.r2;                     \
2259       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2260       _argvec[2+1] = (unsigned long)arg1;                         \
2261       _argvec[2+2] = (unsigned long)arg2;                         \
2262       _argvec[2+3] = (unsigned long)arg3;                         \
2263       _argvec[2+4] = (unsigned long)arg4;                         \
2264       _argvec[2+5] = (unsigned long)arg5;                         \
2265       _argvec[2+6] = (unsigned long)arg6;                         \
2266       _argvec[2+7] = (unsigned long)arg7;                         \
2267       __asm__ volatile(                                           \
2268          "mr 11,%1\n\t"                                           \
2269          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2270          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2271          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2272          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2273          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2274          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2275          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2276          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2277          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2278          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2279          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2280          "mr 11,%1\n\t"                                           \
2281          "mr %0,3\n\t"                                            \
2282          "ld 2,-16(11)" /* restore tocptr */                      \
2283          : /*out*/   "=r" (_res)                                  \
2284          : /*in*/    "r" (&_argvec[2])                            \
2285          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2286       );                                                          \
2287       lval = (__typeof__(lval)) _res;                             \
2288    } while (0)
2289
2290 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2291                                  arg7,arg8)                       \
2292    do {                                                           \
2293       volatile OrigFn        _orig = (orig);                      \
2294       volatile unsigned long _argvec[3+8];                        \
2295       volatile unsigned long _res;                                \
2296       /* _argvec[0] holds current r2 across the call */           \
2297       _argvec[1]   = (unsigned long)_orig.r2;                     \
2298       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2299       _argvec[2+1] = (unsigned long)arg1;                         \
2300       _argvec[2+2] = (unsigned long)arg2;                         \
2301       _argvec[2+3] = (unsigned long)arg3;                         \
2302       _argvec[2+4] = (unsigned long)arg4;                         \
2303       _argvec[2+5] = (unsigned long)arg5;                         \
2304       _argvec[2+6] = (unsigned long)arg6;                         \
2305       _argvec[2+7] = (unsigned long)arg7;                         \
2306       _argvec[2+8] = (unsigned long)arg8;                         \
2307       __asm__ volatile(                                           \
2308          "mr 11,%1\n\t"                                           \
2309          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2310          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2311          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2312          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2313          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2314          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2315          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2316          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2317          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2318          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2319          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2320          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2321          "mr 11,%1\n\t"                                           \
2322          "mr %0,3\n\t"                                            \
2323          "ld 2,-16(11)" /* restore tocptr */                      \
2324          : /*out*/   "=r" (_res)                                  \
2325          : /*in*/    "r" (&_argvec[2])                            \
2326          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2327       );                                                          \
2328       lval = (__typeof__(lval)) _res;                             \
2329    } while (0)
2330
2331 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2332                                  arg7,arg8,arg9)                  \
2333    do {                                                           \
2334       volatile OrigFn        _orig = (orig);                      \
2335       volatile unsigned long _argvec[3+9];                        \
2336       volatile unsigned long _res;                                \
2337       /* _argvec[0] holds current r2 across the call */           \
2338       _argvec[1]   = (unsigned long)_orig.r2;                     \
2339       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2340       _argvec[2+1] = (unsigned long)arg1;                         \
2341       _argvec[2+2] = (unsigned long)arg2;                         \
2342       _argvec[2+3] = (unsigned long)arg3;                         \
2343       _argvec[2+4] = (unsigned long)arg4;                         \
2344       _argvec[2+5] = (unsigned long)arg5;                         \
2345       _argvec[2+6] = (unsigned long)arg6;                         \
2346       _argvec[2+7] = (unsigned long)arg7;                         \
2347       _argvec[2+8] = (unsigned long)arg8;                         \
2348       _argvec[2+9] = (unsigned long)arg9;                         \
2349       __asm__ volatile(                                           \
2350          "mr 11,%1\n\t"                                           \
2351          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2352          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2353          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2354          /* arg9 */                                               \
2355          "ld  3,72(11)\n\t"                                       \
2356          "std 3,112(1)\n\t"                                       \
2357          /* args1-8 */                                            \
2358          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2359          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2360          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2361          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2362          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2363          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2364          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2365          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2366          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2367          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2368          "mr 11,%1\n\t"                                           \
2369          "mr %0,3\n\t"                                            \
2370          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2371          "addi 1,1,128"     /* restore frame */                   \
2372          : /*out*/   "=r" (_res)                                  \
2373          : /*in*/    "r" (&_argvec[2])                            \
2374          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2375       );                                                          \
2376       lval = (__typeof__(lval)) _res;                             \
2377    } while (0)
2378
2379 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2380                                   arg7,arg8,arg9,arg10)           \
2381    do {                                                           \
2382       volatile OrigFn        _orig = (orig);                      \
2383       volatile unsigned long _argvec[3+10];                       \
2384       volatile unsigned long _res;                                \
2385       /* _argvec[0] holds current r2 across the call */           \
2386       _argvec[1]   = (unsigned long)_orig.r2;                     \
2387       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2388       _argvec[2+1] = (unsigned long)arg1;                         \
2389       _argvec[2+2] = (unsigned long)arg2;                         \
2390       _argvec[2+3] = (unsigned long)arg3;                         \
2391       _argvec[2+4] = (unsigned long)arg4;                         \
2392       _argvec[2+5] = (unsigned long)arg5;                         \
2393       _argvec[2+6] = (unsigned long)arg6;                         \
2394       _argvec[2+7] = (unsigned long)arg7;                         \
2395       _argvec[2+8] = (unsigned long)arg8;                         \
2396       _argvec[2+9] = (unsigned long)arg9;                         \
2397       _argvec[2+10] = (unsigned long)arg10;                       \
2398       __asm__ volatile(                                           \
2399          "mr 11,%1\n\t"                                           \
2400          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2401          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2402          "addi 1,1,-128\n\t"  /* expand stack frame */            \
2403          /* arg10 */                                              \
2404          "ld  3,80(11)\n\t"                                       \
2405          "std 3,120(1)\n\t"                                       \
2406          /* arg9 */                                               \
2407          "ld  3,72(11)\n\t"                                       \
2408          "std 3,112(1)\n\t"                                       \
2409          /* args1-8 */                                            \
2410          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2411          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2412          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2413          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2414          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2415          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2416          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2417          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2418          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2419          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2420          "mr 11,%1\n\t"                                           \
2421          "mr %0,3\n\t"                                            \
2422          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2423          "addi 1,1,128"     /* restore frame */                   \
2424          : /*out*/   "=r" (_res)                                  \
2425          : /*in*/    "r" (&_argvec[2])                            \
2426          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2427       );                                                          \
2428       lval = (__typeof__(lval)) _res;                             \
2429    } while (0)
2430
2431 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2432                                   arg7,arg8,arg9,arg10,arg11)     \
2433    do {                                                           \
2434       volatile OrigFn        _orig = (orig);                      \
2435       volatile unsigned long _argvec[3+11];                       \
2436       volatile unsigned long _res;                                \
2437       /* _argvec[0] holds current r2 across the call */           \
2438       _argvec[1]   = (unsigned long)_orig.r2;                     \
2439       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2440       _argvec[2+1] = (unsigned long)arg1;                         \
2441       _argvec[2+2] = (unsigned long)arg2;                         \
2442       _argvec[2+3] = (unsigned long)arg3;                         \
2443       _argvec[2+4] = (unsigned long)arg4;                         \
2444       _argvec[2+5] = (unsigned long)arg5;                         \
2445       _argvec[2+6] = (unsigned long)arg6;                         \
2446       _argvec[2+7] = (unsigned long)arg7;                         \
2447       _argvec[2+8] = (unsigned long)arg8;                         \
2448       _argvec[2+9] = (unsigned long)arg9;                         \
2449       _argvec[2+10] = (unsigned long)arg10;                       \
2450       _argvec[2+11] = (unsigned long)arg11;                       \
2451       __asm__ volatile(                                           \
2452          "mr 11,%1\n\t"                                           \
2453          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2454          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2455          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2456          /* arg11 */                                              \
2457          "ld  3,88(11)\n\t"                                       \
2458          "std 3,128(1)\n\t"                                       \
2459          /* arg10 */                                              \
2460          "ld  3,80(11)\n\t"                                       \
2461          "std 3,120(1)\n\t"                                       \
2462          /* arg9 */                                               \
2463          "ld  3,72(11)\n\t"                                       \
2464          "std 3,112(1)\n\t"                                       \
2465          /* args1-8 */                                            \
2466          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2467          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2468          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2469          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2470          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2471          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2472          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2473          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2474          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2475          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2476          "mr 11,%1\n\t"                                           \
2477          "mr %0,3\n\t"                                            \
2478          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2479          "addi 1,1,144"     /* restore frame */                   \
2480          : /*out*/   "=r" (_res)                                  \
2481          : /*in*/    "r" (&_argvec[2])                            \
2482          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2483       );                                                          \
2484       lval = (__typeof__(lval)) _res;                             \
2485    } while (0)
2486
2487 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2488                                 arg7,arg8,arg9,arg10,arg11,arg12) \
2489    do {                                                           \
2490       volatile OrigFn        _orig = (orig);                      \
2491       volatile unsigned long _argvec[3+12];                       \
2492       volatile unsigned long _res;                                \
2493       /* _argvec[0] holds current r2 across the call */           \
2494       _argvec[1]   = (unsigned long)_orig.r2;                     \
2495       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2496       _argvec[2+1] = (unsigned long)arg1;                         \
2497       _argvec[2+2] = (unsigned long)arg2;                         \
2498       _argvec[2+3] = (unsigned long)arg3;                         \
2499       _argvec[2+4] = (unsigned long)arg4;                         \
2500       _argvec[2+5] = (unsigned long)arg5;                         \
2501       _argvec[2+6] = (unsigned long)arg6;                         \
2502       _argvec[2+7] = (unsigned long)arg7;                         \
2503       _argvec[2+8] = (unsigned long)arg8;                         \
2504       _argvec[2+9] = (unsigned long)arg9;                         \
2505       _argvec[2+10] = (unsigned long)arg10;                       \
2506       _argvec[2+11] = (unsigned long)arg11;                       \
2507       _argvec[2+12] = (unsigned long)arg12;                       \
2508       __asm__ volatile(                                           \
2509          "mr 11,%1\n\t"                                           \
2510          "std 2,-16(11)\n\t"  /* save tocptr */                   \
2511          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2512          "addi 1,1,-144\n\t"  /* expand stack frame */            \
2513          /* arg12 */                                              \
2514          "ld  3,96(11)\n\t"                                       \
2515          "std 3,136(1)\n\t"                                       \
2516          /* arg11 */                                              \
2517          "ld  3,88(11)\n\t"                                       \
2518          "std 3,128(1)\n\t"                                       \
2519          /* arg10 */                                              \
2520          "ld  3,80(11)\n\t"                                       \
2521          "std 3,120(1)\n\t"                                       \
2522          /* arg9 */                                               \
2523          "ld  3,72(11)\n\t"                                       \
2524          "std 3,112(1)\n\t"                                       \
2525          /* args1-8 */                                            \
2526          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2527          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2528          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2529          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2530          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2531          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2532          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2533          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2534          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2535          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2536          "mr 11,%1\n\t"                                           \
2537          "mr %0,3\n\t"                                            \
2538          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2539          "addi 1,1,144"     /* restore frame */                   \
2540          : /*out*/   "=r" (_res)                                  \
2541          : /*in*/    "r" (&_argvec[2])                            \
2542          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2543       );                                                          \
2544       lval = (__typeof__(lval)) _res;                             \
2545    } while (0)
2546
2547 #endif /* PLAT_ppc64_linux */
2548
2549 /* ------------------------- arm-linux ------------------------- */
2550
2551 #if defined(PLAT_arm_linux)
2552
2553 /* These regs are trashed by the hidden call. */
2554 #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4","r14"
2555
2556 /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
2557    long) == 4. */
2558
2559 #define CALL_FN_W_v(lval, orig)                                   \
2560    do {                                                           \
2561       volatile OrigFn        _orig = (orig);                      \
2562       volatile unsigned long _argvec[1];                          \
2563       volatile unsigned long _res;                                \
2564       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2565       __asm__ volatile(                                           \
2566          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2567          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2568          "mov %0, r0\n"                                           \
2569          : /*out*/   "=r" (_res)                                  \
2570          : /*in*/    "0" (&_argvec[0])                            \
2571          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2572       );                                                          \
2573       lval = (__typeof__(lval)) _res;                             \
2574    } while (0)
2575
2576 #define CALL_FN_W_W(lval, orig, arg1)                             \
2577    do {                                                           \
2578       volatile OrigFn        _orig = (orig);                      \
2579       volatile unsigned long _argvec[2];                          \
2580       volatile unsigned long _res;                                \
2581       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2582       _argvec[1] = (unsigned long)(arg1);                         \
2583       __asm__ volatile(                                           \
2584          "ldr r0, [%1, #4] \n\t"                                  \
2585          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2586          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2587          "mov %0, r0\n"                                           \
2588          : /*out*/   "=r" (_res)                                  \
2589          : /*in*/    "0" (&_argvec[0])                            \
2590          : /*trash*/ "cc", "memory",  __CALLER_SAVED_REGS         \
2591       );                                                          \
2592       lval = (__typeof__(lval)) _res;                             \
2593    } while (0)
2594
2595 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2596    do {                                                           \
2597       volatile OrigFn        _orig = (orig);                      \
2598       volatile unsigned long _argvec[3];                          \
2599       volatile unsigned long _res;                                \
2600       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2601       _argvec[1] = (unsigned long)(arg1);                         \
2602       _argvec[2] = (unsigned long)(arg2);                         \
2603       __asm__ volatile(                                           \
2604          "ldr r0, [%1, #4] \n\t"                                  \
2605          "ldr r1, [%1, #8] \n\t"                                  \
2606          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2607          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2608          "mov %0, r0\n"                                           \
2609          : /*out*/   "=r" (_res)                                  \
2610          : /*in*/    "0" (&_argvec[0])                            \
2611          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2612       );                                                          \
2613       lval = (__typeof__(lval)) _res;                             \
2614    } while (0)
2615
2616 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2617    do {                                                           \
2618       volatile OrigFn        _orig = (orig);                      \
2619       volatile unsigned long _argvec[4];                          \
2620       volatile unsigned long _res;                                \
2621       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2622       _argvec[1] = (unsigned long)(arg1);                         \
2623       _argvec[2] = (unsigned long)(arg2);                         \
2624       _argvec[3] = (unsigned long)(arg3);                         \
2625       __asm__ volatile(                                           \
2626          "ldr r0, [%1, #4] \n\t"                                  \
2627          "ldr r1, [%1, #8] \n\t"                                  \
2628          "ldr r2, [%1, #12] \n\t"                                 \
2629          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2630          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2631          "mov %0, r0\n"                                           \
2632          : /*out*/   "=r" (_res)                                  \
2633          : /*in*/    "0" (&_argvec[0])                            \
2634          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2635       );                                                          \
2636       lval = (__typeof__(lval)) _res;                             \
2637    } while (0)
2638
2639 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2640    do {                                                           \
2641       volatile OrigFn        _orig = (orig);                      \
2642       volatile unsigned long _argvec[5];                          \
2643       volatile unsigned long _res;                                \
2644       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2645       _argvec[1] = (unsigned long)(arg1);                         \
2646       _argvec[2] = (unsigned long)(arg2);                         \
2647       _argvec[3] = (unsigned long)(arg3);                         \
2648       _argvec[4] = (unsigned long)(arg4);                         \
2649       __asm__ volatile(                                           \
2650          "ldr r0, [%1, #4] \n\t"                                  \
2651          "ldr r1, [%1, #8] \n\t"                                  \
2652          "ldr r2, [%1, #12] \n\t"                                 \
2653          "ldr r3, [%1, #16] \n\t"                                 \
2654          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2655          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2656          "mov %0, r0"                                             \
2657          : /*out*/   "=r" (_res)                                  \
2658          : /*in*/    "0" (&_argvec[0])                            \
2659          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2660       );                                                          \
2661       lval = (__typeof__(lval)) _res;                             \
2662    } while (0)
2663
2664 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2665    do {                                                           \
2666       volatile OrigFn        _orig = (orig);                      \
2667       volatile unsigned long _argvec[6];                          \
2668       volatile unsigned long _res;                                \
2669       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2670       _argvec[1] = (unsigned long)(arg1);                         \
2671       _argvec[2] = (unsigned long)(arg2);                         \
2672       _argvec[3] = (unsigned long)(arg3);                         \
2673       _argvec[4] = (unsigned long)(arg4);                         \
2674       _argvec[5] = (unsigned long)(arg5);                         \
2675       __asm__ volatile(                                           \
2676          "ldr r0, [%1, #20] \n\t"                                 \
2677          "push {r0} \n\t"                                         \
2678          "ldr r0, [%1, #4] \n\t"                                  \
2679          "ldr r1, [%1, #8] \n\t"                                  \
2680          "ldr r2, [%1, #12] \n\t"                                 \
2681          "ldr r3, [%1, #16] \n\t"                                 \
2682          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2683          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2684          "add sp, sp, #4 \n\t"                                    \
2685          "mov %0, r0"                                             \
2686          : /*out*/   "=r" (_res)                                  \
2687          : /*in*/    "0" (&_argvec[0])                            \
2688          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2689       );                                                          \
2690       lval = (__typeof__(lval)) _res;                             \
2691    } while (0)
2692
2693 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2694    do {                                                           \
2695       volatile OrigFn        _orig = (orig);                      \
2696       volatile unsigned long _argvec[7];                          \
2697       volatile unsigned long _res;                                \
2698       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2699       _argvec[1] = (unsigned long)(arg1);                         \
2700       _argvec[2] = (unsigned long)(arg2);                         \
2701       _argvec[3] = (unsigned long)(arg3);                         \
2702       _argvec[4] = (unsigned long)(arg4);                         \
2703       _argvec[5] = (unsigned long)(arg5);                         \
2704       _argvec[6] = (unsigned long)(arg6);                         \
2705       __asm__ volatile(                                           \
2706          "ldr r0, [%1, #20] \n\t"                                 \
2707          "ldr r1, [%1, #24] \n\t"                                 \
2708          "push {r0, r1} \n\t"                                     \
2709          "ldr r0, [%1, #4] \n\t"                                  \
2710          "ldr r1, [%1, #8] \n\t"                                  \
2711          "ldr r2, [%1, #12] \n\t"                                 \
2712          "ldr r3, [%1, #16] \n\t"                                 \
2713          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2714          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2715          "add sp, sp, #8 \n\t"                                    \
2716          "mov %0, r0"                                             \
2717          : /*out*/   "=r" (_res)                                  \
2718          : /*in*/    "0" (&_argvec[0])                            \
2719          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2720       );                                                          \
2721       lval = (__typeof__(lval)) _res;                             \
2722    } while (0)
2723
2724 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2725                                  arg7)                            \
2726    do {                                                           \
2727       volatile OrigFn        _orig = (orig);                      \
2728       volatile unsigned long _argvec[8];                          \
2729       volatile unsigned long _res;                                \
2730       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2731       _argvec[1] = (unsigned long)(arg1);                         \
2732       _argvec[2] = (unsigned long)(arg2);                         \
2733       _argvec[3] = (unsigned long)(arg3);                         \
2734       _argvec[4] = (unsigned long)(arg4);                         \
2735       _argvec[5] = (unsigned long)(arg5);                         \
2736       _argvec[6] = (unsigned long)(arg6);                         \
2737       _argvec[7] = (unsigned long)(arg7);                         \
2738       __asm__ volatile(                                           \
2739          "ldr r0, [%1, #20] \n\t"                                 \
2740          "ldr r1, [%1, #24] \n\t"                                 \
2741          "ldr r2, [%1, #28] \n\t"                                 \
2742          "push {r0, r1, r2} \n\t"                                 \
2743          "ldr r0, [%1, #4] \n\t"                                  \
2744          "ldr r1, [%1, #8] \n\t"                                  \
2745          "ldr r2, [%1, #12] \n\t"                                 \
2746          "ldr r3, [%1, #16] \n\t"                                 \
2747          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2748          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2749          "add sp, sp, #12 \n\t"                                   \
2750          "mov %0, r0"                                             \
2751          : /*out*/   "=r" (_res)                                  \
2752          : /*in*/    "0" (&_argvec[0])                            \
2753          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2754       );                                                          \
2755       lval = (__typeof__(lval)) _res;                             \
2756    } while (0)
2757
2758 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2759                                  arg7,arg8)                       \
2760    do {                                                           \
2761       volatile OrigFn        _orig = (orig);                      \
2762       volatile unsigned long _argvec[9];                          \
2763       volatile unsigned long _res;                                \
2764       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2765       _argvec[1] = (unsigned long)(arg1);                         \
2766       _argvec[2] = (unsigned long)(arg2);                         \
2767       _argvec[3] = (unsigned long)(arg3);                         \
2768       _argvec[4] = (unsigned long)(arg4);                         \
2769       _argvec[5] = (unsigned long)(arg5);                         \
2770       _argvec[6] = (unsigned long)(arg6);                         \
2771       _argvec[7] = (unsigned long)(arg7);                         \
2772       _argvec[8] = (unsigned long)(arg8);                         \
2773       __asm__ volatile(                                           \
2774          "ldr r0, [%1, #20] \n\t"                                 \
2775          "ldr r1, [%1, #24] \n\t"                                 \
2776          "ldr r2, [%1, #28] \n\t"                                 \
2777          "ldr r3, [%1, #32] \n\t"                                 \
2778          "push {r0, r1, r2, r3} \n\t"                             \
2779          "ldr r0, [%1, #4] \n\t"                                  \
2780          "ldr r1, [%1, #8] \n\t"                                  \
2781          "ldr r2, [%1, #12] \n\t"                                 \
2782          "ldr r3, [%1, #16] \n\t"                                 \
2783          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2784          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2785          "add sp, sp, #16 \n\t"                                   \
2786          "mov %0, r0"                                             \
2787          : /*out*/   "=r" (_res)                                  \
2788          : /*in*/    "0" (&_argvec[0])                            \
2789          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2790       );                                                          \
2791       lval = (__typeof__(lval)) _res;                             \
2792    } while (0)
2793
2794 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2795                                  arg7,arg8,arg9)                  \
2796    do {                                                           \
2797       volatile OrigFn        _orig = (orig);                      \
2798       volatile unsigned long _argvec[10];                         \
2799       volatile unsigned long _res;                                \
2800       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2801       _argvec[1] = (unsigned long)(arg1);                         \
2802       _argvec[2] = (unsigned long)(arg2);                         \
2803       _argvec[3] = (unsigned long)(arg3);                         \
2804       _argvec[4] = (unsigned long)(arg4);                         \
2805       _argvec[5] = (unsigned long)(arg5);                         \
2806       _argvec[6] = (unsigned long)(arg6);                         \
2807       _argvec[7] = (unsigned long)(arg7);                         \
2808       _argvec[8] = (unsigned long)(arg8);                         \
2809       _argvec[9] = (unsigned long)(arg9);                         \
2810       __asm__ volatile(                                           \
2811          "ldr r0, [%1, #20] \n\t"                                 \
2812          "ldr r1, [%1, #24] \n\t"                                 \
2813          "ldr r2, [%1, #28] \n\t"                                 \
2814          "ldr r3, [%1, #32] \n\t"                                 \
2815          "ldr r4, [%1, #36] \n\t"                                 \
2816          "push {r0, r1, r2, r3, r4} \n\t"                         \
2817          "ldr r0, [%1, #4] \n\t"                                  \
2818          "ldr r1, [%1, #8] \n\t"                                  \
2819          "ldr r2, [%1, #12] \n\t"                                 \
2820          "ldr r3, [%1, #16] \n\t"                                 \
2821          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2822          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2823          "add sp, sp, #20 \n\t"                                   \
2824          "mov %0, r0"                                             \
2825          : /*out*/   "=r" (_res)                                  \
2826          : /*in*/    "0" (&_argvec[0])                            \
2827          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2828       );                                                          \
2829       lval = (__typeof__(lval)) _res;                             \
2830    } while (0)
2831
2832 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2833                                   arg7,arg8,arg9,arg10)           \
2834    do {                                                           \
2835       volatile OrigFn        _orig = (orig);                      \
2836       volatile unsigned long _argvec[11];                         \
2837       volatile unsigned long _res;                                \
2838       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2839       _argvec[1] = (unsigned long)(arg1);                         \
2840       _argvec[2] = (unsigned long)(arg2);                         \
2841       _argvec[3] = (unsigned long)(arg3);                         \
2842       _argvec[4] = (unsigned long)(arg4);                         \
2843       _argvec[5] = (unsigned long)(arg5);                         \
2844       _argvec[6] = (unsigned long)(arg6);                         \
2845       _argvec[7] = (unsigned long)(arg7);                         \
2846       _argvec[8] = (unsigned long)(arg8);                         \
2847       _argvec[9] = (unsigned long)(arg9);                         \
2848       _argvec[10] = (unsigned long)(arg10);                       \
2849       __asm__ volatile(                                           \
2850          "ldr r0, [%1, #40] \n\t"                                 \
2851          "push {r0} \n\t"                                         \
2852          "ldr r0, [%1, #20] \n\t"                                 \
2853          "ldr r1, [%1, #24] \n\t"                                 \
2854          "ldr r2, [%1, #28] \n\t"                                 \
2855          "ldr r3, [%1, #32] \n\t"                                 \
2856          "ldr r4, [%1, #36] \n\t"                                 \
2857          "push {r0, r1, r2, r3, r4} \n\t"                         \
2858          "ldr r0, [%1, #4] \n\t"                                  \
2859          "ldr r1, [%1, #8] \n\t"                                  \
2860          "ldr r2, [%1, #12] \n\t"                                 \
2861          "ldr r3, [%1, #16] \n\t"                                 \
2862          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2863          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2864          "add sp, sp, #24 \n\t"                                   \
2865          "mov %0, r0"                                             \
2866          : /*out*/   "=r" (_res)                                  \
2867          : /*in*/    "0" (&_argvec[0])                            \
2868          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2869       );                                                          \
2870       lval = (__typeof__(lval)) _res;                             \
2871    } while (0)
2872
2873 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
2874                                   arg6,arg7,arg8,arg9,arg10,      \
2875                                   arg11)                          \
2876    do {                                                           \
2877       volatile OrigFn        _orig = (orig);                      \
2878       volatile unsigned long _argvec[12];                         \
2879       volatile unsigned long _res;                                \
2880       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2881       _argvec[1] = (unsigned long)(arg1);                         \
2882       _argvec[2] = (unsigned long)(arg2);                         \
2883       _argvec[3] = (unsigned long)(arg3);                         \
2884       _argvec[4] = (unsigned long)(arg4);                         \
2885       _argvec[5] = (unsigned long)(arg5);                         \
2886       _argvec[6] = (unsigned long)(arg6);                         \
2887       _argvec[7] = (unsigned long)(arg7);                         \
2888       _argvec[8] = (unsigned long)(arg8);                         \
2889       _argvec[9] = (unsigned long)(arg9);                         \
2890       _argvec[10] = (unsigned long)(arg10);                       \
2891       _argvec[11] = (unsigned long)(arg11);                       \
2892       __asm__ volatile(                                           \
2893          "ldr r0, [%1, #40] \n\t"                                 \
2894          "ldr r1, [%1, #44] \n\t"                                 \
2895          "push {r0, r1} \n\t"                                     \
2896          "ldr r0, [%1, #20] \n\t"                                 \
2897          "ldr r1, [%1, #24] \n\t"                                 \
2898          "ldr r2, [%1, #28] \n\t"                                 \
2899          "ldr r3, [%1, #32] \n\t"                                 \
2900          "ldr r4, [%1, #36] \n\t"                                 \
2901          "push {r0, r1, r2, r3, r4} \n\t"                         \
2902          "ldr r0, [%1, #4] \n\t"                                  \
2903          "ldr r1, [%1, #8] \n\t"                                  \
2904          "ldr r2, [%1, #12] \n\t"                                 \
2905          "ldr r3, [%1, #16] \n\t"                                 \
2906          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2907          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2908          "add sp, sp, #28 \n\t"                                   \
2909          "mov %0, r0"                                             \
2910          : /*out*/   "=r" (_res)                                  \
2911          : /*in*/    "0" (&_argvec[0])                            \
2912          : /*trash*/ "cc", "memory",__CALLER_SAVED_REGS           \
2913       );                                                          \
2914       lval = (__typeof__(lval)) _res;                             \
2915    } while (0)
2916
2917 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
2918                                   arg6,arg7,arg8,arg9,arg10,      \
2919                                   arg11,arg12)                    \
2920    do {                                                           \
2921       volatile OrigFn        _orig = (orig);                      \
2922       volatile unsigned long _argvec[13];                         \
2923       volatile unsigned long _res;                                \
2924       _argvec[0] = (unsigned long)_orig.nraddr;                   \
2925       _argvec[1] = (unsigned long)(arg1);                         \
2926       _argvec[2] = (unsigned long)(arg2);                         \
2927       _argvec[3] = (unsigned long)(arg3);                         \
2928       _argvec[4] = (unsigned long)(arg4);                         \
2929       _argvec[5] = (unsigned long)(arg5);                         \
2930       _argvec[6] = (unsigned long)(arg6);                         \
2931       _argvec[7] = (unsigned long)(arg7);                         \
2932       _argvec[8] = (unsigned long)(arg8);                         \
2933       _argvec[9] = (unsigned long)(arg9);                         \
2934       _argvec[10] = (unsigned long)(arg10);                       \
2935       _argvec[11] = (unsigned long)(arg11);                       \
2936       _argvec[12] = (unsigned long)(arg12);                       \
2937       __asm__ volatile(                                           \
2938          "ldr r0, [%1, #40] \n\t"                                 \
2939          "ldr r1, [%1, #44] \n\t"                                 \
2940          "ldr r2, [%1, #48] \n\t"                                 \
2941          "push {r0, r1, r2} \n\t"                                 \
2942          "ldr r0, [%1, #20] \n\t"                                 \
2943          "ldr r1, [%1, #24] \n\t"                                 \
2944          "ldr r2, [%1, #28] \n\t"                                 \
2945          "ldr r3, [%1, #32] \n\t"                                 \
2946          "ldr r4, [%1, #36] \n\t"                                 \
2947          "push {r0, r1, r2, r3, r4} \n\t"                         \
2948          "ldr r0, [%1, #4] \n\t"                                  \
2949          "ldr r1, [%1, #8] \n\t"                                  \
2950          "ldr r2, [%1, #12] \n\t"                                 \
2951          "ldr r3, [%1, #16] \n\t"                                 \
2952          "ldr r4, [%1] \n\t"  /* target->r4 */                    \
2953          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
2954          "add sp, sp, #32 \n\t"                                   \
2955          "mov %0, r0"                                             \
2956          : /*out*/   "=r" (_res)                                  \
2957          : /*in*/    "0" (&_argvec[0])                            \
2958          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
2959       );                                                          \
2960       lval = (__typeof__(lval)) _res;                             \
2961    } while (0)
2962
2963 #endif /* PLAT_arm_linux */
2964
2965 /* ------------------------ ppc32-aix5 ------------------------- */
2966
2967 #if defined(PLAT_ppc32_aix5)
2968
2969 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2970
2971 /* These regs are trashed by the hidden call. */
2972 #define __CALLER_SAVED_REGS                                       \
2973    "lr", "ctr", "xer",                                            \
2974    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2975    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2976    "r11", "r12", "r13"
2977
2978 /* Expand the stack frame, copying enough info that unwinding
2979    still works.  Trashes r3. */
2980
2981 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
2982          "addi 1,1,-" #_n_fr "\n\t"                               \
2983          "lwz  3," #_n_fr "(1)\n\t"                               \
2984          "stw  3,0(1)\n\t"
2985
2986 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
2987          "addi 1,1," #_n_fr "\n\t"
2988
2989 /* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
2990    long) == 4. */
2991
2992 #define CALL_FN_W_v(lval, orig)                                   \
2993    do {                                                           \
2994       volatile OrigFn        _orig = (orig);                      \
2995       volatile unsigned long _argvec[3+0];                        \
2996       volatile unsigned long _res;                                \
2997       /* _argvec[0] holds current r2 across the call */           \
2998       _argvec[1] = (unsigned long)_orig.r2;                       \
2999       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3000       __asm__ volatile(                                           \
3001          "mr 11,%1\n\t"                                           \
3002          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3003          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3004          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3005          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3006          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3007          "mr 11,%1\n\t"                                           \
3008          "mr %0,3\n\t"                                            \
3009          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3010          VG_CONTRACT_FRAME_BY(512)                                \
3011          : /*out*/   "=r" (_res)                                  \
3012          : /*in*/    "r" (&_argvec[2])                            \
3013          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3014       );                                                          \
3015       lval = (__typeof__(lval)) _res;                             \
3016    } while (0)
3017
3018 #define CALL_FN_W_W(lval, orig, arg1)                             \
3019    do {                                                           \
3020       volatile OrigFn        _orig = (orig);                      \
3021       volatile unsigned long _argvec[3+1];                        \
3022       volatile unsigned long _res;                                \
3023       /* _argvec[0] holds current r2 across the call */           \
3024       _argvec[1]   = (unsigned long)_orig.r2;                     \
3025       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3026       _argvec[2+1] = (unsigned long)arg1;                         \
3027       __asm__ volatile(                                           \
3028          "mr 11,%1\n\t"                                           \
3029          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3030          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3031          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3032          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3033          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3034          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3035          "mr 11,%1\n\t"                                           \
3036          "mr %0,3\n\t"                                            \
3037          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3038          VG_CONTRACT_FRAME_BY(512)                                \
3039          : /*out*/   "=r" (_res)                                  \
3040          : /*in*/    "r" (&_argvec[2])                            \
3041          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3042       );                                                          \
3043       lval = (__typeof__(lval)) _res;                             \
3044    } while (0)
3045
3046 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3047    do {                                                           \
3048       volatile OrigFn        _orig = (orig);                      \
3049       volatile unsigned long _argvec[3+2];                        \
3050       volatile unsigned long _res;                                \
3051       /* _argvec[0] holds current r2 across the call */           \
3052       _argvec[1]   = (unsigned long)_orig.r2;                     \
3053       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3054       _argvec[2+1] = (unsigned long)arg1;                         \
3055       _argvec[2+2] = (unsigned long)arg2;                         \
3056       __asm__ volatile(                                           \
3057          "mr 11,%1\n\t"                                           \
3058          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3059          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3060          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3061          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3062          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3063          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3064          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3065          "mr 11,%1\n\t"                                           \
3066          "mr %0,3\n\t"                                            \
3067          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3068          VG_CONTRACT_FRAME_BY(512)                                \
3069          : /*out*/   "=r" (_res)                                  \
3070          : /*in*/    "r" (&_argvec[2])                            \
3071          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3072       );                                                          \
3073       lval = (__typeof__(lval)) _res;                             \
3074    } while (0)
3075
3076 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3077    do {                                                           \
3078       volatile OrigFn        _orig = (orig);                      \
3079       volatile unsigned long _argvec[3+3];                        \
3080       volatile unsigned long _res;                                \
3081       /* _argvec[0] holds current r2 across the call */           \
3082       _argvec[1]   = (unsigned long)_orig.r2;                     \
3083       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3084       _argvec[2+1] = (unsigned long)arg1;                         \
3085       _argvec[2+2] = (unsigned long)arg2;                         \
3086       _argvec[2+3] = (unsigned long)arg3;                         \
3087       __asm__ volatile(                                           \
3088          "mr 11,%1\n\t"                                           \
3089          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3090          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3091          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3092          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3093          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3094          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3095          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3096          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3097          "mr 11,%1\n\t"                                           \
3098          "mr %0,3\n\t"                                            \
3099          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3100          VG_CONTRACT_FRAME_BY(512)                                \
3101          : /*out*/   "=r" (_res)                                  \
3102          : /*in*/    "r" (&_argvec[2])                            \
3103          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3104       );                                                          \
3105       lval = (__typeof__(lval)) _res;                             \
3106    } while (0)
3107
3108 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3109    do {                                                           \
3110       volatile OrigFn        _orig = (orig);                      \
3111       volatile unsigned long _argvec[3+4];                        \
3112       volatile unsigned long _res;                                \
3113       /* _argvec[0] holds current r2 across the call */           \
3114       _argvec[1]   = (unsigned long)_orig.r2;                     \
3115       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3116       _argvec[2+1] = (unsigned long)arg1;                         \
3117       _argvec[2+2] = (unsigned long)arg2;                         \
3118       _argvec[2+3] = (unsigned long)arg3;                         \
3119       _argvec[2+4] = (unsigned long)arg4;                         \
3120       __asm__ volatile(                                           \
3121          "mr 11,%1\n\t"                                           \
3122          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3123          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3124          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3125          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3126          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3127          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3128          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3129          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3130          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3131          "mr 11,%1\n\t"                                           \
3132          "mr %0,3\n\t"                                            \
3133          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3134          VG_CONTRACT_FRAME_BY(512)                                \
3135          : /*out*/   "=r" (_res)                                  \
3136          : /*in*/    "r" (&_argvec[2])                            \
3137          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3138       );                                                          \
3139       lval = (__typeof__(lval)) _res;                             \
3140    } while (0)
3141
3142 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3143    do {                                                           \
3144       volatile OrigFn        _orig = (orig);                      \
3145       volatile unsigned long _argvec[3+5];                        \
3146       volatile unsigned long _res;                                \
3147       /* _argvec[0] holds current r2 across the call */           \
3148       _argvec[1]   = (unsigned long)_orig.r2;                     \
3149       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3150       _argvec[2+1] = (unsigned long)arg1;                         \
3151       _argvec[2+2] = (unsigned long)arg2;                         \
3152       _argvec[2+3] = (unsigned long)arg3;                         \
3153       _argvec[2+4] = (unsigned long)arg4;                         \
3154       _argvec[2+5] = (unsigned long)arg5;                         \
3155       __asm__ volatile(                                           \
3156          "mr 11,%1\n\t"                                           \
3157          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3158          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3159          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3160          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3161          "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
3162          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3163          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3164          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3165          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3166          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3167          "mr 11,%1\n\t"                                           \
3168          "mr %0,3\n\t"                                            \
3169          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3170          VG_CONTRACT_FRAME_BY(512)                                \
3171          : /*out*/   "=r" (_res)                                  \
3172          : /*in*/    "r" (&_argvec[2])                            \
3173          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3174       );                                                          \
3175       lval = (__typeof__(lval)) _res;                             \
3176    } while (0)
3177
3178 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3179    do {                                                           \
3180       volatile OrigFn        _orig = (orig);                      \
3181       volatile unsigned long _argvec[3+6];                        \
3182       volatile unsigned long _res;                                \
3183       /* _argvec[0] holds current r2 across the call */           \
3184       _argvec[1]   = (unsigned long)_orig.r2;                     \
3185       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3186       _argvec[2+1] = (unsigned long)arg1;                         \
3187       _argvec[2+2] = (unsigned long)arg2;                         \
3188       _argvec[2+3] = (unsigned long)arg3;                         \
3189       _argvec[2+4] = (unsigned long)arg4;                         \
3190       _argvec[2+5] = (unsigned long)arg5;                         \
3191       _argvec[2+6] = (unsigned long)arg6;                         \
3192       __asm__ volatile(                                           \
3193          "mr 11,%1\n\t"                                           \
3194          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3195          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3196          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3197          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3198          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3199          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3200          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3201          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3202          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3203          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3204          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3205          "mr 11,%1\n\t"                                           \
3206          "mr %0,3\n\t"                                            \
3207          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3208          VG_CONTRACT_FRAME_BY(512)                                \
3209          : /*out*/   "=r" (_res)                                  \
3210          : /*in*/    "r" (&_argvec[2])                            \
3211          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3212       );                                                          \
3213       lval = (__typeof__(lval)) _res;                             \
3214    } while (0)
3215
3216 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3217                                  arg7)                            \
3218    do {                                                           \
3219       volatile OrigFn        _orig = (orig);                      \
3220       volatile unsigned long _argvec[3+7];                        \
3221       volatile unsigned long _res;                                \
3222       /* _argvec[0] holds current r2 across the call */           \
3223       _argvec[1]   = (unsigned long)_orig.r2;                     \
3224       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3225       _argvec[2+1] = (unsigned long)arg1;                         \
3226       _argvec[2+2] = (unsigned long)arg2;                         \
3227       _argvec[2+3] = (unsigned long)arg3;                         \
3228       _argvec[2+4] = (unsigned long)arg4;                         \
3229       _argvec[2+5] = (unsigned long)arg5;                         \
3230       _argvec[2+6] = (unsigned long)arg6;                         \
3231       _argvec[2+7] = (unsigned long)arg7;                         \
3232       __asm__ volatile(                                           \
3233          "mr 11,%1\n\t"                                           \
3234          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3235          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3236          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3237          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3238          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3239          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3240          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3241          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3242          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3243          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3244          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3245          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3246          "mr 11,%1\n\t"                                           \
3247          "mr %0,3\n\t"                                            \
3248          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3249          VG_CONTRACT_FRAME_BY(512)                                \
3250          : /*out*/   "=r" (_res)                                  \
3251          : /*in*/    "r" (&_argvec[2])                            \
3252          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3253       );                                                          \
3254       lval = (__typeof__(lval)) _res;                             \
3255    } while (0)
3256
3257 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3258                                  arg7,arg8)                       \
3259    do {                                                           \
3260       volatile OrigFn        _orig = (orig);                      \
3261       volatile unsigned long _argvec[3+8];                        \
3262       volatile unsigned long _res;                                \
3263       /* _argvec[0] holds current r2 across the call */           \
3264       _argvec[1]   = (unsigned long)_orig.r2;                     \
3265       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3266       _argvec[2+1] = (unsigned long)arg1;                         \
3267       _argvec[2+2] = (unsigned long)arg2;                         \
3268       _argvec[2+3] = (unsigned long)arg3;                         \
3269       _argvec[2+4] = (unsigned long)arg4;                         \
3270       _argvec[2+5] = (unsigned long)arg5;                         \
3271       _argvec[2+6] = (unsigned long)arg6;                         \
3272       _argvec[2+7] = (unsigned long)arg7;                         \
3273       _argvec[2+8] = (unsigned long)arg8;                         \
3274       __asm__ volatile(                                           \
3275          "mr 11,%1\n\t"                                           \
3276          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3277          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3278          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3279          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3280          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3281          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3282          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3283          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3284          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3285          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3286          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3287          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3288          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3289          "mr 11,%1\n\t"                                           \
3290          "mr %0,3\n\t"                                            \
3291          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3292          VG_CONTRACT_FRAME_BY(512)                                \
3293          : /*out*/   "=r" (_res)                                  \
3294          : /*in*/    "r" (&_argvec[2])                            \
3295          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3296       );                                                          \
3297       lval = (__typeof__(lval)) _res;                             \
3298    } while (0)
3299
3300 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3301                                  arg7,arg8,arg9)                  \
3302    do {                                                           \
3303       volatile OrigFn        _orig = (orig);                      \
3304       volatile unsigned long _argvec[3+9];                        \
3305       volatile unsigned long _res;                                \
3306       /* _argvec[0] holds current r2 across the call */           \
3307       _argvec[1]   = (unsigned long)_orig.r2;                     \
3308       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3309       _argvec[2+1] = (unsigned long)arg1;                         \
3310       _argvec[2+2] = (unsigned long)arg2;                         \
3311       _argvec[2+3] = (unsigned long)arg3;                         \
3312       _argvec[2+4] = (unsigned long)arg4;                         \
3313       _argvec[2+5] = (unsigned long)arg5;                         \
3314       _argvec[2+6] = (unsigned long)arg6;                         \
3315       _argvec[2+7] = (unsigned long)arg7;                         \
3316       _argvec[2+8] = (unsigned long)arg8;                         \
3317       _argvec[2+9] = (unsigned long)arg9;                         \
3318       __asm__ volatile(                                           \
3319          "mr 11,%1\n\t"                                           \
3320          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3321          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3322          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3323          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3324          /* arg9 */                                               \
3325          "lwz 3,36(11)\n\t"                                       \
3326          "stw 3,56(1)\n\t"                                        \
3327          /* args1-8 */                                            \
3328          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3329          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3330          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3331          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3332          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3333          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3334          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3335          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3336          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3337          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3338          "mr 11,%1\n\t"                                           \
3339          "mr %0,3\n\t"                                            \
3340          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3341          VG_CONTRACT_FRAME_BY(64)                                 \
3342          VG_CONTRACT_FRAME_BY(512)                                \
3343          : /*out*/   "=r" (_res)                                  \
3344          : /*in*/    "r" (&_argvec[2])                            \
3345          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3346       );                                                          \
3347       lval = (__typeof__(lval)) _res;                             \
3348    } while (0)
3349
3350 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3351                                   arg7,arg8,arg9,arg10)           \
3352    do {                                                           \
3353       volatile OrigFn        _orig = (orig);                      \
3354       volatile unsigned long _argvec[3+10];                       \
3355       volatile unsigned long _res;                                \
3356       /* _argvec[0] holds current r2 across the call */           \
3357       _argvec[1]   = (unsigned long)_orig.r2;                     \
3358       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3359       _argvec[2+1] = (unsigned long)arg1;                         \
3360       _argvec[2+2] = (unsigned long)arg2;                         \
3361       _argvec[2+3] = (unsigned long)arg3;                         \
3362       _argvec[2+4] = (unsigned long)arg4;                         \
3363       _argvec[2+5] = (unsigned long)arg5;                         \
3364       _argvec[2+6] = (unsigned long)arg6;                         \
3365       _argvec[2+7] = (unsigned long)arg7;                         \
3366       _argvec[2+8] = (unsigned long)arg8;                         \
3367       _argvec[2+9] = (unsigned long)arg9;                         \
3368       _argvec[2+10] = (unsigned long)arg10;                       \
3369       __asm__ volatile(                                           \
3370          "mr 11,%1\n\t"                                           \
3371          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3372          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3373          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3374          VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
3375          /* arg10 */                                              \
3376          "lwz 3,40(11)\n\t"                                       \
3377          "stw 3,60(1)\n\t"                                        \
3378          /* arg9 */                                               \
3379          "lwz 3,36(11)\n\t"                                       \
3380          "stw 3,56(1)\n\t"                                        \
3381          /* args1-8 */                                            \
3382          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3383          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3384          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3385          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3386          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3387          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3388          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3389          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3390          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3391          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3392          "mr 11,%1\n\t"                                           \
3393          "mr %0,3\n\t"                                            \
3394          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3395          VG_CONTRACT_FRAME_BY(64)                                 \
3396          VG_CONTRACT_FRAME_BY(512)                                \
3397          : /*out*/   "=r" (_res)                                  \
3398          : /*in*/    "r" (&_argvec[2])                            \
3399          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3400       );                                                          \
3401       lval = (__typeof__(lval)) _res;                             \
3402    } while (0)
3403
3404 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3405                                   arg7,arg8,arg9,arg10,arg11)     \
3406    do {                                                           \
3407       volatile OrigFn        _orig = (orig);                      \
3408       volatile unsigned long _argvec[3+11];                       \
3409       volatile unsigned long _res;                                \
3410       /* _argvec[0] holds current r2 across the call */           \
3411       _argvec[1]   = (unsigned long)_orig.r2;                     \
3412       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3413       _argvec[2+1] = (unsigned long)arg1;                         \
3414       _argvec[2+2] = (unsigned long)arg2;                         \
3415       _argvec[2+3] = (unsigned long)arg3;                         \
3416       _argvec[2+4] = (unsigned long)arg4;                         \
3417       _argvec[2+5] = (unsigned long)arg5;                         \
3418       _argvec[2+6] = (unsigned long)arg6;                         \
3419       _argvec[2+7] = (unsigned long)arg7;                         \
3420       _argvec[2+8] = (unsigned long)arg8;                         \
3421       _argvec[2+9] = (unsigned long)arg9;                         \
3422       _argvec[2+10] = (unsigned long)arg10;                       \
3423       _argvec[2+11] = (unsigned long)arg11;                       \
3424       __asm__ volatile(                                           \
3425          "mr 11,%1\n\t"                                           \
3426          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3427          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3428          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3429          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3430          /* arg11 */                                              \
3431          "lwz 3,44(11)\n\t"                                       \
3432          "stw 3,64(1)\n\t"                                        \
3433          /* arg10 */                                              \
3434          "lwz 3,40(11)\n\t"                                       \
3435          "stw 3,60(1)\n\t"                                        \
3436          /* arg9 */                                               \
3437          "lwz 3,36(11)\n\t"                                       \
3438          "stw 3,56(1)\n\t"                                        \
3439          /* args1-8 */                                            \
3440          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3441          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3442          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3443          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3444          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3445          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3446          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3447          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3448          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3449          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3450          "mr 11,%1\n\t"                                           \
3451          "mr %0,3\n\t"                                            \
3452          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3453          VG_CONTRACT_FRAME_BY(72)                                 \
3454          VG_CONTRACT_FRAME_BY(512)                                \
3455          : /*out*/   "=r" (_res)                                  \
3456          : /*in*/    "r" (&_argvec[2])                            \
3457          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3458       );                                                          \
3459       lval = (__typeof__(lval)) _res;                             \
3460    } while (0)
3461
3462 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3463                                 arg7,arg8,arg9,arg10,arg11,arg12) \
3464    do {                                                           \
3465       volatile OrigFn        _orig = (orig);                      \
3466       volatile unsigned long _argvec[3+12];                       \
3467       volatile unsigned long _res;                                \
3468       /* _argvec[0] holds current r2 across the call */           \
3469       _argvec[1]   = (unsigned long)_orig.r2;                     \
3470       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3471       _argvec[2+1] = (unsigned long)arg1;                         \
3472       _argvec[2+2] = (unsigned long)arg2;                         \
3473       _argvec[2+3] = (unsigned long)arg3;                         \
3474       _argvec[2+4] = (unsigned long)arg4;                         \
3475       _argvec[2+5] = (unsigned long)arg5;                         \
3476       _argvec[2+6] = (unsigned long)arg6;                         \
3477       _argvec[2+7] = (unsigned long)arg7;                         \
3478       _argvec[2+8] = (unsigned long)arg8;                         \
3479       _argvec[2+9] = (unsigned long)arg9;                         \
3480       _argvec[2+10] = (unsigned long)arg10;                       \
3481       _argvec[2+11] = (unsigned long)arg11;                       \
3482       _argvec[2+12] = (unsigned long)arg12;                       \
3483       __asm__ volatile(                                           \
3484          "mr 11,%1\n\t"                                           \
3485          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3486          "stw  2,-8(11)\n\t"  /* save tocptr */                   \
3487          "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
3488          VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
3489          /* arg12 */                                              \
3490          "lwz 3,48(11)\n\t"                                       \
3491          "stw 3,68(1)\n\t"                                        \
3492          /* arg11 */                                              \
3493          "lwz 3,44(11)\n\t"                                       \
3494          "stw 3,64(1)\n\t"                                        \
3495          /* arg10 */                                              \
3496          "lwz 3,40(11)\n\t"                                       \
3497          "stw 3,60(1)\n\t"                                        \
3498          /* arg9 */                                               \
3499          "lwz 3,36(11)\n\t"                                       \
3500          "stw 3,56(1)\n\t"                                        \
3501          /* args1-8 */                                            \
3502          "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
3503          "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
3504          "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
3505          "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
3506          "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
3507          "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
3508          "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
3509          "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
3510          "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
3511          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3512          "mr 11,%1\n\t"                                           \
3513          "mr %0,3\n\t"                                            \
3514          "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
3515          VG_CONTRACT_FRAME_BY(72)                                 \
3516          VG_CONTRACT_FRAME_BY(512)                                \
3517          : /*out*/   "=r" (_res)                                  \
3518          : /*in*/    "r" (&_argvec[2])                            \
3519          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3520       );                                                          \
3521       lval = (__typeof__(lval)) _res;                             \
3522    } while (0)
3523
3524 #endif /* PLAT_ppc32_aix5 */
3525
3526 /* ------------------------ ppc64-aix5 ------------------------- */
3527
3528 #if defined(PLAT_ppc64_aix5)
3529
3530 /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3531
3532 /* These regs are trashed by the hidden call. */
3533 #define __CALLER_SAVED_REGS                                       \
3534    "lr", "ctr", "xer",                                            \
3535    "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3536    "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
3537    "r11", "r12", "r13"
3538
3539 /* Expand the stack frame, copying enough info that unwinding
3540    still works.  Trashes r3. */
3541
3542 #define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
3543          "addi 1,1,-" #_n_fr "\n\t"                               \
3544          "ld   3," #_n_fr "(1)\n\t"                               \
3545          "std  3,0(1)\n\t"
3546
3547 #define VG_CONTRACT_FRAME_BY(_n_fr)                               \
3548          "addi 1,1," #_n_fr "\n\t"
3549
3550 /* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
3551    long) == 8. */
3552
3553 #define CALL_FN_W_v(lval, orig)                                   \
3554    do {                                                           \
3555       volatile OrigFn        _orig = (orig);                      \
3556       volatile unsigned long _argvec[3+0];                        \
3557       volatile unsigned long _res;                                \
3558       /* _argvec[0] holds current r2 across the call */           \
3559       _argvec[1] = (unsigned long)_orig.r2;                       \
3560       _argvec[2] = (unsigned long)_orig.nraddr;                   \
3561       __asm__ volatile(                                           \
3562          "mr 11,%1\n\t"                                           \
3563          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3564          "std  2,-16(11)\n\t" /* save tocptr */                   \
3565          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3566          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3567          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3568          "mr 11,%1\n\t"                                           \
3569          "mr %0,3\n\t"                                            \
3570          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3571          VG_CONTRACT_FRAME_BY(512)                                \
3572          : /*out*/   "=r" (_res)                                  \
3573          : /*in*/    "r" (&_argvec[2])                            \
3574          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3575       );                                                          \
3576       lval = (__typeof__(lval)) _res;                             \
3577    } while (0)
3578
3579 #define CALL_FN_W_W(lval, orig, arg1)                             \
3580    do {                                                           \
3581       volatile OrigFn        _orig = (orig);                      \
3582       volatile unsigned long _argvec[3+1];                        \
3583       volatile unsigned long _res;                                \
3584       /* _argvec[0] holds current r2 across the call */           \
3585       _argvec[1]   = (unsigned long)_orig.r2;                     \
3586       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3587       _argvec[2+1] = (unsigned long)arg1;                         \
3588       __asm__ volatile(                                           \
3589          "mr 11,%1\n\t"                                           \
3590          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3591          "std  2,-16(11)\n\t" /* save tocptr */                   \
3592          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3593          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3594          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3595          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3596          "mr 11,%1\n\t"                                           \
3597          "mr %0,3\n\t"                                            \
3598          "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3599          VG_CONTRACT_FRAME_BY(512)                                \
3600          : /*out*/   "=r" (_res)                                  \
3601          : /*in*/    "r" (&_argvec[2])                            \
3602          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3603       );                                                          \
3604       lval = (__typeof__(lval)) _res;                             \
3605    } while (0)
3606
3607 #define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3608    do {                                                           \
3609       volatile OrigFn        _orig = (orig);                      \
3610       volatile unsigned long _argvec[3+2];                        \
3611       volatile unsigned long _res;                                \
3612       /* _argvec[0] holds current r2 across the call */           \
3613       _argvec[1]   = (unsigned long)_orig.r2;                     \
3614       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3615       _argvec[2+1] = (unsigned long)arg1;                         \
3616       _argvec[2+2] = (unsigned long)arg2;                         \
3617       __asm__ volatile(                                           \
3618          "mr 11,%1\n\t"                                           \
3619          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3620          "std  2,-16(11)\n\t" /* save tocptr */                   \
3621          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3622          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3623          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3624          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3625          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3626          "mr 11,%1\n\t"                                           \
3627          "mr %0,3\n\t"                                            \
3628          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3629          VG_CONTRACT_FRAME_BY(512)                                \
3630          : /*out*/   "=r" (_res)                                  \
3631          : /*in*/    "r" (&_argvec[2])                            \
3632          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3633       );                                                          \
3634       lval = (__typeof__(lval)) _res;                             \
3635    } while (0)
3636
3637 #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3638    do {                                                           \
3639       volatile OrigFn        _orig = (orig);                      \
3640       volatile unsigned long _argvec[3+3];                        \
3641       volatile unsigned long _res;                                \
3642       /* _argvec[0] holds current r2 across the call */           \
3643       _argvec[1]   = (unsigned long)_orig.r2;                     \
3644       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3645       _argvec[2+1] = (unsigned long)arg1;                         \
3646       _argvec[2+2] = (unsigned long)arg2;                         \
3647       _argvec[2+3] = (unsigned long)arg3;                         \
3648       __asm__ volatile(                                           \
3649          "mr 11,%1\n\t"                                           \
3650          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3651          "std  2,-16(11)\n\t" /* save tocptr */                   \
3652          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3653          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3654          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3655          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3656          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3657          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3658          "mr 11,%1\n\t"                                           \
3659          "mr %0,3\n\t"                                            \
3660          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3661          VG_CONTRACT_FRAME_BY(512)                                \
3662          : /*out*/   "=r" (_res)                                  \
3663          : /*in*/    "r" (&_argvec[2])                            \
3664          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3665       );                                                          \
3666       lval = (__typeof__(lval)) _res;                             \
3667    } while (0)
3668
3669 #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3670    do {                                                           \
3671       volatile OrigFn        _orig = (orig);                      \
3672       volatile unsigned long _argvec[3+4];                        \
3673       volatile unsigned long _res;                                \
3674       /* _argvec[0] holds current r2 across the call */           \
3675       _argvec[1]   = (unsigned long)_orig.r2;                     \
3676       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3677       _argvec[2+1] = (unsigned long)arg1;                         \
3678       _argvec[2+2] = (unsigned long)arg2;                         \
3679       _argvec[2+3] = (unsigned long)arg3;                         \
3680       _argvec[2+4] = (unsigned long)arg4;                         \
3681       __asm__ volatile(                                           \
3682          "mr 11,%1\n\t"                                           \
3683          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3684          "std  2,-16(11)\n\t" /* save tocptr */                   \
3685          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3686          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3687          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3688          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3689          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3690          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3691          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3692          "mr 11,%1\n\t"                                           \
3693          "mr %0,3\n\t"                                            \
3694          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3695          VG_CONTRACT_FRAME_BY(512)                                \
3696          : /*out*/   "=r" (_res)                                  \
3697          : /*in*/    "r" (&_argvec[2])                            \
3698          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3699       );                                                          \
3700       lval = (__typeof__(lval)) _res;                             \
3701    } while (0)
3702
3703 #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3704    do {                                                           \
3705       volatile OrigFn        _orig = (orig);                      \
3706       volatile unsigned long _argvec[3+5];                        \
3707       volatile unsigned long _res;                                \
3708       /* _argvec[0] holds current r2 across the call */           \
3709       _argvec[1]   = (unsigned long)_orig.r2;                     \
3710       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3711       _argvec[2+1] = (unsigned long)arg1;                         \
3712       _argvec[2+2] = (unsigned long)arg2;                         \
3713       _argvec[2+3] = (unsigned long)arg3;                         \
3714       _argvec[2+4] = (unsigned long)arg4;                         \
3715       _argvec[2+5] = (unsigned long)arg5;                         \
3716       __asm__ volatile(                                           \
3717          "mr 11,%1\n\t"                                           \
3718          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3719          "std  2,-16(11)\n\t" /* save tocptr */                   \
3720          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3721          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3722          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3723          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3724          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3725          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3726          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3727          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3728          "mr 11,%1\n\t"                                           \
3729          "mr %0,3\n\t"                                            \
3730          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3731          VG_CONTRACT_FRAME_BY(512)                                \
3732          : /*out*/   "=r" (_res)                                  \
3733          : /*in*/    "r" (&_argvec[2])                            \
3734          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3735       );                                                          \
3736       lval = (__typeof__(lval)) _res;                             \
3737    } while (0)
3738
3739 #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3740    do {                                                           \
3741       volatile OrigFn        _orig = (orig);                      \
3742       volatile unsigned long _argvec[3+6];                        \
3743       volatile unsigned long _res;                                \
3744       /* _argvec[0] holds current r2 across the call */           \
3745       _argvec[1]   = (unsigned long)_orig.r2;                     \
3746       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3747       _argvec[2+1] = (unsigned long)arg1;                         \
3748       _argvec[2+2] = (unsigned long)arg2;                         \
3749       _argvec[2+3] = (unsigned long)arg3;                         \
3750       _argvec[2+4] = (unsigned long)arg4;                         \
3751       _argvec[2+5] = (unsigned long)arg5;                         \
3752       _argvec[2+6] = (unsigned long)arg6;                         \
3753       __asm__ volatile(                                           \
3754          "mr 11,%1\n\t"                                           \
3755          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3756          "std  2,-16(11)\n\t" /* save tocptr */                   \
3757          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3758          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3759          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3760          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3761          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3762          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3763          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3764          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3765          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3766          "mr 11,%1\n\t"                                           \
3767          "mr %0,3\n\t"                                            \
3768          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3769          VG_CONTRACT_FRAME_BY(512)                                \
3770          : /*out*/   "=r" (_res)                                  \
3771          : /*in*/    "r" (&_argvec[2])                            \
3772          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3773       );                                                          \
3774       lval = (__typeof__(lval)) _res;                             \
3775    } while (0)
3776
3777 #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3778                                  arg7)                            \
3779    do {                                                           \
3780       volatile OrigFn        _orig = (orig);                      \
3781       volatile unsigned long _argvec[3+7];                        \
3782       volatile unsigned long _res;                                \
3783       /* _argvec[0] holds current r2 across the call */           \
3784       _argvec[1]   = (unsigned long)_orig.r2;                     \
3785       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3786       _argvec[2+1] = (unsigned long)arg1;                         \
3787       _argvec[2+2] = (unsigned long)arg2;                         \
3788       _argvec[2+3] = (unsigned long)arg3;                         \
3789       _argvec[2+4] = (unsigned long)arg4;                         \
3790       _argvec[2+5] = (unsigned long)arg5;                         \
3791       _argvec[2+6] = (unsigned long)arg6;                         \
3792       _argvec[2+7] = (unsigned long)arg7;                         \
3793       __asm__ volatile(                                           \
3794          "mr 11,%1\n\t"                                           \
3795          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3796          "std  2,-16(11)\n\t" /* save tocptr */                   \
3797          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3798          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3799          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3800          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3801          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3802          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3803          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3804          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3805          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3806          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3807          "mr 11,%1\n\t"                                           \
3808          "mr %0,3\n\t"                                            \
3809          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3810          VG_CONTRACT_FRAME_BY(512)                                \
3811          : /*out*/   "=r" (_res)                                  \
3812          : /*in*/    "r" (&_argvec[2])                            \
3813          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3814       );                                                          \
3815       lval = (__typeof__(lval)) _res;                             \
3816    } while (0)
3817
3818 #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3819                                  arg7,arg8)                       \
3820    do {                                                           \
3821       volatile OrigFn        _orig = (orig);                      \
3822       volatile unsigned long _argvec[3+8];                        \
3823       volatile unsigned long _res;                                \
3824       /* _argvec[0] holds current r2 across the call */           \
3825       _argvec[1]   = (unsigned long)_orig.r2;                     \
3826       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3827       _argvec[2+1] = (unsigned long)arg1;                         \
3828       _argvec[2+2] = (unsigned long)arg2;                         \
3829       _argvec[2+3] = (unsigned long)arg3;                         \
3830       _argvec[2+4] = (unsigned long)arg4;                         \
3831       _argvec[2+5] = (unsigned long)arg5;                         \
3832       _argvec[2+6] = (unsigned long)arg6;                         \
3833       _argvec[2+7] = (unsigned long)arg7;                         \
3834       _argvec[2+8] = (unsigned long)arg8;                         \
3835       __asm__ volatile(                                           \
3836          "mr 11,%1\n\t"                                           \
3837          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3838          "std  2,-16(11)\n\t" /* save tocptr */                   \
3839          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3840          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3841          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3842          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3843          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3844          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3845          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3846          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3847          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3848          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3849          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3850          "mr 11,%1\n\t"                                           \
3851          "mr %0,3\n\t"                                            \
3852          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3853          VG_CONTRACT_FRAME_BY(512)                                \
3854          : /*out*/   "=r" (_res)                                  \
3855          : /*in*/    "r" (&_argvec[2])                            \
3856          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3857       );                                                          \
3858       lval = (__typeof__(lval)) _res;                             \
3859    } while (0)
3860
3861 #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3862                                  arg7,arg8,arg9)                  \
3863    do {                                                           \
3864       volatile OrigFn        _orig = (orig);                      \
3865       volatile unsigned long _argvec[3+9];                        \
3866       volatile unsigned long _res;                                \
3867       /* _argvec[0] holds current r2 across the call */           \
3868       _argvec[1]   = (unsigned long)_orig.r2;                     \
3869       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3870       _argvec[2+1] = (unsigned long)arg1;                         \
3871       _argvec[2+2] = (unsigned long)arg2;                         \
3872       _argvec[2+3] = (unsigned long)arg3;                         \
3873       _argvec[2+4] = (unsigned long)arg4;                         \
3874       _argvec[2+5] = (unsigned long)arg5;                         \
3875       _argvec[2+6] = (unsigned long)arg6;                         \
3876       _argvec[2+7] = (unsigned long)arg7;                         \
3877       _argvec[2+8] = (unsigned long)arg8;                         \
3878       _argvec[2+9] = (unsigned long)arg9;                         \
3879       __asm__ volatile(                                           \
3880          "mr 11,%1\n\t"                                           \
3881          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3882          "std  2,-16(11)\n\t" /* save tocptr */                   \
3883          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3884          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3885          /* arg9 */                                               \
3886          "ld  3,72(11)\n\t"                                       \
3887          "std 3,112(1)\n\t"                                       \
3888          /* args1-8 */                                            \
3889          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3890          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3891          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3892          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3893          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3894          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3895          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3896          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3897          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3898          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3899          "mr 11,%1\n\t"                                           \
3900          "mr %0,3\n\t"                                            \
3901          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3902          VG_CONTRACT_FRAME_BY(128)                                \
3903          VG_CONTRACT_FRAME_BY(512)                                \
3904          : /*out*/   "=r" (_res)                                  \
3905          : /*in*/    "r" (&_argvec[2])                            \
3906          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3907       );                                                          \
3908       lval = (__typeof__(lval)) _res;                             \
3909    } while (0)
3910
3911 #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3912                                   arg7,arg8,arg9,arg10)           \
3913    do {                                                           \
3914       volatile OrigFn        _orig = (orig);                      \
3915       volatile unsigned long _argvec[3+10];                       \
3916       volatile unsigned long _res;                                \
3917       /* _argvec[0] holds current r2 across the call */           \
3918       _argvec[1]   = (unsigned long)_orig.r2;                     \
3919       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3920       _argvec[2+1] = (unsigned long)arg1;                         \
3921       _argvec[2+2] = (unsigned long)arg2;                         \
3922       _argvec[2+3] = (unsigned long)arg3;                         \
3923       _argvec[2+4] = (unsigned long)arg4;                         \
3924       _argvec[2+5] = (unsigned long)arg5;                         \
3925       _argvec[2+6] = (unsigned long)arg6;                         \
3926       _argvec[2+7] = (unsigned long)arg7;                         \
3927       _argvec[2+8] = (unsigned long)arg8;                         \
3928       _argvec[2+9] = (unsigned long)arg9;                         \
3929       _argvec[2+10] = (unsigned long)arg10;                       \
3930       __asm__ volatile(                                           \
3931          "mr 11,%1\n\t"                                           \
3932          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3933          "std  2,-16(11)\n\t" /* save tocptr */                   \
3934          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3935          VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
3936          /* arg10 */                                              \
3937          "ld  3,80(11)\n\t"                                       \
3938          "std 3,120(1)\n\t"                                       \
3939          /* arg9 */                                               \
3940          "ld  3,72(11)\n\t"                                       \
3941          "std 3,112(1)\n\t"                                       \
3942          /* args1-8 */                                            \
3943          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3944          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3945          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3946          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3947          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3948          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3949          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3950          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3951          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3952          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3953          "mr 11,%1\n\t"                                           \
3954          "mr %0,3\n\t"                                            \
3955          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
3956          VG_CONTRACT_FRAME_BY(128)                                \
3957          VG_CONTRACT_FRAME_BY(512)                                \
3958          : /*out*/   "=r" (_res)                                  \
3959          : /*in*/    "r" (&_argvec[2])                            \
3960          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
3961       );                                                          \
3962       lval = (__typeof__(lval)) _res;                             \
3963    } while (0)
3964
3965 #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3966                                   arg7,arg8,arg9,arg10,arg11)     \
3967    do {                                                           \
3968       volatile OrigFn        _orig = (orig);                      \
3969       volatile unsigned long _argvec[3+11];                       \
3970       volatile unsigned long _res;                                \
3971       /* _argvec[0] holds current r2 across the call */           \
3972       _argvec[1]   = (unsigned long)_orig.r2;                     \
3973       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3974       _argvec[2+1] = (unsigned long)arg1;                         \
3975       _argvec[2+2] = (unsigned long)arg2;                         \
3976       _argvec[2+3] = (unsigned long)arg3;                         \
3977       _argvec[2+4] = (unsigned long)arg4;                         \
3978       _argvec[2+5] = (unsigned long)arg5;                         \
3979       _argvec[2+6] = (unsigned long)arg6;                         \
3980       _argvec[2+7] = (unsigned long)arg7;                         \
3981       _argvec[2+8] = (unsigned long)arg8;                         \
3982       _argvec[2+9] = (unsigned long)arg9;                         \
3983       _argvec[2+10] = (unsigned long)arg10;                       \
3984       _argvec[2+11] = (unsigned long)arg11;                       \
3985       __asm__ volatile(                                           \
3986          "mr 11,%1\n\t"                                           \
3987          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
3988          "std  2,-16(11)\n\t" /* save tocptr */                   \
3989          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3990          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
3991          /* arg11 */                                              \
3992          "ld  3,88(11)\n\t"                                       \
3993          "std 3,128(1)\n\t"                                       \
3994          /* arg10 */                                              \
3995          "ld  3,80(11)\n\t"                                       \
3996          "std 3,120(1)\n\t"                                       \
3997          /* arg9 */                                               \
3998          "ld  3,72(11)\n\t"                                       \
3999          "std 3,112(1)\n\t"                                       \
4000          /* args1-8 */                                            \
4001          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4002          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4003          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4004          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4005          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4006          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4007          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4008          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4009          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4010          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4011          "mr 11,%1\n\t"                                           \
4012          "mr %0,3\n\t"                                            \
4013          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4014          VG_CONTRACT_FRAME_BY(144)                                \
4015          VG_CONTRACT_FRAME_BY(512)                                \
4016          : /*out*/   "=r" (_res)                                  \
4017          : /*in*/    "r" (&_argvec[2])                            \
4018          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4019       );                                                          \
4020       lval = (__typeof__(lval)) _res;                             \
4021    } while (0)
4022
4023 #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4024                                 arg7,arg8,arg9,arg10,arg11,arg12) \
4025    do {                                                           \
4026       volatile OrigFn        _orig = (orig);                      \
4027       volatile unsigned long _argvec[3+12];                       \
4028       volatile unsigned long _res;                                \
4029       /* _argvec[0] holds current r2 across the call */           \
4030       _argvec[1]   = (unsigned long)_orig.r2;                     \
4031       _argvec[2]   = (unsigned long)_orig.nraddr;                 \
4032       _argvec[2+1] = (unsigned long)arg1;                         \
4033       _argvec[2+2] = (unsigned long)arg2;                         \
4034       _argvec[2+3] = (unsigned long)arg3;                         \
4035       _argvec[2+4] = (unsigned long)arg4;                         \
4036       _argvec[2+5] = (unsigned long)arg5;                         \
4037       _argvec[2+6] = (unsigned long)arg6;                         \
4038       _argvec[2+7] = (unsigned long)arg7;                         \
4039       _argvec[2+8] = (unsigned long)arg8;                         \
4040       _argvec[2+9] = (unsigned long)arg9;                         \
4041       _argvec[2+10] = (unsigned long)arg10;                       \
4042       _argvec[2+11] = (unsigned long)arg11;                       \
4043       _argvec[2+12] = (unsigned long)arg12;                       \
4044       __asm__ volatile(                                           \
4045          "mr 11,%1\n\t"                                           \
4046          VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
4047          "std  2,-16(11)\n\t" /* save tocptr */                   \
4048          "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
4049          VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
4050          /* arg12 */                                              \
4051          "ld  3,96(11)\n\t"                                       \
4052          "std 3,136(1)\n\t"                                       \
4053          /* arg11 */                                              \
4054          "ld  3,88(11)\n\t"                                       \
4055          "std 3,128(1)\n\t"                                       \
4056          /* arg10 */                                              \
4057          "ld  3,80(11)\n\t"                                       \
4058          "std 3,120(1)\n\t"                                       \
4059          /* arg9 */                                               \
4060          "ld  3,72(11)\n\t"                                       \
4061          "std 3,112(1)\n\t"                                       \
4062          /* args1-8 */                                            \
4063          "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
4064          "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
4065          "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
4066          "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
4067          "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
4068          "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
4069          "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
4070          "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
4071          "ld  11, 0(11)\n\t"  /* target->r11 */                   \
4072          VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
4073          "mr 11,%1\n\t"                                           \
4074          "mr %0,3\n\t"                                            \
4075          "ld  2,-16(11)\n\t" /* restore tocptr */                 \
4076          VG_CONTRACT_FRAME_BY(144)                                \
4077          VG_CONTRACT_FRAME_BY(512)                                \
4078          : /*out*/   "=r" (_res)                                  \
4079          : /*in*/    "r" (&_argvec[2])                            \
4080          : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
4081       );                                                          \
4082       lval = (__typeof__(lval)) _res;                             \
4083    } while (0)
4084
4085 #endif /* PLAT_ppc64_aix5 */
4086
4087
4088 /* ------------------------------------------------------------------ */
4089 /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
4090 /*                                                                    */
4091 /* ------------------------------------------------------------------ */
4092
4093 /* Some request codes.  There are many more of these, but most are not
4094    exposed to end-user view.  These are the public ones, all of the
4095    form 0x1000 + small_number.
4096
4097    Core ones are in the range 0x00000000--0x0000ffff.  The non-public
4098    ones start at 0x2000.
4099 */
4100
4101 /* These macros are used by tools -- they must be public, but don't
4102    embed them into other programs. */
4103 #define VG_USERREQ_TOOL_BASE(a,b) \
4104    ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
4105 #define VG_IS_TOOL_USERREQ(a, b, v) \
4106    (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
4107
4108 /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !!
4109    This enum comprises an ABI exported by Valgrind to programs
4110    which use client requests.  DO NOT CHANGE THE ORDER OF THESE
4111    ENTRIES, NOR DELETE ANY -- add new ones at the end. */
4112 typedef
4113    enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
4114           VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
4115
4116           /* These allow any function to be called from the simulated
4117              CPU but run on the real CPU.  Nb: the first arg passed to
4118              the function is always the ThreadId of the running
4119              thread!  So CLIENT_CALL0 actually requires a 1 arg
4120              function, etc. */
4121           VG_USERREQ__CLIENT_CALL0 = 0x1101,
4122           VG_USERREQ__CLIENT_CALL1 = 0x1102,
4123           VG_USERREQ__CLIENT_CALL2 = 0x1103,
4124           VG_USERREQ__CLIENT_CALL3 = 0x1104,
4125
4126           /* Can be useful in regression testing suites -- eg. can
4127              send Valgrind's output to /dev/null and still count
4128              errors. */
4129           VG_USERREQ__COUNT_ERRORS = 0x1201,
4130
4131           /* These are useful and can be interpreted by any tool that
4132              tracks malloc() et al, by using vg_replace_malloc.c. */
4133           VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
4134           VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
4135           /* Memory pool support. */
4136           VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
4137           VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
4138           VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
4139           VG_USERREQ__MEMPOOL_FREE     = 0x1306,
4140           VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
4141           VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
4142           VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
4143           VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
4144
4145           /* Allow printfs to valgrind log. */
4146           /* The first two pass the va_list argument by value, which
4147              assumes it is the same size as or smaller than a UWord,
4148              which generally isn't the case.  Hence are deprecated.
4149              The second two pass the vargs by reference and so are
4150              immune to this problem. */
4151           /* both :: char* fmt, va_list vargs (DEPRECATED) */
4152           VG_USERREQ__PRINTF           = 0x1401,
4153           VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
4154           /* both :: char* fmt, va_list* vargs */
4155           VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
4156           VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
4157
4158           /* Stack support. */
4159           VG_USERREQ__STACK_REGISTER   = 0x1501,
4160           VG_USERREQ__STACK_DEREGISTER = 0x1502,
4161           VG_USERREQ__STACK_CHANGE     = 0x1503,
4162
4163           /* Wine support */
4164           VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601
4165    } Vg_ClientRequest;
4166
4167 #if !defined(__GNUC__)
4168 #  define __extension__ /* */
4169 #endif
4170
4171 /* Returns the number of Valgrinds this code is running under.  That
4172    is, 0 if running natively, 1 if running under Valgrind, 2 if
4173    running under Valgrind which is running under another Valgrind,
4174    etc. */
4175 #define RUNNING_ON_VALGRIND  __extension__                        \
4176    ({unsigned int _qzz_res;                                       \
4177     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */,          \
4178                                VG_USERREQ__RUNNING_ON_VALGRIND,   \
4179                                0, 0, 0, 0, 0);                    \
4180     _qzz_res;                                                     \
4181    })
4182
4183
4184 /* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
4185    _qzz_len - 1].  Useful if you are debugging a JITter or some such,
4186    since it provides a way to make sure valgrind will retranslate the
4187    invalidated area.  Returns no value. */
4188 #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
4189    {unsigned int _qzz_res;                                        \
4190     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4191                                VG_USERREQ__DISCARD_TRANSLATIONS,  \
4192                                _qzz_addr, _qzz_len, 0, 0, 0);     \
4193    }
4194
4195
4196 /* These requests are for getting Valgrind itself to print something.
4197    Possibly with a backtrace.  This is a really ugly hack.  The return value
4198    is the number of characters printed, excluding the "**<pid>** " part at the
4199    start and the backtrace (if present). */
4200
4201 #if defined(NVALGRIND)
4202
4203 #  define VALGRIND_PRINTF(...)
4204 #  define VALGRIND_PRINTF_BACKTRACE(...)
4205
4206 #else /* NVALGRIND */
4207
4208 /* Modern GCC will optimize the static routine out if unused,
4209    and unused attribute will shut down warnings about it.  */
4210 static int VALGRIND_PRINTF(const char *format, ...)
4211    __attribute__((format(__printf__, 1, 2), __unused__));
4212 static int
4213 VALGRIND_PRINTF(const char *format, ...)
4214 {
4215    unsigned long _qzz_res;
4216    va_list vargs;
4217    va_start(vargs, format);
4218    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4219                               VG_USERREQ__PRINTF_VALIST_BY_REF,
4220                               (unsigned long)format,
4221                               (unsigned long)&vargs,
4222                               0, 0, 0);
4223    va_end(vargs);
4224    return (int)_qzz_res;
4225 }
4226
4227 static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4228    __attribute__((format(__printf__, 1, 2), __unused__));
4229 static int
4230 VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
4231 {
4232    unsigned long _qzz_res;
4233    va_list vargs;
4234    va_start(vargs, format);
4235    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,
4236                               VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
4237                               (unsigned long)format,
4238                               (unsigned long)&vargs,
4239                               0, 0, 0);
4240    va_end(vargs);
4241    return (int)_qzz_res;
4242 }
4243
4244 #endif /* NVALGRIND */
4245
4246
4247 /* These requests allow control to move from the simulated CPU to the
4248    real CPU, calling an arbitary function.
4249
4250    Note that the current ThreadId is inserted as the first argument.
4251    So this call:
4252
4253      VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
4254
4255    requires f to have this signature:
4256
4257      Word f(Word tid, Word arg1, Word arg2)
4258
4259    where "Word" is a word-sized type.
4260
4261    Note that these client requests are not entirely reliable.  For example,
4262    if you call a function with them that subsequently calls printf(),
4263    there's a high chance Valgrind will crash.  Generally, your prospects of
4264    these working are made higher if the called function does not refer to
4265    any global variables, and does not refer to any libc or other functions
4266    (printf et al).  Any kind of entanglement with libc or dynamic linking is
4267    likely to have a bad outcome, for tricky reasons which we've grappled
4268    with a lot in the past.
4269 */
4270 #define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
4271    __extension__                                                  \
4272    ({unsigned long _qyy_res;                                      \
4273     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4274                                VG_USERREQ__CLIENT_CALL0,          \
4275                                _qyy_fn,                           \
4276                                0, 0, 0, 0);                       \
4277     _qyy_res;                                                     \
4278    })
4279
4280 #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
4281    __extension__                                                  \
4282    ({unsigned long _qyy_res;                                      \
4283     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4284                                VG_USERREQ__CLIENT_CALL1,          \
4285                                _qyy_fn,                           \
4286                                _qyy_arg1, 0, 0, 0);               \
4287     _qyy_res;                                                     \
4288    })
4289
4290 #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
4291    __extension__                                                  \
4292    ({unsigned long _qyy_res;                                      \
4293     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4294                                VG_USERREQ__CLIENT_CALL2,          \
4295                                _qyy_fn,                           \
4296                                _qyy_arg1, _qyy_arg2, 0, 0);       \
4297     _qyy_res;                                                     \
4298    })
4299
4300 #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
4301    __extension__                                                  \
4302    ({unsigned long _qyy_res;                                      \
4303     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4304                                VG_USERREQ__CLIENT_CALL3,          \
4305                                _qyy_fn,                           \
4306                                _qyy_arg1, _qyy_arg2,              \
4307                                _qyy_arg3, 0);                     \
4308     _qyy_res;                                                     \
4309    })
4310
4311
4312 /* Counts the number of errors that have been recorded by a tool.  Nb:
4313    the tool must record the errors with VG_(maybe_record_error)() or
4314    VG_(unique_error)() for them to be counted. */
4315 #define VALGRIND_COUNT_ERRORS                                     \
4316    __extension__                                                  \
4317    ({unsigned int _qyy_res;                                       \
4318     VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
4319                                VG_USERREQ__COUNT_ERRORS,          \
4320                                0, 0, 0, 0, 0);                    \
4321     _qyy_res;                                                     \
4322    })
4323
4324 /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
4325    when heap blocks are allocated in order to give accurate results.  This
4326    happens automatically for the standard allocator functions such as
4327    malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
4328    delete[], etc.
4329
4330    But if your program uses a custom allocator, this doesn't automatically
4331    happen, and Valgrind will not do as well.  For example, if you allocate
4332    superblocks with mmap() and then allocates chunks of the superblocks, all
4333    Valgrind's observations will be at the mmap() level and it won't know that
4334    the chunks should be considered separate entities.  In Memcheck's case,
4335    that means you probably won't get heap block overrun detection (because
4336    there won't be redzones marked as unaddressable) and you definitely won't
4337    get any leak detection.
4338
4339    The following client requests allow a custom allocator to be annotated so
4340    that it can be handled accurately by Valgrind.
4341
4342    VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
4343    by a malloc()-like function.  For Memcheck (an illustrative case), this
4344    does two things:
4345
4346    - It records that the block has been allocated.  This means any addresses
4347      within the block mentioned in error messages will be
4348      identified as belonging to the block.  It also means that if the block
4349      isn't freed it will be detected by the leak checker.
4350
4351    - It marks the block as being addressable and undefined (if 'is_zeroed' is
4352      not set), or addressable and defined (if 'is_zeroed' is set).  This
4353      controls how accesses to the block by the program are handled.
4354
4355    'addr' is the start of the usable block (ie. after any
4356    redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
4357    can apply redzones -- these are blocks of padding at the start and end of
4358    each block.  Adding redzones is recommended as it makes it much more likely
4359    Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
4360    zeroed (or filled with another predictable value), as is the case for
4361    calloc().
4362
4363    VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
4364    heap block -- that will be used by the client program -- is allocated.
4365    It's best to put it at the outermost level of the allocator if possible;
4366    for example, if you have a function my_alloc() which calls
4367    internal_alloc(), and the client request is put inside internal_alloc(),
4368    stack traces relating to the heap block will contain entries for both
4369    my_alloc() and internal_alloc(), which is probably not what you want.
4370
4371    For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
4372    custom blocks from within a heap block, B, that has been allocated with
4373    malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
4374    -- the custom blocks will take precedence.
4375
4376    VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
4377    Memcheck, it does two things:
4378
4379    - It records that the block has been deallocated.  This assumes that the
4380      block was annotated as having been allocated via
4381      VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
4382
4383    - It marks the block as being unaddressable.
4384
4385    VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
4386    heap block is deallocated.
4387
4388    In many cases, these two client requests will not be enough to get your
4389    allocator working well with Memcheck.  More specifically, if your allocator
4390    writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
4391    will be necessary to mark the memory as addressable just before the zeroing
4392    occurs, otherwise you'll get a lot of invalid write errors.  For example,
4393    you'll need to do this if your allocator recycles freed blocks, but it
4394    zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
4395    Alternatively, if your allocator reuses freed blocks for allocator-internal
4396    data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
4397
4398    Really, what's happening is a blurring of the lines between the client
4399    program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
4400    memory should be considered unaddressable to the client program, but the
4401    allocator knows more than the rest of the client program and so may be able
4402    to safely access it.  Extra client requests are necessary for Valgrind to
4403    understand the distinction between the allocator and the rest of the
4404    program.
4405
4406    Note: there is currently no VALGRIND_REALLOCLIKE_BLOCK client request;  it
4407    has to be emulated with MALLOCLIKE/FREELIKE and memory copying.
4408
4409    Ignored if addr == 0.
4410 */
4411 #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
4412    {unsigned int __unused _qzz_res;                               \
4413     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4414                                VG_USERREQ__MALLOCLIKE_BLOCK,      \
4415                                addr, sizeB, rzB, is_zeroed, 0);   \
4416    }
4417
4418 /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
4419    Ignored if addr == 0.
4420 */
4421 #define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
4422    {unsigned int __unused _qzz_res;                               \
4423     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4424                                VG_USERREQ__FREELIKE_BLOCK,        \
4425                                addr, rzB, 0, 0, 0);               \
4426    }
4427
4428 /* Create a memory pool. */
4429 #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
4430    {unsigned int _qzz_res;                                        \
4431     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4432                                VG_USERREQ__CREATE_MEMPOOL,        \
4433                                pool, rzB, is_zeroed, 0, 0);       \
4434    }
4435
4436 /* Destroy a memory pool. */
4437 #define VALGRIND_DESTROY_MEMPOOL(pool)                            \
4438    {unsigned int _qzz_res;                                        \
4439     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4440                                VG_USERREQ__DESTROY_MEMPOOL,       \
4441                                pool, 0, 0, 0, 0);                 \
4442    }
4443
4444 /* Associate a piece of memory with a memory pool. */
4445 #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
4446    {unsigned int _qzz_res;                                        \
4447     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4448                                VG_USERREQ__MEMPOOL_ALLOC,         \
4449                                pool, addr, size, 0, 0);           \
4450    }
4451
4452 /* Disassociate a piece of memory from a memory pool. */
4453 #define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
4454    {unsigned int _qzz_res;                                        \
4455     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4456                                VG_USERREQ__MEMPOOL_FREE,          \
4457                                pool, addr, 0, 0, 0);              \
4458    }
4459
4460 /* Disassociate any pieces outside a particular range. */
4461 #define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
4462    {unsigned int _qzz_res;                                        \
4463     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4464                                VG_USERREQ__MEMPOOL_TRIM,          \
4465                                pool, addr, size, 0, 0);           \
4466    }
4467
4468 /* Resize and/or move a piece associated with a memory pool. */
4469 #define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
4470    {unsigned int _qzz_res;                                        \
4471     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4472                                VG_USERREQ__MOVE_MEMPOOL,          \
4473                                poolA, poolB, 0, 0, 0);            \
4474    }
4475
4476 /* Resize and/or move a piece associated with a memory pool. */
4477 #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
4478    {unsigned int _qzz_res;                                        \
4479     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4480                                VG_USERREQ__MEMPOOL_CHANGE,        \
4481                                pool, addrA, addrB, size, 0);      \
4482    }
4483
4484 /* Return 1 if a mempool exists, else 0. */
4485 #define VALGRIND_MEMPOOL_EXISTS(pool)                             \
4486    __extension__                                                  \
4487    ({unsigned int _qzz_res;                                       \
4488     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4489                                VG_USERREQ__MEMPOOL_EXISTS,        \
4490                                pool, 0, 0, 0, 0);                 \
4491     _qzz_res;                                                     \
4492    })
4493
4494 /* Mark a piece of memory as being a stack. Returns a stack id. */
4495 #define VALGRIND_STACK_REGISTER(start, end)                       \
4496    __extension__                                                  \
4497    ({unsigned int _qzz_res;                                       \
4498     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4499                                VG_USERREQ__STACK_REGISTER,        \
4500                                start, end, 0, 0, 0);              \
4501     _qzz_res;                                                     \
4502    })
4503
4504 /* Unmark the piece of memory associated with a stack id as being a
4505    stack. */
4506 #define VALGRIND_STACK_DEREGISTER(id)                             \
4507    {unsigned int _qzz_res;                                        \
4508     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4509                                VG_USERREQ__STACK_DEREGISTER,      \
4510                                id, 0, 0, 0, 0);                   \
4511    }
4512
4513 /* Change the start and end address of the stack id. */
4514 #define VALGRIND_STACK_CHANGE(id, start, end)                     \
4515    {unsigned int _qzz_res;                                        \
4516     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4517                                VG_USERREQ__STACK_CHANGE,          \
4518                                id, start, end, 0, 0);             \
4519    }
4520
4521 /* Load PDB debug info for Wine PE image_map. */
4522 #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)   \
4523    {unsigned int _qzz_res;                                        \
4524     VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
4525                                VG_USERREQ__LOAD_PDB_DEBUGINFO,    \
4526                                fd, ptr, total_size, delta, 0);    \
4527    }
4528
4529
4530 #undef PLAT_x86_linux
4531 #undef PLAT_amd64_linux
4532 #undef PLAT_ppc32_linux
4533 #undef PLAT_ppc64_linux
4534 #undef PLAT_arm_linux
4535 #undef PLAT_ppc32_aix5
4536 #undef PLAT_ppc64_aix5
4537
4538 #endif   /* __VALGRIND_H */