kernel/arch/x86/lib/usercopy_32.c

   1 /*
   2  * User address space access functions.
   3  * The non inlined parts of asm-i386/uaccess.h are here.
   4  *
   5  * Copyright 1997 Andi Kleen <ak@muc.de>
   6  * Copyright 1997 Linus Torvalds
   7  */
   8 #include <linux/mm.h>
   9 #include <linux/highmem.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/module.h>
  12 #include <linux/backing-dev.h>
  13 #include <linux/interrupt.h>
  14 #include <asm/uaccess.h>
  15 #include <asm/mmx.h>
  16 #include <asm/asm.h>
  17
  18 #ifdef CONFIG_X86_INTEL_USERCOPY
  19 /*
  20  * Alignment at which movsl is preferred for bulk memory copies.
  21  */
  22 struct movsl_mask movsl_mask __read_mostly;
  23 #endif
  24
  25 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  26 {
  27 #ifdef CONFIG_X86_INTEL_USERCOPY
  28         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  29                 return 0;
  30 #endif
  31         return 1;
  32 }
  33 #define movsl_is_ok(a1, a2, n) \
  34         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
  35
  36 /*
  37  * Zero Userspace
  38  */
  39
  40 #define __do_clear_user(addr,size)                                      \
  41 do {                                                                    \
  42         int __d0;                                                       \
  43         might_fault();                                                  \
  44         __asm__ __volatile__(                                           \
  45                 ASM_STAC "\n"                                           \
  46                 "0:     rep; stosl\n"                                   \
  47                 "       movl %2,%0\n"                                   \
  48                 "1:     rep; stosb\n"                                   \
  49                 "2: " ASM_CLAC "\n"                                     \
  50                 ".section .fixup,\"ax\"\n"                              \
  51                 "3:     lea 0(%2,%0,4),%0\n"                            \
  52                 "       jmp 2b\n"                                       \
  53                 ".previous\n"                                           \
  54                 _ASM_EXTABLE(0b,3b)                                     \
  55                 _ASM_EXTABLE(1b,2b)                                     \
  56                 : "=&c"(size), "=&D" (__d0)                             \
  57                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
  58 } while (0)
  59
  60 /**
  61  * clear_user: - Zero a block of memory in user space.
  62  * @to:   Destination address, in user space.
  63  * @n:    Number of bytes to zero.
  64  *
  65  * Zero a block of memory in user space.
  66  *
  67  * Returns number of bytes that could not be cleared.
  68  * On success, this will be zero.
  69  */
  70 unsigned long
  71 clear_user(void __user *to, unsigned long n)
  72 {
  73         might_fault();
  74         if (access_ok(VERIFY_WRITE, to, n))
  75                 __do_clear_user(to, n);
  76         return n;
  77 }
  78 EXPORT_SYMBOL(clear_user);
  79
  80 /**
  81  * __clear_user: - Zero a block of memory in user space, with less checking.
  82  * @to:   Destination address, in user space.
  83  * @n:    Number of bytes to zero.
  84  *
  85  * Zero a block of memory in user space.  Caller must check
  86  * the specified block with access_ok() before calling this function.
  87  *
  88  * Returns number of bytes that could not be cleared.
  89  * On success, this will be zero.
  90  */
  91 unsigned long
  92 __clear_user(void __user *to, unsigned long n)
  93 {
  94         __do_clear_user(to, n);
  95         return n;
  96 }
  97 EXPORT_SYMBOL(__clear_user);
  98
  99 #ifdef CONFIG_X86_INTEL_USERCOPY
 100 static unsigned long
 101 __copy_user_intel(void __user *to, const void *from, unsigned long size)
 102 {
 103         int d0, d1;
 104         __asm__ __volatile__(
 105                        "       .align 2,0x90\n"
 106                        "1:     movl 32(%4), %%eax\n"
 107                        "       cmpl $67, %0\n"
 108                        "       jbe 3f\n"
 109                        "2:     movl 64(%4), %%eax\n"
 110                        "       .align 2,0x90\n"
 111                        "3:     movl 0(%4), %%eax\n"
 112                        "4:     movl 4(%4), %%edx\n"
 113                        "5:     movl %%eax, 0(%3)\n"
 114                        "6:     movl %%edx, 4(%3)\n"
 115                        "7:     movl 8(%4), %%eax\n"
 116                        "8:     movl 12(%4),%%edx\n"
 117                        "9:     movl %%eax, 8(%3)\n"
 118                        "10:    movl %%edx, 12(%3)\n"
 119                        "11:    movl 16(%4), %%eax\n"
 120                        "12:    movl 20(%4), %%edx\n"
 121                        "13:    movl %%eax, 16(%3)\n"
 122                        "14:    movl %%edx, 20(%3)\n"
 123                        "15:    movl 24(%4), %%eax\n"
 124                        "16:    movl 28(%4), %%edx\n"
 125                        "17:    movl %%eax, 24(%3)\n"
 126                        "18:    movl %%edx, 28(%3)\n"
 127                        "19:    movl 32(%4), %%eax\n"
 128                        "20:    movl 36(%4), %%edx\n"
 129                        "21:    movl %%eax, 32(%3)\n"
 130                        "22:    movl %%edx, 36(%3)\n"
 131                        "23:    movl 40(%4), %%eax\n"
 132                        "24:    movl 44(%4), %%edx\n"
 133                        "25:    movl %%eax, 40(%3)\n"
 134                        "26:    movl %%edx, 44(%3)\n"
 135                        "27:    movl 48(%4), %%eax\n"
 136                        "28:    movl 52(%4), %%edx\n"
 137                        "29:    movl %%eax, 48(%3)\n"
 138                        "30:    movl %%edx, 52(%3)\n"
 139                        "31:    movl 56(%4), %%eax\n"
 140                        "32:    movl 60(%4), %%edx\n"
 141                        "33:    movl %%eax, 56(%3)\n"
 142                        "34:    movl %%edx, 60(%3)\n"
 143                        "       addl $-64, %0\n"
 144                        "       addl $64, %4\n"
 145                        "       addl $64, %3\n"
 146                        "       cmpl $63, %0\n"
 147                        "       ja  1b\n"
 148                        "35:    movl  %0, %%eax\n"
 149                        "       shrl  $2, %0\n"
 150                        "       andl  $3, %%eax\n"
 151                        "       cld\n"
 152                        "99:    rep; movsl\n"
 153                        "36:    movl %%eax, %0\n"
 154                        "37:    rep; movsb\n"
 155                        "100:\n"
 156                        ".section .fixup,\"ax\"\n"
 157                        "101:   lea 0(%%eax,%0,4),%0\n"
 158                        "       jmp 100b\n"
 159                        ".previous\n"
 160                        _ASM_EXTABLE(1b,100b)
 161                        _ASM_EXTABLE(2b,100b)
 162                        _ASM_EXTABLE(3b,100b)
 163                        _ASM_EXTABLE(4b,100b)
 164                        _ASM_EXTABLE(5b,100b)
 165                        _ASM_EXTABLE(6b,100b)
 166                        _ASM_EXTABLE(7b,100b)
 167                        _ASM_EXTABLE(8b,100b)
 168                        _ASM_EXTABLE(9b,100b)
 169                        _ASM_EXTABLE(10b,100b)
 170                        _ASM_EXTABLE(11b,100b)
 171                        _ASM_EXTABLE(12b,100b)
 172                        _ASM_EXTABLE(13b,100b)
 173                        _ASM_EXTABLE(14b,100b)
 174                        _ASM_EXTABLE(15b,100b)
 175                        _ASM_EXTABLE(16b,100b)
 176                        _ASM_EXTABLE(17b,100b)
 177                        _ASM_EXTABLE(18b,100b)
 178                        _ASM_EXTABLE(19b,100b)
 179                        _ASM_EXTABLE(20b,100b)
 180                        _ASM_EXTABLE(21b,100b)
 181                        _ASM_EXTABLE(22b,100b)
 182                        _ASM_EXTABLE(23b,100b)
 183                        _ASM_EXTABLE(24b,100b)
 184                        _ASM_EXTABLE(25b,100b)
 185                        _ASM_EXTABLE(26b,100b)
 186                        _ASM_EXTABLE(27b,100b)
 187                        _ASM_EXTABLE(28b,100b)
 188                        _ASM_EXTABLE(29b,100b)
 189                        _ASM_EXTABLE(30b,100b)
 190                        _ASM_EXTABLE(31b,100b)
 191                        _ASM_EXTABLE(32b,100b)
 192                        _ASM_EXTABLE(33b,100b)
 193                        _ASM_EXTABLE(34b,100b)
 194                        _ASM_EXTABLE(35b,100b)
 195                        _ASM_EXTABLE(36b,100b)
 196                        _ASM_EXTABLE(37b,100b)
 197                        _ASM_EXTABLE(99b,101b)
 198                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 199                        :  "1"(to), "2"(from), "0"(size)
 200                        : "eax", "edx", "memory");
 201         return size;
 202 }
 203
 204 static unsigned long
 205 __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size)
 206 {
 207         int d0, d1;
 208         __asm__ __volatile__(
 209                        "        .align 2,0x90\n"
 210                        "0:      movl 32(%4), %%eax\n"
 211                        "        cmpl $67, %0\n"
 212                        "        jbe 2f\n"
 213                        "1:      movl 64(%4), %%eax\n"
 214                        "        .align 2,0x90\n"
 215                        "2:      movl 0(%4), %%eax\n"
 216                        "21:     movl 4(%4), %%edx\n"
 217                        "        movl %%eax, 0(%3)\n"
 218                        "        movl %%edx, 4(%3)\n"
 219                        "3:      movl 8(%4), %%eax\n"
 220                        "31:     movl 12(%4),%%edx\n"
 221                        "        movl %%eax, 8(%3)\n"
 222                        "        movl %%edx, 12(%3)\n"
 223                        "4:      movl 16(%4), %%eax\n"
 224                        "41:     movl 20(%4), %%edx\n"
 225                        "        movl %%eax, 16(%3)\n"
 226                        "        movl %%edx, 20(%3)\n"
 227                        "10:     movl 24(%4), %%eax\n"
 228                        "51:     movl 28(%4), %%edx\n"
 229                        "        movl %%eax, 24(%3)\n"
 230                        "        movl %%edx, 28(%3)\n"
 231                        "11:     movl 32(%4), %%eax\n"
 232                        "61:     movl 36(%4), %%edx\n"
 233                        "        movl %%eax, 32(%3)\n"
 234                        "        movl %%edx, 36(%3)\n"
 235                        "12:     movl 40(%4), %%eax\n"
 236                        "71:     movl 44(%4), %%edx\n"
 237                        "        movl %%eax, 40(%3)\n"
 238                        "        movl %%edx, 44(%3)\n"
 239                        "13:     movl 48(%4), %%eax\n"
 240                        "81:     movl 52(%4), %%edx\n"
 241                        "        movl %%eax, 48(%3)\n"
 242                        "        movl %%edx, 52(%3)\n"
 243                        "14:     movl 56(%4), %%eax\n"
 244                        "91:     movl 60(%4), %%edx\n"
 245                        "        movl %%eax, 56(%3)\n"
 246                        "        movl %%edx, 60(%3)\n"
 247                        "        addl $-64, %0\n"
 248                        "        addl $64, %4\n"
 249                        "        addl $64, %3\n"
 250                        "        cmpl $63, %0\n"
 251                        "        ja  0b\n"
 252                        "5:      movl  %0, %%eax\n"
 253                        "        shrl  $2, %0\n"
 254                        "        andl $3, %%eax\n"
 255                        "        cld\n"
 256                        "6:      rep; movsl\n"
 257                        "        movl %%eax,%0\n"
 258                        "7:      rep; movsb\n"
 259                        "8:\n"
 260                        ".section .fixup,\"ax\"\n"
 261                        "9:      lea 0(%%eax,%0,4),%0\n"
 262                        "16:     pushl %0\n"
 263                        "        pushl %%eax\n"
 264                        "        xorl %%eax,%%eax\n"
 265                        "        rep; stosb\n"
 266                        "        popl %%eax\n"
 267                        "        popl %0\n"
 268                        "        jmp 8b\n"
 269                        ".previous\n"
 270                        _ASM_EXTABLE(0b,16b)
 271                        _ASM_EXTABLE(1b,16b)
 272                        _ASM_EXTABLE(2b,16b)
 273                        _ASM_EXTABLE(21b,16b)
 274                        _ASM_EXTABLE(3b,16b)
 275                        _ASM_EXTABLE(31b,16b)
 276                        _ASM_EXTABLE(4b,16b)
 277                        _ASM_EXTABLE(41b,16b)
 278                        _ASM_EXTABLE(10b,16b)
 279                        _ASM_EXTABLE(51b,16b)
 280                        _ASM_EXTABLE(11b,16b)
 281                        _ASM_EXTABLE(61b,16b)
 282                        _ASM_EXTABLE(12b,16b)
 283                        _ASM_EXTABLE(71b,16b)
 284                        _ASM_EXTABLE(13b,16b)
 285                        _ASM_EXTABLE(81b,16b)
 286                        _ASM_EXTABLE(14b,16b)
 287                        _ASM_EXTABLE(91b,16b)
 288                        _ASM_EXTABLE(6b,9b)
 289                        _ASM_EXTABLE(7b,16b)
 290                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 291                        :  "1"(to), "2"(from), "0"(size)
 292                        : "eax", "edx", "memory");
 293         return size;
 294 }
 295
 296 /*
 297  * Non Temporal Hint version of __copy_user_zeroing_intel.  It is cache aware.
 298  * hyoshiok@miraclelinux.com
 299  */
 300
 301 static unsigned long __copy_user_zeroing_intel_nocache(void *to,
 302                                 const void __user *from, unsigned long size)
 303 {
 304         int d0, d1;
 305
 306         __asm__ __volatile__(
 307                "        .align 2,0x90\n"
 308                "0:      movl 32(%4), %%eax\n"
 309                "        cmpl $67, %0\n"
 310                "        jbe 2f\n"
 311                "1:      movl 64(%4), %%eax\n"
 312                "        .align 2,0x90\n"
 313                "2:      movl 0(%4), %%eax\n"
 314                "21:     movl 4(%4), %%edx\n"
 315                "        movnti %%eax, 0(%3)\n"
 316                "        movnti %%edx, 4(%3)\n"
 317                "3:      movl 8(%4), %%eax\n"
 318                "31:     movl 12(%4),%%edx\n"
 319                "        movnti %%eax, 8(%3)\n"
 320                "        movnti %%edx, 12(%3)\n"
 321                "4:      movl 16(%4), %%eax\n"
 322                "41:     movl 20(%4), %%edx\n"
 323                "        movnti %%eax, 16(%3)\n"
 324                "        movnti %%edx, 20(%3)\n"
 325                "10:     movl 24(%4), %%eax\n"
 326                "51:     movl 28(%4), %%edx\n"
 327                "        movnti %%eax, 24(%3)\n"
 328                "        movnti %%edx, 28(%3)\n"
 329                "11:     movl 32(%4), %%eax\n"
 330                "61:     movl 36(%4), %%edx\n"
 331                "        movnti %%eax, 32(%3)\n"
 332                "        movnti %%edx, 36(%3)\n"
 333                "12:     movl 40(%4), %%eax\n"
 334                "71:     movl 44(%4), %%edx\n"
 335                "        movnti %%eax, 40(%3)\n"
 336                "        movnti %%edx, 44(%3)\n"
 337                "13:     movl 48(%4), %%eax\n"
 338                "81:     movl 52(%4), %%edx\n"
 339                "        movnti %%eax, 48(%3)\n"
 340                "        movnti %%edx, 52(%3)\n"
 341                "14:     movl 56(%4), %%eax\n"
 342                "91:     movl 60(%4), %%edx\n"
 343                "        movnti %%eax, 56(%3)\n"
 344                "        movnti %%edx, 60(%3)\n"
 345                "        addl $-64, %0\n"
 346                "        addl $64, %4\n"
 347                "        addl $64, %3\n"
 348                "        cmpl $63, %0\n"
 349                "        ja  0b\n"
 350                "        sfence \n"
 351                "5:      movl  %0, %%eax\n"
 352                "        shrl  $2, %0\n"
 353                "        andl $3, %%eax\n"
 354                "        cld\n"
 355                "6:      rep; movsl\n"
 356                "        movl %%eax,%0\n"
 357                "7:      rep; movsb\n"
 358                "8:\n"
 359                ".section .fixup,\"ax\"\n"
 360                "9:      lea 0(%%eax,%0,4),%0\n"
 361                "16:     pushl %0\n"
 362                "        pushl %%eax\n"
 363                "        xorl %%eax,%%eax\n"
 364                "        rep; stosb\n"
 365                "        popl %%eax\n"
 366                "        popl %0\n"
 367                "        jmp 8b\n"
 368                ".previous\n"
 369                _ASM_EXTABLE(0b,16b)
 370                _ASM_EXTABLE(1b,16b)
 371                _ASM_EXTABLE(2b,16b)
 372                _ASM_EXTABLE(21b,16b)
 373                _ASM_EXTABLE(3b,16b)
 374                _ASM_EXTABLE(31b,16b)
 375                _ASM_EXTABLE(4b,16b)
 376                _ASM_EXTABLE(41b,16b)
 377                _ASM_EXTABLE(10b,16b)
 378                _ASM_EXTABLE(51b,16b)
 379                _ASM_EXTABLE(11b,16b)
 380                _ASM_EXTABLE(61b,16b)
 381                _ASM_EXTABLE(12b,16b)
 382                _ASM_EXTABLE(71b,16b)
 383                _ASM_EXTABLE(13b,16b)
 384                _ASM_EXTABLE(81b,16b)
 385                _ASM_EXTABLE(14b,16b)
 386                _ASM_EXTABLE(91b,16b)
 387                _ASM_EXTABLE(6b,9b)
 388                _ASM_EXTABLE(7b,16b)
 389                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 390                :  "1"(to), "2"(from), "0"(size)
 391                : "eax", "edx", "memory");
 392         return size;
 393 }
 394
 395 static unsigned long __copy_user_intel_nocache(void *to,
 396                                 const void __user *from, unsigned long size)
 397 {
 398         int d0, d1;
 399
 400         __asm__ __volatile__(
 401                "        .align 2,0x90\n"
 402                "0:      movl 32(%4), %%eax\n"
 403                "        cmpl $67, %0\n"
 404                "        jbe 2f\n"
 405                "1:      movl 64(%4), %%eax\n"
 406                "        .align 2,0x90\n"
 407                "2:      movl 0(%4), %%eax\n"
 408                "21:     movl 4(%4), %%edx\n"
 409                "        movnti %%eax, 0(%3)\n"
 410                "        movnti %%edx, 4(%3)\n"
 411                "3:      movl 8(%4), %%eax\n"
 412                "31:     movl 12(%4),%%edx\n"
 413                "        movnti %%eax, 8(%3)\n"
 414                "        movnti %%edx, 12(%3)\n"
 415                "4:      movl 16(%4), %%eax\n"
 416                "41:     movl 20(%4), %%edx\n"
 417                "        movnti %%eax, 16(%3)\n"
 418                "        movnti %%edx, 20(%3)\n"
 419                "10:     movl 24(%4), %%eax\n"
 420                "51:     movl 28(%4), %%edx\n"
 421                "        movnti %%eax, 24(%3)\n"
 422                "        movnti %%edx, 28(%3)\n"
 423                "11:     movl 32(%4), %%eax\n"
 424                "61:     movl 36(%4), %%edx\n"
 425                "        movnti %%eax, 32(%3)\n"
 426                "        movnti %%edx, 36(%3)\n"
 427                "12:     movl 40(%4), %%eax\n"
 428                "71:     movl 44(%4), %%edx\n"
 429                "        movnti %%eax, 40(%3)\n"
 430                "        movnti %%edx, 44(%3)\n"
 431                "13:     movl 48(%4), %%eax\n"
 432                "81:     movl 52(%4), %%edx\n"
 433                "        movnti %%eax, 48(%3)\n"
 434                "        movnti %%edx, 52(%3)\n"
 435                "14:     movl 56(%4), %%eax\n"
 436                "91:     movl 60(%4), %%edx\n"
 437                "        movnti %%eax, 56(%3)\n"
 438                "        movnti %%edx, 60(%3)\n"
 439                "        addl $-64, %0\n"
 440                "        addl $64, %4\n"
 441                "        addl $64, %3\n"
 442                "        cmpl $63, %0\n"
 443                "        ja  0b\n"
 444                "        sfence \n"
 445                "5:      movl  %0, %%eax\n"
 446                "        shrl  $2, %0\n"
 447                "        andl $3, %%eax\n"
 448                "        cld\n"
 449                "6:      rep; movsl\n"
 450                "        movl %%eax,%0\n"
 451                "7:      rep; movsb\n"
 452                "8:\n"
 453                ".section .fixup,\"ax\"\n"
 454                "9:      lea 0(%%eax,%0,4),%0\n"
 455                "16:     jmp 8b\n"
 456                ".previous\n"
 457                _ASM_EXTABLE(0b,16b)
 458                _ASM_EXTABLE(1b,16b)
 459                _ASM_EXTABLE(2b,16b)
 460                _ASM_EXTABLE(21b,16b)
 461                _ASM_EXTABLE(3b,16b)
 462                _ASM_EXTABLE(31b,16b)
 463                _ASM_EXTABLE(4b,16b)
 464                _ASM_EXTABLE(41b,16b)
 465                _ASM_EXTABLE(10b,16b)
 466                _ASM_EXTABLE(51b,16b)
 467                _ASM_EXTABLE(11b,16b)
 468                _ASM_EXTABLE(61b,16b)
 469                _ASM_EXTABLE(12b,16b)
 470                _ASM_EXTABLE(71b,16b)
 471                _ASM_EXTABLE(13b,16b)
 472                _ASM_EXTABLE(81b,16b)
 473                _ASM_EXTABLE(14b,16b)
 474                _ASM_EXTABLE(91b,16b)
 475                _ASM_EXTABLE(6b,9b)
 476                _ASM_EXTABLE(7b,16b)
 477                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 478                :  "1"(to), "2"(from), "0"(size)
 479                : "eax", "edx", "memory");
 480         return size;
 481 }
 482
 483 #else
 484
 485 /*
 486  * Leave these declared but undefined.  They should not be any references to
 487  * them
 488  */
 489 unsigned long __copy_user_zeroing_intel(void *to, const void __user *from,
 490                                         unsigned long size);
 491 unsigned long __copy_user_intel(void __user *to, const void *from,
 492                                         unsigned long size);
 493 unsigned long __copy_user_zeroing_intel_nocache(void *to,
 494                                 const void __user *from, unsigned long size);
 495 #endif /* CONFIG_X86_INTEL_USERCOPY */
 496
 497 /* Generic arbitrary sized copy.  */
 498 #define __copy_user(to, from, size)                                     \
 499 do {                                                                    \
 500         int __d0, __d1, __d2;                                           \
 501         __asm__ __volatile__(                                           \
 502                 "       cmp  $7,%0\n"                                   \
 503                 "       jbe  1f\n"                                      \
 504                 "       movl %1,%0\n"                                   \
 505                 "       negl %0\n"                                      \
 506                 "       andl $7,%0\n"                                   \
 507                 "       subl %0,%3\n"                                   \
 508                 "4:     rep; movsb\n"                                   \
 509                 "       movl %3,%0\n"                                   \
 510                 "       shrl $2,%0\n"                                   \
 511                 "       andl $3,%3\n"                                   \
 512                 "       .align 2,0x90\n"                                \
 513                 "0:     rep; movsl\n"                                   \
 514                 "       movl %3,%0\n"                                   \
 515                 "1:     rep; movsb\n"                                   \
 516                 "2:\n"                                                  \
 517                 ".section .fixup,\"ax\"\n"                              \
 518                 "5:     addl %3,%0\n"                                   \
 519                 "       jmp 2b\n"                                       \
 520                 "3:     lea 0(%3,%0,4),%0\n"                            \
 521                 "       jmp 2b\n"                                       \
 522                 ".previous\n"                                           \
 523                 _ASM_EXTABLE(4b,5b)                                     \
 524                 _ASM_EXTABLE(0b,3b)                                     \
 525                 _ASM_EXTABLE(1b,2b)                                     \
 526                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 527                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 528                 : "memory");                                            \
 529 } while (0)
 530
 531 #define __copy_user_zeroing(to, from, size)                             \
 532 do {                                                                    \
 533         int __d0, __d1, __d2;                                           \
 534         __asm__ __volatile__(                                           \
 535                 "       cmp  $7,%0\n"                                   \
 536                 "       jbe  1f\n"                                      \
 537                 "       movl %1,%0\n"                                   \
 538                 "       negl %0\n"                                      \
 539                 "       andl $7,%0\n"                                   \
 540                 "       subl %0,%3\n"                                   \
 541                 "4:     rep; movsb\n"                                   \
 542                 "       movl %3,%0\n"                                   \
 543                 "       shrl $2,%0\n"                                   \
 544                 "       andl $3,%3\n"                                   \
 545                 "       .align 2,0x90\n"                                \
 546                 "0:     rep; movsl\n"                                   \
 547                 "       movl %3,%0\n"                                   \
 548                 "1:     rep; movsb\n"                                   \
 549                 "2:\n"                                                  \
 550                 ".section .fixup,\"ax\"\n"                              \
 551                 "5:     addl %3,%0\n"                                   \
 552                 "       jmp 6f\n"                                       \
 553                 "3:     lea 0(%3,%0,4),%0\n"                            \
 554                 "6:     pushl %0\n"                                     \
 555                 "       pushl %%eax\n"                                  \
 556                 "       xorl %%eax,%%eax\n"                             \
 557                 "       rep; stosb\n"                                   \
 558                 "       popl %%eax\n"                                   \
 559                 "       popl %0\n"                                      \
 560                 "       jmp 2b\n"                                       \
 561                 ".previous\n"                                           \
 562                 _ASM_EXTABLE(4b,5b)                                     \
 563                 _ASM_EXTABLE(0b,3b)                                     \
 564                 _ASM_EXTABLE(1b,6b)                                     \
 565                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 566                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 567                 : "memory");                                            \
 568 } while (0)
 569
 570 unsigned long __copy_to_user_ll(void __user *to, const void *from,
 571                                 unsigned long n)
 572 {
 573         stac();
 574         if (movsl_is_ok(to, from, n))
 575                 __copy_user(to, from, n);
 576         else
 577                 n = __copy_user_intel(to, from, n);
 578         clac();
 579         return n;
 580 }
 581 EXPORT_SYMBOL(__copy_to_user_ll);
 582
 583 unsigned long __copy_from_user_ll(void *to, const void __user *from,
 584                                         unsigned long n)
 585 {
 586         stac();
 587         if (movsl_is_ok(to, from, n))
 588                 __copy_user_zeroing(to, from, n);
 589         else
 590                 n = __copy_user_zeroing_intel(to, from, n);
 591         clac();
 592         return n;
 593 }
 594 EXPORT_SYMBOL(__copy_from_user_ll);
 595
 596 unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from,
 597                                          unsigned long n)
 598 {
 599         stac();
 600         if (movsl_is_ok(to, from, n))
 601                 __copy_user(to, from, n);
 602         else
 603                 n = __copy_user_intel((void __user *)to,
 604                                       (const void *)from, n);
 605         clac();
 606         return n;
 607 }
 608 EXPORT_SYMBOL(__copy_from_user_ll_nozero);
 609
 610 unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
 611                                         unsigned long n)
 612 {
 613         stac();
 614 #ifdef CONFIG_X86_INTEL_USERCOPY
 615         if (n > 64 && cpu_has_xmm2)
 616                 n = __copy_user_zeroing_intel_nocache(to, from, n);
 617         else
 618                 __copy_user_zeroing(to, from, n);
 619 #else
 620         __copy_user_zeroing(to, from, n);
 621 #endif
 622         clac();
 623         return n;
 624 }
 625 EXPORT_SYMBOL(__copy_from_user_ll_nocache);
 626
 627 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 628                                         unsigned long n)
 629 {
 630         stac();
 631 #ifdef CONFIG_X86_INTEL_USERCOPY
 632         if (n > 64 && cpu_has_xmm2)
 633                 n = __copy_user_intel_nocache(to, from, n);
 634         else
 635                 __copy_user(to, from, n);
 636 #else
 637         __copy_user(to, from, n);
 638 #endif
 639         clac();
 640         return n;
 641 }
 642 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
 643
 644 /**
 645  * copy_to_user: - Copy a block of data into user space.
 646  * @to:   Destination address, in user space.
 647  * @from: Source address, in kernel space.
 648  * @n:    Number of bytes to copy.
 649  *
 650  * Context: User context only. This function may sleep if pagefaults are
 651  *          enabled.
 652  *
 653  * Copy data from kernel space to user space.
 654  *
 655  * Returns number of bytes that could not be copied.
 656  * On success, this will be zero.
 657  */
 658 unsigned long _copy_to_user(void __user *to, const void *from, unsigned n)
 659 {
 660         if (access_ok(VERIFY_WRITE, to, n))
 661                 n = __copy_to_user(to, from, n);
 662         return n;
 663 }
 664 EXPORT_SYMBOL(_copy_to_user);
 665
 666 /**
 667  * copy_from_user: - Copy a block of data from user space.
 668  * @to:   Destination address, in kernel space.
 669  * @from: Source address, in user space.
 670  * @n:    Number of bytes to copy.
 671  *
 672  * Context: User context only. This function may sleep if pagefaults are
 673  *          enabled.
 674  *
 675  * Copy data from user space to kernel space.
 676  *
 677  * Returns number of bytes that could not be copied.
 678  * On success, this will be zero.
 679  *
 680  * If some data could not be copied, this function will pad the copied
 681  * data to the requested size using zero bytes.
 682  */
 683 unsigned long _copy_from_user(void *to, const void __user *from, unsigned n)
 684 {
 685         if (access_ok(VERIFY_READ, from, n))
 686                 n = __copy_from_user(to, from, n);
 687         else
 688                 memset(to, 0, n);
 689         return n;
 690 }
 691 EXPORT_SYMBOL(_copy_from_user);