kernel/arch/metag/lib/usercopy.c

   1 /*
   2  * User address space access functions.
   3  * The non-inlined parts of asm-metag/uaccess.h are here.
   4  *
   5  * Copyright (C) 2006, Imagination Technologies.
   6  * Copyright (C) 2000, Axis Communications AB.
   7  *
   8  * Written by Hans-Peter Nilsson.
   9  * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
  10  * Modified for Meta by Will Newton.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/uaccess.h>
  15 #include <asm/cache.h>                  /* def of L1_CACHE_BYTES */
  16
  17 #define USE_RAPF
  18 #define RAPF_MIN_BUF_SIZE       (3*L1_CACHE_BYTES)
  19
  20
  21 /* The "double write" in this code is because the Meta will not fault
  22  * immediately unless the memory pipe is forced to by e.g. a data stall or
  23  * another memory op. The second write should be discarded by the write
  24  * combiner so should have virtually no cost.
  25  */
  26
  27 #define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  28         asm volatile (                                           \
  29                 COPY                                             \
  30                 "1:\n"                                           \
  31                 "       .section .fixup,\"ax\"\n"                \
  32                 "       MOV D1Ar1,#0\n"                          \
  33                 FIXUP                                            \
  34                 "       MOVT    D1Ar1,#HI(1b)\n"                 \
  35                 "       JUMP    D1Ar1,#LO(1b)\n"                 \
  36                 "       .previous\n"                             \
  37                 "       .section __ex_table,\"a\"\n"             \
  38                 TENTRY                                           \
  39                 "       .previous\n"                             \
  40                 : "=r" (to), "=r" (from), "=r" (ret)             \
  41                 : "0" (to), "1" (from), "2" (ret)                \
  42                 : "D1Ar1", "memory")
  43
  44
  45 #define __asm_copy_to_user_1(to, from, ret)     \
  46         __asm_copy_user_cont(to, from, ret,     \
  47                 "       GETB D1Ar1,[%1++]\n"    \
  48                 "       SETB [%0],D1Ar1\n"      \
  49                 "2:     SETB [%0++],D1Ar1\n",   \
  50                 "3:     ADD  %2,%2,#1\n",       \
  51                 "       .long 2b,3b\n")
  52
  53 #define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  54         __asm_copy_user_cont(to, from, ret,             \
  55                 "       GETW D1Ar1,[%1++]\n"            \
  56                 "       SETW [%0],D1Ar1\n"              \
  57                 "2:     SETW [%0++],D1Ar1\n" COPY,      \
  58                 "3:     ADD  %2,%2,#2\n" FIXUP,         \
  59                 "       .long 2b,3b\n" TENTRY)
  60
  61 #define __asm_copy_to_user_2(to, from, ret) \
  62         __asm_copy_to_user_2x_cont(to, from, ret, "", "", "")
  63
  64 #define __asm_copy_to_user_3(to, from, ret) \
  65         __asm_copy_to_user_2x_cont(to, from, ret,       \
  66                 "       GETB D1Ar1,[%1++]\n"            \
  67                 "       SETB [%0],D1Ar1\n"              \
  68                 "4:     SETB [%0++],D1Ar1\n",           \
  69                 "5:     ADD  %2,%2,#1\n",               \
  70                 "       .long 4b,5b\n")
  71
  72 #define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  73         __asm_copy_user_cont(to, from, ret,             \
  74                 "       GETD D1Ar1,[%1++]\n"            \
  75                 "       SETD [%0],D1Ar1\n"              \
  76                 "2:     SETD [%0++],D1Ar1\n" COPY,      \
  77                 "3:     ADD  %2,%2,#4\n" FIXUP,         \
  78                 "       .long 2b,3b\n" TENTRY)
  79
  80 #define __asm_copy_to_user_4(to, from, ret) \
  81         __asm_copy_to_user_4x_cont(to, from, ret, "", "", "")
  82
  83 #define __asm_copy_to_user_5(to, from, ret) \
  84         __asm_copy_to_user_4x_cont(to, from, ret,       \
  85                 "       GETB D1Ar1,[%1++]\n"            \
  86                 "       SETB [%0],D1Ar1\n"              \
  87                 "4:     SETB [%0++],D1Ar1\n",           \
  88                 "5:     ADD  %2,%2,#1\n",               \
  89                 "       .long 4b,5b\n")
  90
  91 #define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  92         __asm_copy_to_user_4x_cont(to, from, ret,       \
  93                 "       GETW D1Ar1,[%1++]\n"            \
  94                 "       SETW [%0],D1Ar1\n"              \
  95                 "4:     SETW [%0++],D1Ar1\n" COPY,      \
  96                 "5:     ADD  %2,%2,#2\n" FIXUP,         \
  97                 "       .long 4b,5b\n" TENTRY)
  98
  99 #define __asm_copy_to_user_6(to, from, ret) \
 100         __asm_copy_to_user_6x_cont(to, from, ret, "", "", "")
 101
 102 #define __asm_copy_to_user_7(to, from, ret) \
 103         __asm_copy_to_user_6x_cont(to, from, ret,       \
 104                 "       GETB D1Ar1,[%1++]\n"            \
 105                 "       SETB [%0],D1Ar1\n"              \
 106                 "6:     SETB [%0++],D1Ar1\n",           \
 107                 "7:     ADD  %2,%2,#1\n",               \
 108                 "       .long 6b,7b\n")
 109
 110 #define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 111         __asm_copy_to_user_4x_cont(to, from, ret,       \
 112                 "       GETD D1Ar1,[%1++]\n"            \
 113                 "       SETD [%0],D1Ar1\n"              \
 114                 "4:     SETD [%0++],D1Ar1\n" COPY,      \
 115                 "5:     ADD  %2,%2,#4\n"  FIXUP,        \
 116                 "       .long 4b,5b\n" TENTRY)
 117
 118 #define __asm_copy_to_user_8(to, from, ret) \
 119         __asm_copy_to_user_8x_cont(to, from, ret, "", "", "")
 120
 121 #define __asm_copy_to_user_9(to, from, ret) \
 122         __asm_copy_to_user_8x_cont(to, from, ret,       \
 123                 "       GETB D1Ar1,[%1++]\n"            \
 124                 "       SETB [%0],D1Ar1\n"              \
 125                 "6:     SETB [%0++],D1Ar1\n",           \
 126                 "7:     ADD  %2,%2,#1\n",               \
 127                 "       .long 6b,7b\n")
 128
 129 #define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 130         __asm_copy_to_user_8x_cont(to, from, ret,       \
 131                 "       GETW D1Ar1,[%1++]\n"            \
 132                 "       SETW [%0],D1Ar1\n"              \
 133                 "6:     SETW [%0++],D1Ar1\n" COPY,      \
 134                 "7:     ADD  %2,%2,#2\n" FIXUP,         \
 135                 "       .long 6b,7b\n" TENTRY)
 136
 137 #define __asm_copy_to_user_10(to, from, ret) \
 138         __asm_copy_to_user_10x_cont(to, from, ret, "", "", "")
 139
 140 #define __asm_copy_to_user_11(to, from, ret) \
 141         __asm_copy_to_user_10x_cont(to, from, ret,      \
 142                 "       GETB D1Ar1,[%1++]\n"            \
 143                 "       SETB [%0],D1Ar1\n"              \
 144                 "8:     SETB [%0++],D1Ar1\n",           \
 145                 "9:     ADD  %2,%2,#1\n",               \
 146                 "       .long 8b,9b\n")
 147
 148 #define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 149         __asm_copy_to_user_8x_cont(to, from, ret,       \
 150                 "       GETD D1Ar1,[%1++]\n"            \
 151                 "       SETD [%0],D1Ar1\n"              \
 152                 "6:     SETD [%0++],D1Ar1\n" COPY,      \
 153                 "7:     ADD  %2,%2,#4\n" FIXUP,         \
 154                 "       .long 6b,7b\n" TENTRY)
 155 #define __asm_copy_to_user_12(to, from, ret) \
 156         __asm_copy_to_user_12x_cont(to, from, ret, "", "", "")
 157
 158 #define __asm_copy_to_user_13(to, from, ret) \
 159         __asm_copy_to_user_12x_cont(to, from, ret,      \
 160                 "       GETB D1Ar1,[%1++]\n"            \
 161                 "       SETB [%0],D1Ar1\n"              \
 162                 "8:     SETB [%0++],D1Ar1\n",           \
 163                 "9:     ADD  %2,%2,#1\n",               \
 164                 "       .long 8b,9b\n")
 165
 166 #define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 167         __asm_copy_to_user_12x_cont(to, from, ret,      \
 168                 "       GETW D1Ar1,[%1++]\n"            \
 169                 "       SETW [%0],D1Ar1\n"              \
 170                 "8:     SETW [%0++],D1Ar1\n" COPY,      \
 171                 "9:     ADD  %2,%2,#2\n" FIXUP,         \
 172                 "       .long 8b,9b\n" TENTRY)
 173
 174 #define __asm_copy_to_user_14(to, from, ret) \
 175         __asm_copy_to_user_14x_cont(to, from, ret, "", "", "")
 176
 177 #define __asm_copy_to_user_15(to, from, ret) \
 178         __asm_copy_to_user_14x_cont(to, from, ret,      \
 179                 "       GETB D1Ar1,[%1++]\n"            \
 180                 "       SETB [%0],D1Ar1\n"              \
 181                 "10:    SETB [%0++],D1Ar1\n",           \
 182                 "11:    ADD  %2,%2,#1\n",               \
 183                 "       .long 10b,11b\n")
 184
 185 #define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 186         __asm_copy_to_user_12x_cont(to, from, ret,      \
 187                 "       GETD D1Ar1,[%1++]\n"            \
 188                 "       SETD [%0],D1Ar1\n"              \
 189                 "8:     SETD [%0++],D1Ar1\n" COPY,      \
 190                 "9:     ADD  %2,%2,#4\n" FIXUP,         \
 191                 "       .long 8b,9b\n" TENTRY)
 192
 193 #define __asm_copy_to_user_16(to, from, ret) \
 194                 __asm_copy_to_user_16x_cont(to, from, ret, "", "", "")
 195
 196 #define __asm_copy_to_user_8x64(to, from, ret) \
 197         asm volatile (                                  \
 198                 "       GETL D0Ar2,D1Ar1,[%1++]\n"      \
 199                 "       SETL [%0],D0Ar2,D1Ar1\n"        \
 200                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 201                 "1:\n"                                  \
 202                 "       .section .fixup,\"ax\"\n"       \
 203                 "3:     ADD  %2,%2,#8\n"                \
 204                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 205                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 206                 "       .previous\n"                    \
 207                 "       .section __ex_table,\"a\"\n"    \
 208                 "       .long 2b,3b\n"                  \
 209                 "       .previous\n"                    \
 210                 : "=r" (to), "=r" (from), "=r" (ret)    \
 211                 : "0" (to), "1" (from), "2" (ret)       \
 212                 : "D1Ar1", "D0Ar2", "memory")
 213
 214 /*
 215  *      optimized copying loop using RAPF when 64 bit aligned
 216  *
 217  *      n               will be automatically decremented inside the loop
 218  *      ret             will be left intact. if error occurs we will rewind
 219  *                      so that the original non optimized code will fill up
 220  *                      this value correctly.
 221  *
 222  *      on fault:
 223  *              >       n will hold total number of uncopied bytes
 224  *
 225  *              >       {'to','from'} will be rewind back so that
 226  *                      the non-optimized code will do the proper fix up
 227  *
 228  *      DCACHE drops the cacheline which helps in reducing cache
 229  *      pollution.
 230  *
 231  *      We introduce an extra SETL at the end of the loop to
 232  *      ensure we don't fall off the loop before we catch all
 233  *      erros.
 234  *
 235  *      NOTICE:
 236  *              LSM_STEP in TXSTATUS must be cleared in fix up code.
 237  *              since we're using M{S,G}ETL, a fault might happen at
 238  *              any address in the middle of M{S,G}ETL causing
 239  *              the value of LSM_STEP to be incorrect which can
 240  *              cause subsequent use of M{S,G}ET{L,D} to go wrong.
 241  *              ie: if LSM_STEP was 1 when a fault occurs, the
 242  *              next call to M{S,G}ET{L,D} will skip the first
 243  *              copy/getting as it think that the first 1 has already
 244  *              been done.
 245  *
 246  */
 247 #define __asm_copy_user_64bit_rapf_loop(                                \
 248                 to, from, ret, n, id, FIXUP)                            \
 249         asm volatile (                                                  \
 250                 ".balign 8\n"                                           \
 251                 "MOV    RAPF, %1\n"                                     \
 252                 "MSETL  [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n"    \
 253                 "MOV    D0Ar6, #0\n"                                    \
 254                 "LSR    D1Ar5, %3, #6\n"                                \
 255                 "SUB    TXRPT, D1Ar5, #2\n"                             \
 256                 "MOV    RAPF, %1\n"                                     \
 257                 "$Lloop"id":\n"                                         \
 258                 "ADD    RAPF, %1, #64\n"                                \
 259                 "21:\n"                                                 \
 260                 "MGETL  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 261                 "22:\n"                                                 \
 262                 "MSETL  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 263                 "SUB    %3, %3, #32\n"                                  \
 264                 "23:\n"                                                 \
 265                 "MGETL  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 266                 "24:\n"                                                 \
 267                 "MSETL  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 268                 "SUB    %3, %3, #32\n"                                  \
 269                 "DCACHE [%1+#-64], D0Ar6\n"                             \
 270                 "BR     $Lloop"id"\n"                                   \
 271                                                                         \
 272                 "MOV    RAPF, %1\n"                                     \
 273                 "25:\n"                                                 \
 274                 "MGETL  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 275                 "26:\n"                                                 \
 276                 "MSETL  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 277                 "SUB    %3, %3, #32\n"                                  \
 278                 "27:\n"                                                 \
 279                 "MGETL  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 280                 "28:\n"                                                 \
 281                 "MSETL  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 282                 "SUB    %0, %0, #8\n"                                   \
 283                 "29:\n"                                                 \
 284                 "SETL   [%0++], D0.7, D1.7\n"                           \
 285                 "SUB    %3, %3, #32\n"                                  \
 286                 "1:"                                                    \
 287                 "DCACHE [%1+#-64], D0Ar6\n"                             \
 288                 "GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"                  \
 289                 "GETL    D0FrT, D1RtP, [A0StP+#-32]\n"                  \
 290                 "GETL    D0.5, D1.5, [A0StP+#-24]\n"                    \
 291                 "GETL    D0.6, D1.6, [A0StP+#-16]\n"                    \
 292                 "GETL    D0.7, D1.7, [A0StP+#-8]\n"                     \
 293                 "SUB A0StP, A0StP, #40\n"                               \
 294                 "       .section .fixup,\"ax\"\n"                       \
 295                 "4:\n"                                                  \
 296                 "       ADD     %0, %0, #8\n"                           \
 297                 "3:\n"                                                  \
 298                 "       MOV     D0Ar2, TXSTATUS\n"                      \
 299                 "       MOV     D1Ar1, TXSTATUS\n"                      \
 300                 "       AND     D1Ar1, D1Ar1, #0xFFFFF8FF\n"            \
 301                 "       MOV     TXSTATUS, D1Ar1\n"                      \
 302                         FIXUP                                           \
 303                 "       MOVT    D0Ar2,#HI(1b)\n"                        \
 304                 "       JUMP    D0Ar2,#LO(1b)\n"                        \
 305                 "       .previous\n"                                    \
 306                 "       .section __ex_table,\"a\"\n"                    \
 307                 "       .long 21b,3b\n"                                 \
 308                 "       .long 22b,3b\n"                                 \
 309                 "       .long 23b,3b\n"                                 \
 310                 "       .long 24b,3b\n"                                 \
 311                 "       .long 25b,3b\n"                                 \
 312                 "       .long 26b,3b\n"                                 \
 313                 "       .long 27b,3b\n"                                 \
 314                 "       .long 28b,3b\n"                                 \
 315                 "       .long 29b,4b\n"                                 \
 316                 "       .previous\n"                                    \
 317                 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n)          \
 318                 : "0" (to), "1" (from), "2" (ret), "3" (n)              \
 319                 : "D1Ar1", "D0Ar2", "memory")
 320
 321 /*      rewind 'to' and 'from'  pointers when a fault occurs
 322  *
 323  *      Rationale:
 324  *              A fault always occurs on writing to user buffer. A fault
 325  *              is at a single address, so we need to rewind by only 4
 326  *              bytes.
 327  *              Since we do a complete read from kernel buffer before
 328  *              writing, we need to rewind it also. The amount to be
 329  *              rewind equals the number of faulty writes in MSETD
 330  *              which is: [4 - (LSM_STEP-1)]*8
 331  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 332  *              and stored in D0Ar2
 333  *
 334  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 335  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 336  *                      a fault happens at the 4th write, LSM_STEP will be 0
 337  *                      instead of 4. The code copes with that.
 338  *
 339  *              n is updated by the number of successful writes, which is:
 340  *              n = n - (LSM_STEP-1)*8
 341  */
 342 #define __asm_copy_to_user_64bit_rapf_loop(to,  from, ret, n, id)\
 343         __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,           \
 344                 "LSR    D0Ar2, D0Ar2, #8\n"                             \
 345                 "AND    D0Ar2, D0Ar2, #0x7\n"                           \
 346                 "ADDZ   D0Ar2, D0Ar2, #4\n"                             \
 347                 "SUB    D0Ar2, D0Ar2, #1\n"                             \
 348                 "MOV    D1Ar1, #4\n"                                    \
 349                 "SUB    D0Ar2, D1Ar1, D0Ar2\n"                          \
 350                 "LSL    D0Ar2, D0Ar2, #3\n"                             \
 351                 "LSL    D1Ar1, D1Ar1, #3\n"                             \
 352                 "SUB    D1Ar1, D1Ar1, D0Ar2\n"                          \
 353                 "SUB    %0, %0, #8\n"                                   \
 354                 "SUB    %1,     %1,D0Ar2\n"                             \
 355                 "SUB    %3, %3, D1Ar1\n")
 356
 357 /*
 358  *      optimized copying loop using RAPF when 32 bit aligned
 359  *
 360  *      n               will be automatically decremented inside the loop
 361  *      ret             will be left intact. if error occurs we will rewind
 362  *                      so that the original non optimized code will fill up
 363  *                      this value correctly.
 364  *
 365  *      on fault:
 366  *              >       n will hold total number of uncopied bytes
 367  *
 368  *              >       {'to','from'} will be rewind back so that
 369  *                      the non-optimized code will do the proper fix up
 370  *
 371  *      DCACHE drops the cacheline which helps in reducing cache
 372  *      pollution.
 373  *
 374  *      We introduce an extra SETD at the end of the loop to
 375  *      ensure we don't fall off the loop before we catch all
 376  *      erros.
 377  *
 378  *      NOTICE:
 379  *              LSM_STEP in TXSTATUS must be cleared in fix up code.
 380  *              since we're using M{S,G}ETL, a fault might happen at
 381  *              any address in the middle of M{S,G}ETL causing
 382  *              the value of LSM_STEP to be incorrect which can
 383  *              cause subsequent use of M{S,G}ET{L,D} to go wrong.
 384  *              ie: if LSM_STEP was 1 when a fault occurs, the
 385  *              next call to M{S,G}ET{L,D} will skip the first
 386  *              copy/getting as it think that the first 1 has already
 387  *              been done.
 388  *
 389  */
 390 #define __asm_copy_user_32bit_rapf_loop(                                \
 391                         to,     from, ret, n, id, FIXUP)                \
 392         asm volatile (                                                  \
 393                 ".balign 8\n"                                           \
 394                 "MOV    RAPF, %1\n"                                     \
 395                 "MSETL  [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n"    \
 396                 "MOV    D0Ar6, #0\n"                                    \
 397                 "LSR    D1Ar5, %3, #6\n"                                \
 398                 "SUB    TXRPT, D1Ar5, #2\n"                             \
 399                 "MOV    RAPF, %1\n"                                     \
 400         "$Lloop"id":\n"                                                 \
 401                 "ADD    RAPF, %1, #64\n"                                \
 402                 "21:\n"                                                 \
 403                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 404                 "22:\n"                                                 \
 405                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 406                 "SUB    %3, %3, #16\n"                                  \
 407                 "23:\n"                                                 \
 408                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 409                 "24:\n"                                                 \
 410                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 411                 "SUB    %3, %3, #16\n"                                  \
 412                 "25:\n"                                                 \
 413                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 414                 "26:\n"                                                 \
 415                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 416                 "SUB    %3, %3, #16\n"                                  \
 417                 "27:\n"                                                 \
 418                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 419                 "28:\n"                                                 \
 420                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 421                 "SUB    %3, %3, #16\n"                                  \
 422                 "DCACHE [%1+#-64], D0Ar6\n"                             \
 423                 "BR     $Lloop"id"\n"                                   \
 424                                                                         \
 425                 "MOV    RAPF, %1\n"                                     \
 426                 "29:\n"                                                 \
 427                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 428                 "30:\n"                                                 \
 429                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 430                 "SUB    %3, %3, #16\n"                                  \
 431                 "31:\n"                                                 \
 432                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 433                 "32:\n"                                                 \
 434                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 435                 "SUB    %3, %3, #16\n"                                  \
 436                 "33:\n"                                                 \
 437                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 438                 "34:\n"                                                 \
 439                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 440                 "SUB    %3, %3, #16\n"                                  \
 441                 "35:\n"                                                 \
 442                 "MGETD  D0FrT, D0.5, D0.6, D0.7, [%1++]\n"              \
 443                 "36:\n"                                                 \
 444                 "MSETD  [%0++], D0FrT, D0.5, D0.6, D0.7\n"              \
 445                 "SUB    %0, %0, #4\n"                                   \
 446                 "37:\n"                                                 \
 447                 "SETD   [%0++], D0.7\n"                                 \
 448                 "SUB    %3, %3, #16\n"                                  \
 449                 "1:"                                                    \
 450                 "DCACHE [%1+#-64], D0Ar6\n"                             \
 451                 "GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"                  \
 452                 "GETL    D0FrT, D1RtP, [A0StP+#-32]\n"                  \
 453                 "GETL    D0.5, D1.5, [A0StP+#-24]\n"                    \
 454                 "GETL    D0.6, D1.6, [A0StP+#-16]\n"                    \
 455                 "GETL    D0.7, D1.7, [A0StP+#-8]\n"                     \
 456                 "SUB A0StP, A0StP, #40\n"                               \
 457                 "       .section .fixup,\"ax\"\n"                       \
 458                 "4:\n"                                                  \
 459                 "       ADD             %0, %0, #4\n"                   \
 460                 "3:\n"                                                  \
 461                 "       MOV     D0Ar2, TXSTATUS\n"                      \
 462                 "       MOV     D1Ar1, TXSTATUS\n"                      \
 463                 "       AND     D1Ar1, D1Ar1, #0xFFFFF8FF\n"            \
 464                 "       MOV     TXSTATUS, D1Ar1\n"                      \
 465                         FIXUP                                           \
 466                 "       MOVT    D0Ar2,#HI(1b)\n"                        \
 467                 "       JUMP    D0Ar2,#LO(1b)\n"                        \
 468                 "       .previous\n"                                    \
 469                 "       .section __ex_table,\"a\"\n"                    \
 470                 "       .long 21b,3b\n"                                 \
 471                 "       .long 22b,3b\n"                                 \
 472                 "       .long 23b,3b\n"                                 \
 473                 "       .long 24b,3b\n"                                 \
 474                 "       .long 25b,3b\n"                                 \
 475                 "       .long 26b,3b\n"                                 \
 476                 "       .long 27b,3b\n"                                 \
 477                 "       .long 28b,3b\n"                                 \
 478                 "       .long 29b,3b\n"                                 \
 479                 "       .long 30b,3b\n"                                 \
 480                 "       .long 31b,3b\n"                                 \
 481                 "       .long 32b,3b\n"                                 \
 482                 "       .long 33b,3b\n"                                 \
 483                 "       .long 34b,3b\n"                                 \
 484                 "       .long 35b,3b\n"                                 \
 485                 "       .long 36b,3b\n"                                 \
 486                 "       .long 37b,4b\n"                                 \
 487                 "       .previous\n"                                    \
 488                 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n)          \
 489                 : "0" (to), "1" (from), "2" (ret), "3" (n)              \
 490                 : "D1Ar1", "D0Ar2", "memory")
 491
 492 /*      rewind 'to' and 'from'  pointers when a fault occurs
 493  *
 494  *      Rationale:
 495  *              A fault always occurs on writing to user buffer. A fault
 496  *              is at a single address, so we need to rewind by only 4
 497  *              bytes.
 498  *              Since we do a complete read from kernel buffer before
 499  *              writing, we need to rewind it also. The amount to be
 500  *              rewind equals the number of faulty writes in MSETD
 501  *              which is: [4 - (LSM_STEP-1)]*4
 502  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 503  *              and stored in D0Ar2
 504  *
 505  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 506  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 507  *                      a fault happens at the 4th write, LSM_STEP will be 0
 508  *                      instead of 4. The code copes with that.
 509  *
 510  *              n is updated by the number of successful writes, which is:
 511  *              n = n - (LSM_STEP-1)*4
 512  */
 513 #define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
 514         __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,           \
 515                 "LSR    D0Ar2, D0Ar2, #8\n"                             \
 516                 "AND    D0Ar2, D0Ar2, #0x7\n"                           \
 517                 "ADDZ   D0Ar2, D0Ar2, #4\n"                             \
 518                 "SUB    D0Ar2, D0Ar2, #1\n"                             \
 519                 "MOV    D1Ar1, #4\n"                                    \
 520                 "SUB    D0Ar2, D1Ar1, D0Ar2\n"                          \
 521                 "LSL    D0Ar2, D0Ar2, #2\n"                             \
 522                 "LSL    D1Ar1, D1Ar1, #2\n"                             \
 523                 "SUB    D1Ar1, D1Ar1, D0Ar2\n"                          \
 524                 "SUB    %0, %0, #4\n"                                   \
 525                 "SUB    %1,     %1,     D0Ar2\n"                        \
 526                 "SUB    %3, %3, D1Ar1\n")
 527
 528 unsigned long __copy_user(void __user *pdst, const void *psrc,
 529                           unsigned long n)
 530 {
 531         register char __user *dst asm ("A0.2") = pdst;
 532         register const char *src asm ("A1.2") = psrc;
 533         unsigned long retn = 0;
 534
 535         if (n == 0)
 536                 return 0;
 537
 538         if ((unsigned long) src & 1) {
 539                 __asm_copy_to_user_1(dst, src, retn);
 540                 n--;
 541         }
 542         if ((unsigned long) dst & 1) {
 543                 /* Worst case - byte copy */
 544                 while (n > 0) {
 545                         __asm_copy_to_user_1(dst, src, retn);
 546                         n--;
 547                 }
 548         }
 549         if (((unsigned long) src & 2) && n >= 2) {
 550                 __asm_copy_to_user_2(dst, src, retn);
 551                 n -= 2;
 552         }
 553         if ((unsigned long) dst & 2) {
 554                 /* Second worst case - word copy */
 555                 while (n >= 2) {
 556                         __asm_copy_to_user_2(dst, src, retn);
 557                         n -= 2;
 558                 }
 559         }
 560
 561 #ifdef USE_RAPF
 562         /* 64 bit copy loop */
 563         if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
 564                 if (n >= RAPF_MIN_BUF_SIZE) {
 565                         /* copy user using 64 bit rapf copy */
 566                         __asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
 567                                                         n, "64cu");
 568                 }
 569                 while (n >= 8) {
 570                         __asm_copy_to_user_8x64(dst, src, retn);
 571                         n -= 8;
 572                 }
 573         }
 574         if (n >= RAPF_MIN_BUF_SIZE) {
 575                 /* copy user using 32 bit rapf copy */
 576                 __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
 577         }
 578 #else
 579         /* 64 bit copy loop */
 580         if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
 581                 while (n >= 8) {
 582                         __asm_copy_to_user_8x64(dst, src, retn);
 583                         n -= 8;
 584                 }
 585         }
 586 #endif
 587
 588         while (n >= 16) {
 589                 __asm_copy_to_user_16(dst, src, retn);
 590                 n -= 16;
 591         }
 592
 593         while (n >= 4) {
 594                 __asm_copy_to_user_4(dst, src, retn);
 595                 n -= 4;
 596         }
 597
 598         switch (n) {
 599         case 0:
 600                 break;
 601         case 1:
 602                 __asm_copy_to_user_1(dst, src, retn);
 603                 break;
 604         case 2:
 605                 __asm_copy_to_user_2(dst, src, retn);
 606                 break;
 607         case 3:
 608                 __asm_copy_to_user_3(dst, src, retn);
 609                 break;
 610         }
 611
 612         return retn;
 613 }
 614 EXPORT_SYMBOL(__copy_user);
 615
 616 #define __asm_copy_from_user_1(to, from, ret) \
 617         __asm_copy_user_cont(to, from, ret,     \
 618                 "       GETB D1Ar1,[%1++]\n"    \
 619                 "2:     SETB [%0++],D1Ar1\n",   \
 620                 "3:     ADD  %2,%2,#1\n"        \
 621                 "       SETB [%0++],D1Ar1\n",   \
 622                 "       .long 2b,3b\n")
 623
 624 #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 625         __asm_copy_user_cont(to, from, ret,             \
 626                 "       GETW D1Ar1,[%1++]\n"            \
 627                 "2:     SETW [%0++],D1Ar1\n" COPY,      \
 628                 "3:     ADD  %2,%2,#2\n"                \
 629                 "       SETW [%0++],D1Ar1\n" FIXUP,     \
 630                 "       .long 2b,3b\n" TENTRY)
 631
 632 #define __asm_copy_from_user_2(to, from, ret) \
 633         __asm_copy_from_user_2x_cont(to, from, ret, "", "", "")
 634
 635 #define __asm_copy_from_user_3(to, from, ret)           \
 636         __asm_copy_from_user_2x_cont(to, from, ret,     \
 637                 "       GETB D1Ar1,[%1++]\n"            \
 638                 "4:     SETB [%0++],D1Ar1\n",           \
 639                 "5:     ADD  %2,%2,#1\n"                \
 640                 "       SETB [%0++],D1Ar1\n",           \
 641                 "       .long 4b,5b\n")
 642
 643 #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 644         __asm_copy_user_cont(to, from, ret,             \
 645                 "       GETD D1Ar1,[%1++]\n"            \
 646                 "2:     SETD [%0++],D1Ar1\n" COPY,      \
 647                 "3:     ADD  %2,%2,#4\n"                \
 648                 "       SETD [%0++],D1Ar1\n" FIXUP,     \
 649                 "       .long 2b,3b\n" TENTRY)
 650
 651 #define __asm_copy_from_user_4(to, from, ret) \
 652         __asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
 653
 654 #define __asm_copy_from_user_5(to, from, ret) \
 655         __asm_copy_from_user_4x_cont(to, from, ret,     \
 656                 "       GETB D1Ar1,[%1++]\n"            \
 657                 "4:     SETB [%0++],D1Ar1\n",           \
 658                 "5:     ADD  %2,%2,#1\n"                \
 659                 "       SETB [%0++],D1Ar1\n",           \
 660                 "       .long 4b,5b\n")
 661
 662 #define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 663         __asm_copy_from_user_4x_cont(to, from, ret,     \
 664                 "       GETW D1Ar1,[%1++]\n"            \
 665                 "4:     SETW [%0++],D1Ar1\n" COPY,      \
 666                 "5:     ADD  %2,%2,#2\n"                \
 667                 "       SETW [%0++],D1Ar1\n" FIXUP,     \
 668                 "       .long 4b,5b\n" TENTRY)
 669
 670 #define __asm_copy_from_user_6(to, from, ret) \
 671         __asm_copy_from_user_6x_cont(to, from, ret, "", "", "")
 672
 673 #define __asm_copy_from_user_7(to, from, ret) \
 674         __asm_copy_from_user_6x_cont(to, from, ret,     \
 675                 "       GETB D1Ar1,[%1++]\n"            \
 676                 "6:     SETB [%0++],D1Ar1\n",           \
 677                 "7:     ADD  %2,%2,#1\n"                \
 678                 "       SETB [%0++],D1Ar1\n",           \
 679                 "       .long 6b,7b\n")
 680
 681 #define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 682         __asm_copy_from_user_4x_cont(to, from, ret,     \
 683                 "       GETD D1Ar1,[%1++]\n"            \
 684                 "4:     SETD [%0++],D1Ar1\n" COPY,      \
 685                 "5:     ADD  %2,%2,#4\n"                        \
 686                 "       SETD [%0++],D1Ar1\n" FIXUP,             \
 687                 "       .long 4b,5b\n" TENTRY)
 688
 689 #define __asm_copy_from_user_8(to, from, ret) \
 690         __asm_copy_from_user_8x_cont(to, from, ret, "", "", "")
 691
 692 #define __asm_copy_from_user_9(to, from, ret) \
 693         __asm_copy_from_user_8x_cont(to, from, ret,     \
 694                 "       GETB D1Ar1,[%1++]\n"            \
 695                 "6:     SETB [%0++],D1Ar1\n",           \
 696                 "7:     ADD  %2,%2,#1\n"                \
 697                 "       SETB [%0++],D1Ar1\n",           \
 698                 "       .long 6b,7b\n")
 699
 700 #define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 701         __asm_copy_from_user_8x_cont(to, from, ret,     \
 702                 "       GETW D1Ar1,[%1++]\n"            \
 703                 "6:     SETW [%0++],D1Ar1\n" COPY,      \
 704                 "7:     ADD  %2,%2,#2\n"                \
 705                 "       SETW [%0++],D1Ar1\n" FIXUP,     \
 706                 "       .long 6b,7b\n" TENTRY)
 707
 708 #define __asm_copy_from_user_10(to, from, ret) \
 709         __asm_copy_from_user_10x_cont(to, from, ret, "", "", "")
 710
 711 #define __asm_copy_from_user_11(to, from, ret)          \
 712         __asm_copy_from_user_10x_cont(to, from, ret,    \
 713                 "       GETB D1Ar1,[%1++]\n"            \
 714                 "8:     SETB [%0++],D1Ar1\n",           \
 715                 "9:     ADD  %2,%2,#1\n"                \
 716                 "       SETB [%0++],D1Ar1\n",           \
 717                 "       .long 8b,9b\n")
 718
 719 #define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 720         __asm_copy_from_user_8x_cont(to, from, ret,     \
 721                 "       GETD D1Ar1,[%1++]\n"            \
 722                 "6:     SETD [%0++],D1Ar1\n" COPY,      \
 723                 "7:     ADD  %2,%2,#4\n"                \
 724                 "       SETD [%0++],D1Ar1\n" FIXUP,     \
 725                 "       .long 6b,7b\n" TENTRY)
 726
 727 #define __asm_copy_from_user_12(to, from, ret) \
 728         __asm_copy_from_user_12x_cont(to, from, ret, "", "", "")
 729
 730 #define __asm_copy_from_user_13(to, from, ret) \
 731         __asm_copy_from_user_12x_cont(to, from, ret,    \
 732                 "       GETB D1Ar1,[%1++]\n"            \
 733                 "8:     SETB [%0++],D1Ar1\n",           \
 734                 "9:     ADD  %2,%2,#1\n"                \
 735                 "       SETB [%0++],D1Ar1\n",           \
 736                 "       .long 8b,9b\n")
 737
 738 #define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 739         __asm_copy_from_user_12x_cont(to, from, ret,    \
 740                 "       GETW D1Ar1,[%1++]\n"            \
 741                 "8:     SETW [%0++],D1Ar1\n" COPY,      \
 742                 "9:     ADD  %2,%2,#2\n"                \
 743                 "       SETW [%0++],D1Ar1\n" FIXUP,     \
 744                 "       .long 8b,9b\n" TENTRY)
 745
 746 #define __asm_copy_from_user_14(to, from, ret) \
 747         __asm_copy_from_user_14x_cont(to, from, ret, "", "", "")
 748
 749 #define __asm_copy_from_user_15(to, from, ret) \
 750         __asm_copy_from_user_14x_cont(to, from, ret,    \
 751                 "       GETB D1Ar1,[%1++]\n"            \
 752                 "10:    SETB [%0++],D1Ar1\n",           \
 753                 "11:    ADD  %2,%2,#1\n"                \
 754                 "       SETB [%0++],D1Ar1\n",           \
 755                 "       .long 10b,11b\n")
 756
 757 #define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 758         __asm_copy_from_user_12x_cont(to, from, ret,    \
 759                 "       GETD D1Ar1,[%1++]\n"            \
 760                 "8:     SETD [%0++],D1Ar1\n" COPY,      \
 761                 "9:     ADD  %2,%2,#4\n"                \
 762                 "       SETD [%0++],D1Ar1\n" FIXUP,     \
 763                 "       .long 8b,9b\n" TENTRY)
 764
 765 #define __asm_copy_from_user_16(to, from, ret) \
 766         __asm_copy_from_user_16x_cont(to, from, ret, "", "", "")
 767
 768 #define __asm_copy_from_user_8x64(to, from, ret) \
 769         asm volatile (                          \
 770                 "       GETL D0Ar2,D1Ar1,[%1++]\n"      \
 771                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 772                 "1:\n"                                  \
 773                 "       .section .fixup,\"ax\"\n"       \
 774                 "       MOV D1Ar1,#0\n"                 \
 775                 "       MOV D0Ar2,#0\n"                 \
 776                 "3:     ADD  %2,%2,#8\n"                \
 777                 "       SETL [%0++],D0Ar2,D1Ar1\n"      \
 778                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 779                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 780                 "       .previous\n"                    \
 781                 "       .section __ex_table,\"a\"\n"    \
 782                 "       .long 2b,3b\n"                  \
 783                 "       .previous\n"                    \
 784                 : "=a" (to), "=r" (from), "=r" (ret)    \
 785                 : "0" (to), "1" (from), "2" (ret)       \
 786                 : "D1Ar1", "D0Ar2", "memory")
 787
 788 /*      rewind 'from' pointer when a fault occurs
 789  *
 790  *      Rationale:
 791  *              A fault occurs while reading from user buffer, which is the
 792  *              source. Since the fault is at a single address, we only
 793  *              need to rewind by 8 bytes.
 794  *              Since we don't write to kernel buffer until we read first,
 795  *              the kernel buffer is at the right state and needn't be
 796  *              corrected.
 797  */
 798 #define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id)      \
 799         __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,           \
 800                 "SUB    %1, %1, #8\n")
 801
 802 /*      rewind 'from' pointer when a fault occurs
 803  *
 804  *      Rationale:
 805  *              A fault occurs while reading from user buffer, which is the
 806  *              source. Since the fault is at a single address, we only
 807  *              need to rewind by 4 bytes.
 808  *              Since we don't write to kernel buffer until we read first,
 809  *              the kernel buffer is at the right state and needn't be
 810  *              corrected.
 811  */
 812 #define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id)      \
 813         __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,           \
 814                 "SUB    %1, %1, #4\n")
 815
 816
 817 /* Copy from user to kernel, zeroing the bytes that were inaccessible in
 818    userland.  The return-value is the number of bytes that were
 819    inaccessible.  */
 820 unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
 821                                   unsigned long n)
 822 {
 823         register char *dst asm ("A0.2") = pdst;
 824         register const char __user *src asm ("A1.2") = psrc;
 825         unsigned long retn = 0;
 826
 827         if (n == 0)
 828                 return 0;
 829
 830         if ((unsigned long) src & 1) {
 831                 __asm_copy_from_user_1(dst, src, retn);
 832                 n--;
 833         }
 834         if ((unsigned long) dst & 1) {
 835                 /* Worst case - byte copy */
 836                 while (n > 0) {
 837                         __asm_copy_from_user_1(dst, src, retn);
 838                         n--;
 839                         if (retn)
 840                                 goto copy_exception_bytes;
 841                 }
 842         }
 843         if (((unsigned long) src & 2) && n >= 2) {
 844                 __asm_copy_from_user_2(dst, src, retn);
 845                 n -= 2;
 846         }
 847         if ((unsigned long) dst & 2) {
 848                 /* Second worst case - word copy */
 849                 while (n >= 2) {
 850                         __asm_copy_from_user_2(dst, src, retn);
 851                         n -= 2;
 852                         if (retn)
 853                                 goto copy_exception_bytes;
 854                 }
 855         }
 856
 857         /* We only need one check after the unalignment-adjustments,
 858            because if both adjustments were done, either both or
 859            neither reference had an exception.  */
 860         if (retn != 0)
 861                 goto copy_exception_bytes;
 862
 863 #ifdef USE_RAPF
 864         /* 64 bit copy loop */
 865         if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
 866                 if (n >= RAPF_MIN_BUF_SIZE) {
 867                         /* Copy using fast 64bit rapf */
 868                         __asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
 869                                                         n, "64cuz");
 870                 }
 871                 while (n >= 8) {
 872                         __asm_copy_from_user_8x64(dst, src, retn);
 873                         n -= 8;
 874                         if (retn)
 875                                 goto copy_exception_bytes;
 876                 }
 877         }
 878
 879         if (n >= RAPF_MIN_BUF_SIZE) {
 880                 /* Copy using fast 32bit rapf */
 881                 __asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
 882                                                 n, "32cuz");
 883         }
 884 #else
 885         /* 64 bit copy loop */
 886         if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
 887                 while (n >= 8) {
 888                         __asm_copy_from_user_8x64(dst, src, retn);
 889                         n -= 8;
 890                         if (retn)
 891                                 goto copy_exception_bytes;
 892                 }
 893         }
 894 #endif
 895
 896         while (n >= 4) {
 897                 __asm_copy_from_user_4(dst, src, retn);
 898                 n -= 4;
 899
 900                 if (retn)
 901                         goto copy_exception_bytes;
 902         }
 903
 904         /* If we get here, there were no memory read faults.  */
 905         switch (n) {
 906                 /* These copies are at least "naturally aligned" (so we don't
 907                    have to check each byte), due to the src alignment code.
 908                    The *_3 case *will* get the correct count for retn.  */
 909         case 0:
 910                 /* This case deliberately left in (if you have doubts check the
 911                    generated assembly code).  */
 912                 break;
 913         case 1:
 914                 __asm_copy_from_user_1(dst, src, retn);
 915                 break;
 916         case 2:
 917                 __asm_copy_from_user_2(dst, src, retn);
 918                 break;
 919         case 3:
 920                 __asm_copy_from_user_3(dst, src, retn);
 921                 break;
 922         }
 923
 924         /* If we get here, retn correctly reflects the number of failing
 925            bytes.  */
 926         return retn;
 927
 928  copy_exception_bytes:
 929         /* We already have "retn" bytes cleared, and need to clear the
 930            remaining "n" bytes.  A non-optimized simple byte-for-byte in-line
 931            memset is preferred here, since this isn't speed-critical code and
 932            we'd rather have this a leaf-function than calling memset.  */
 933         {
 934                 char *endp;
 935                 for (endp = dst + n; dst < endp; dst++)
 936                         *dst = 0;
 937         }
 938
 939         return retn + n;
 940 }
 941 EXPORT_SYMBOL(__copy_user_zeroing);
 942
 943 #define __asm_clear_8x64(to, ret) \
 944         asm volatile (                                  \
 945                 "       MOV  D0Ar2,#0\n"                \
 946                 "       MOV  D1Ar1,#0\n"                \
 947                 "       SETL [%0],D0Ar2,D1Ar1\n"        \
 948                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 949                 "1:\n"                                  \
 950                 "       .section .fixup,\"ax\"\n"       \
 951                 "3:     ADD  %1,%1,#8\n"                \
 952                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 953                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 954                 "       .previous\n"                    \
 955                 "       .section __ex_table,\"a\"\n"    \
 956                 "       .long 2b,3b\n"                  \
 957                 "       .previous\n"                    \
 958                 : "=r" (to), "=r" (ret) \
 959                 : "0" (to), "1" (ret)   \
 960                 : "D1Ar1", "D0Ar2", "memory")
 961
 962 /* Zero userspace.  */
 963
 964 #define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
 965         asm volatile (                                  \
 966                 "       MOV D1Ar1,#0\n"                 \
 967                         CLEAR                           \
 968                 "1:\n"                                  \
 969                 "       .section .fixup,\"ax\"\n"       \
 970                         FIXUP                           \
 971                 "       MOVT    D1Ar1,#HI(1b)\n"        \
 972                 "       JUMP    D1Ar1,#LO(1b)\n"        \
 973                 "       .previous\n"                    \
 974                 "       .section __ex_table,\"a\"\n"    \
 975                         TENTRY                          \
 976                 "       .previous"                      \
 977                 : "=r" (to), "=r" (ret)                 \
 978                 : "0" (to), "1" (ret)                   \
 979                 : "D1Ar1", "memory")
 980
 981 #define __asm_clear_1(to, ret) \
 982         __asm_clear(to, ret,                    \
 983                 "       SETB [%0],D1Ar1\n"      \
 984                 "2:     SETB [%0++],D1Ar1\n",   \
 985                 "3:     ADD  %1,%1,#1\n",       \
 986                 "       .long 2b,3b\n")
 987
 988 #define __asm_clear_2(to, ret) \
 989         __asm_clear(to, ret,                    \
 990                 "       SETW [%0],D1Ar1\n"      \
 991                 "2:     SETW [%0++],D1Ar1\n",   \
 992                 "3:     ADD  %1,%1,#2\n",       \
 993                 "       .long 2b,3b\n")
 994
 995 #define __asm_clear_3(to, ret) \
 996         __asm_clear(to, ret,                    \
 997                  "2:    SETW [%0++],D1Ar1\n"    \
 998                  "      SETB [%0],D1Ar1\n"      \
 999                  "3:    SETB [%0++],D1Ar1\n",   \
1000                  "4:    ADD  %1,%1,#2\n"        \
1001                  "5:    ADD  %1,%1,#1\n",       \
1002                  "      .long 2b,4b\n"          \
1003                  "      .long 3b,5b\n")
1004
1005 #define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1006         __asm_clear(to, ret,                            \
1007                 "       SETD [%0],D1Ar1\n"              \
1008                 "2:     SETD [%0++],D1Ar1\n" CLEAR,     \
1009                 "3:     ADD  %1,%1,#4\n" FIXUP,         \
1010                 "       .long 2b,3b\n" TENTRY)
1011
1012 #define __asm_clear_4(to, ret) \
1013         __asm_clear_4x_cont(to, ret, "", "", "")
1014
1015 #define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1016         __asm_clear_4x_cont(to, ret,                    \
1017                 "       SETD [%0],D1Ar1\n"              \
1018                 "4:     SETD [%0++],D1Ar1\n" CLEAR,     \
1019                 "5:     ADD  %1,%1,#4\n" FIXUP,         \
1020                 "       .long 4b,5b\n" TENTRY)
1021
1022 #define __asm_clear_8(to, ret) \
1023         __asm_clear_8x_cont(to, ret, "", "", "")
1024
1025 #define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1026         __asm_clear_8x_cont(to, ret,                    \
1027                 "       SETD [%0],D1Ar1\n"              \
1028                 "6:     SETD [%0++],D1Ar1\n" CLEAR,     \
1029                 "7:     ADD  %1,%1,#4\n" FIXUP,         \
1030                 "       .long 6b,7b\n" TENTRY)
1031
1032 #define __asm_clear_12(to, ret) \
1033         __asm_clear_12x_cont(to, ret, "", "", "")
1034
1035 #define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1036         __asm_clear_12x_cont(to, ret,                   \
1037                 "       SETD [%0],D1Ar1\n"              \
1038                 "8:     SETD [%0++],D1Ar1\n" CLEAR,     \
1039                 "9:     ADD  %1,%1,#4\n" FIXUP,         \
1040                 "       .long 8b,9b\n" TENTRY)
1041
1042 #define __asm_clear_16(to, ret) \
1043         __asm_clear_16x_cont(to, ret, "", "", "")
1044
1045 unsigned long __do_clear_user(void __user *pto, unsigned long pn)
1046 {
1047         register char __user *dst asm ("D0Re0") = pto;
1048         register unsigned long n asm ("D1Re0") = pn;
1049         register unsigned long retn asm ("D0Ar6") = 0;
1050
1051         if ((unsigned long) dst & 1) {
1052                 __asm_clear_1(dst, retn);
1053                 n--;
1054         }
1055
1056         if ((unsigned long) dst & 2) {
1057                 __asm_clear_2(dst, retn);
1058                 n -= 2;
1059         }
1060
1061         /* 64 bit copy loop */
1062         if (!((__force unsigned long) dst & 7)) {
1063                 while (n >= 8) {
1064                         __asm_clear_8x64(dst, retn);
1065                         n -= 8;
1066                 }
1067         }
1068
1069         while (n >= 16) {
1070                 __asm_clear_16(dst, retn);
1071                 n -= 16;
1072         }
1073
1074         while (n >= 4) {
1075                 __asm_clear_4(dst, retn);
1076                 n -= 4;
1077         }
1078
1079         switch (n) {
1080         case 0:
1081                 break;
1082         case 1:
1083                 __asm_clear_1(dst, retn);
1084                 break;
1085         case 2:
1086                 __asm_clear_2(dst, retn);
1087                 break;
1088         case 3:
1089                 __asm_clear_3(dst, retn);
1090                 break;
1091         }
1092
1093         return retn;
1094 }
1095 EXPORT_SYMBOL(__do_clear_user);
1096
1097 unsigned char __get_user_asm_b(const void __user *addr, long *err)
1098 {
1099         register unsigned char x asm ("D0Re0") = 0;
1100         asm volatile (
1101                 "       GETB %0,[%2]\n"
1102                 "1:\n"
1103                 "       GETB %0,[%2]\n"
1104                 "2:\n"
1105                 "       .section .fixup,\"ax\"\n"
1106                 "3:     MOV     D0FrT,%3\n"
1107                 "       SETD    [%1],D0FrT\n"
1108                 "       MOVT    D0FrT,#HI(2b)\n"
1109                 "       JUMP    D0FrT,#LO(2b)\n"
1110                 "       .previous\n"
1111                 "       .section __ex_table,\"a\"\n"
1112                 "       .long 1b,3b\n"
1113                 "       .previous\n"
1114                 : "=r" (x)
1115                 : "r" (err), "r" (addr), "P" (-EFAULT)
1116                 : "D0FrT");
1117         return x;
1118 }
1119 EXPORT_SYMBOL(__get_user_asm_b);
1120
1121 unsigned short __get_user_asm_w(const void __user *addr, long *err)
1122 {
1123         register unsigned short x asm ("D0Re0") = 0;
1124         asm volatile (
1125                 "       GETW %0,[%2]\n"
1126                 "1:\n"
1127                 "       GETW %0,[%2]\n"
1128                 "2:\n"
1129                 "       .section .fixup,\"ax\"\n"
1130                 "3:     MOV     D0FrT,%3\n"
1131                 "       SETD    [%1],D0FrT\n"
1132                 "       MOVT    D0FrT,#HI(2b)\n"
1133                 "       JUMP    D0FrT,#LO(2b)\n"
1134                 "       .previous\n"
1135                 "       .section __ex_table,\"a\"\n"
1136                 "       .long 1b,3b\n"
1137                 "       .previous\n"
1138                 : "=r" (x)
1139                 : "r" (err), "r" (addr), "P" (-EFAULT)
1140                 : "D0FrT");
1141         return x;
1142 }
1143 EXPORT_SYMBOL(__get_user_asm_w);
1144
1145 unsigned int __get_user_asm_d(const void __user *addr, long *err)
1146 {
1147         register unsigned int x asm ("D0Re0") = 0;
1148         asm volatile (
1149                 "       GETD %0,[%2]\n"
1150                 "1:\n"
1151                 "       GETD %0,[%2]\n"
1152                 "2:\n"
1153                 "       .section .fixup,\"ax\"\n"
1154                 "3:     MOV     D0FrT,%3\n"
1155                 "       SETD    [%1],D0FrT\n"
1156                 "       MOVT    D0FrT,#HI(2b)\n"
1157                 "       JUMP    D0FrT,#LO(2b)\n"
1158                 "       .previous\n"
1159                 "       .section __ex_table,\"a\"\n"
1160                 "       .long 1b,3b\n"
1161                 "       .previous\n"
1162                 : "=r" (x)
1163                 : "r" (err), "r" (addr), "P" (-EFAULT)
1164                 : "D0FrT");
1165         return x;
1166 }
1167 EXPORT_SYMBOL(__get_user_asm_d);
1168
1169 long __put_user_asm_b(unsigned int x, void __user *addr)
1170 {
1171         register unsigned int err asm ("D0Re0") = 0;
1172         asm volatile (
1173                 "       MOV  %0,#0\n"
1174                 "       SETB [%2],%1\n"
1175                 "1:\n"
1176                 "       SETB [%2],%1\n"
1177                 "2:\n"
1178                 ".section .fixup,\"ax\"\n"
1179                 "3:     MOV     %0,%3\n"
1180                 "       MOVT    D0FrT,#HI(2b)\n"
1181                 "       JUMP    D0FrT,#LO(2b)\n"
1182                 ".previous\n"
1183                 ".section __ex_table,\"a\"\n"
1184                 "       .long 1b,3b\n"
1185                 ".previous"
1186                 : "=r"(err)
1187                 : "d" (x), "a" (addr), "P"(-EFAULT)
1188                 : "D0FrT");
1189         return err;
1190 }
1191 EXPORT_SYMBOL(__put_user_asm_b);
1192
1193 long __put_user_asm_w(unsigned int x, void __user *addr)
1194 {
1195         register unsigned int err asm ("D0Re0") = 0;
1196         asm volatile (
1197                 "       MOV  %0,#0\n"
1198                 "       SETW [%2],%1\n"
1199                 "1:\n"
1200                 "       SETW [%2],%1\n"
1201                 "2:\n"
1202                 ".section .fixup,\"ax\"\n"
1203                 "3:     MOV     %0,%3\n"
1204                 "       MOVT    D0FrT,#HI(2b)\n"
1205                 "       JUMP    D0FrT,#LO(2b)\n"
1206                 ".previous\n"
1207                 ".section __ex_table,\"a\"\n"
1208                 "       .long 1b,3b\n"
1209                 ".previous"
1210                 : "=r"(err)
1211                 : "d" (x), "a" (addr), "P"(-EFAULT)
1212                 : "D0FrT");
1213         return err;
1214 }
1215 EXPORT_SYMBOL(__put_user_asm_w);
1216
1217 long __put_user_asm_d(unsigned int x, void __user *addr)
1218 {
1219         register unsigned int err asm ("D0Re0") = 0;
1220         asm volatile (
1221                 "       MOV  %0,#0\n"
1222                 "       SETD [%2],%1\n"
1223                 "1:\n"
1224                 "       SETD [%2],%1\n"
1225                 "2:\n"
1226                 ".section .fixup,\"ax\"\n"
1227                 "3:     MOV     %0,%3\n"
1228                 "       MOVT    D0FrT,#HI(2b)\n"
1229                 "       JUMP    D0FrT,#LO(2b)\n"
1230                 ".previous\n"
1231                 ".section __ex_table,\"a\"\n"
1232                 "       .long 1b,3b\n"
1233                 ".previous"
1234                 : "=r"(err)
1235                 : "d" (x), "a" (addr), "P"(-EFAULT)
1236                 : "D0FrT");
1237         return err;
1238 }
1239 EXPORT_SYMBOL(__put_user_asm_d);
1240
1241 long __put_user_asm_l(unsigned long long x, void __user *addr)
1242 {
1243         register unsigned int err asm ("D0Re0") = 0;
1244         asm volatile (
1245                 "       MOV  %0,#0\n"
1246                 "       SETL [%2],%1,%t1\n"
1247                 "1:\n"
1248                 "       SETL [%2],%1,%t1\n"
1249                 "2:\n"
1250                 ".section .fixup,\"ax\"\n"
1251                 "3:     MOV     %0,%3\n"
1252                 "       MOVT    D0FrT,#HI(2b)\n"
1253                 "       JUMP    D0FrT,#LO(2b)\n"
1254                 ".previous\n"
1255                 ".section __ex_table,\"a\"\n"
1256                 "       .long 1b,3b\n"
1257                 ".previous"
1258                 : "=r"(err)
1259                 : "d" (x), "a" (addr), "P"(-EFAULT)
1260                 : "D0FrT");
1261         return err;
1262 }
1263 EXPORT_SYMBOL(__put_user_asm_l);
1264
1265 long strnlen_user(const char __user *src, long count)
1266 {
1267         long res;
1268
1269         if (!access_ok(VERIFY_READ, src, 0))
1270                 return 0;
1271
1272         asm volatile (" MOV     D0Ar4, %1\n"
1273                       " MOV     D0Ar6, %2\n"
1274                       "0:\n"
1275                       " SUBS    D0FrT, D0Ar6, #0\n"
1276                       " SUB     D0Ar6, D0Ar6, #1\n"
1277                       " BLE     2f\n"
1278                       " GETB    D0FrT, [D0Ar4+#1++]\n"
1279                       "1:\n"
1280                       " TST     D0FrT, #255\n"
1281                       " BNE     0b\n"
1282                       "2:\n"
1283                       " SUB     %0, %2, D0Ar6\n"
1284                       "3:\n"
1285                       " .section .fixup,\"ax\"\n"
1286                       "4:\n"
1287                       " MOV     %0, #0\n"
1288                       " MOVT    D0FrT,#HI(3b)\n"
1289                       " JUMP    D0FrT,#LO(3b)\n"
1290                       " .previous\n"
1291                       " .section __ex_table,\"a\"\n"
1292                       " .long 1b,4b\n"
1293                       " .previous\n"
1294                       : "=r" (res)
1295                       : "r" (src), "r" (count)
1296                       : "D0FrT", "D0Ar4", "D0Ar6", "cc");
1297
1298         return res;
1299 }
1300 EXPORT_SYMBOL(strnlen_user);
1301
1302 long __strncpy_from_user(char *dst, const char __user *src, long count)
1303 {
1304         long res;
1305
1306         if (count == 0)
1307                 return 0;
1308
1309         /*
1310          * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
1311          *  So do we.
1312          *
1313          *  This code is deduced from:
1314          *
1315          *      char tmp2;
1316          *      long tmp1, tmp3;
1317          *      tmp1 = count;
1318          *      while ((*dst++ = (tmp2 = *src++)) != 0
1319          *             && --tmp1)
1320          *        ;
1321          *
1322          *      res = count - tmp1;
1323          *
1324          *  with tweaks.
1325          */
1326
1327         asm volatile (" MOV  %0,%3\n"
1328                       "1:\n"
1329                       " GETB D0FrT,[%2++]\n"
1330                       "2:\n"
1331                       " CMP  D0FrT,#0\n"
1332                       " SETB [%1++],D0FrT\n"
1333                       " BEQ  3f\n"
1334                       " SUBS %0,%0,#1\n"
1335                       " BNZ  1b\n"
1336                       "3:\n"
1337                       " SUB  %0,%3,%0\n"
1338                       "4:\n"
1339                       " .section .fixup,\"ax\"\n"
1340                       "5:\n"
1341                       " MOV  %0,%7\n"
1342                       " MOVT    D0FrT,#HI(4b)\n"
1343                       " JUMP    D0FrT,#LO(4b)\n"
1344                       " .previous\n"
1345                       " .section __ex_table,\"a\"\n"
1346                       " .long 2b,5b\n"
1347                       " .previous"
1348                       : "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
1349                       : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
1350                       : "D0FrT", "memory", "cc");
1351
1352         return res;
1353 }
1354 EXPORT_SYMBOL(__strncpy_from_user);