kernel/arch/mips/kernel/unaligned.c

   1 /*
   2  * Handle unaligned accesses by emulation.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9  * Copyright (C) 1999 Silicon Graphics, Inc.
  10  * Copyright (C) 2014 Imagination Technologies Ltd.
  11  *
  12  * This file contains exception handler for address error exception with the
  13  * special capability to execute faulting instructions in software.  The
  14  * handler does not try to handle the case when the program counter points
  15  * to an address not aligned to a word boundary.
  16  *
  17  * Putting data to unaligned addresses is a bad practice even on Intel where
  18  * only the performance is affected.  Much worse is that such code is non-
  19  * portable.  Due to several programs that die on MIPS due to alignment
  20  * problems I decided to implement this handler anyway though I originally
  21  * didn't intend to do this at all for user code.
  22  *
  23  * For now I enable fixing of address errors by default to make life easier.
  24  * I however intend to disable this somewhen in the future when the alignment
  25  * problems with user programs have been fixed.  For programmers this is the
  26  * right way to go.
  27  *
  28  * Fixing address errors is a per process option.  The option is inherited
  29  * across fork(2) and execve(2) calls.  If you really want to use the
  30  * option in your user programs - I discourage the use of the software
  31  * emulation strongly - use the following code in your userland stuff:
  32  *
  33  * #include <sys/sysmips.h>
  34  *
  35  * ...
  36  * sysmips(MIPS_FIXADE, x);
  37  * ...
  38  *
  39  * The argument x is 0 for disabling software emulation, enabled otherwise.
  40  *
  41  * Below a little program to play around with this feature.
  42  *
  43  * #include <stdio.h>
  44  * #include <sys/sysmips.h>
  45  *
  46  * struct foo {
  47  *         unsigned char bar[8];
  48  * };
  49  *
  50  * main(int argc, char *argv[])
  51  * {
  52  *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
  53  *         unsigned int *p = (unsigned int *) (x.bar + 3);
  54  *         int i;
  55  *
  56  *         if (argc > 1)
  57  *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
  58  *
  59  *         printf("*p = %08lx\n", *p);
  60  *
  61  *         *p = 0xdeadface;
  62  *
  63  *         for(i = 0; i <= 7; i++)
  64  *         printf("%02x ", x.bar[i]);
  65  *         printf("\n");
  66  * }
  67  *
  68  * Coprocessor loads are not supported; I think this case is unimportant
  69  * in the practice.
  70  *
  71  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
  72  *       exception for the R6000.
  73  *       A store crossing a page boundary might be executed only partially.
  74  *       Undo the partial store in this case.
  75  */
  76 #include <linux/context_tracking.h>
  77 #include <linux/mm.h>
  78 #include <linux/signal.h>
  79 #include <linux/smp.h>
  80 #include <linux/sched.h>
  81 #include <linux/debugfs.h>
  82 #include <linux/perf_event.h>
  83
  84 #include <asm/asm.h>
  85 #include <asm/branch.h>
  86 #include <asm/byteorder.h>
  87 #include <asm/cop2.h>
  88 #include <asm/debug.h>
  89 #include <asm/fpu.h>
  90 #include <asm/fpu_emulator.h>
  91 #include <asm/inst.h>
  92 #include <asm/uaccess.h>
  93
  94 #define STR(x)  __STR(x)
  95 #define __STR(x)  #x
  96
  97 enum {
  98         UNALIGNED_ACTION_QUIET,
  99         UNALIGNED_ACTION_SIGNAL,
 100         UNALIGNED_ACTION_SHOW,
 101 };
 102 #ifdef CONFIG_DEBUG_FS
 103 static u32 unaligned_instructions;
 104 static u32 unaligned_action;
 105 #else
 106 #define unaligned_action UNALIGNED_ACTION_QUIET
 107 #endif
 108 extern void show_registers(struct pt_regs *regs);
 109
 110 #ifdef __BIG_ENDIAN
 111 #define     _LoadHW(addr, value, res, type)  \
 112 do {                                                        \
 113                 __asm__ __volatile__ (".set\tnoat\n"        \
 114                         "1:\t"type##_lb("%0", "0(%2)")"\n"  \
 115                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 116                         "sll\t%0, 0x8\n\t"                  \
 117                         "or\t%0, $1\n\t"                    \
 118                         "li\t%1, 0\n"                       \
 119                         "3:\t.set\tat\n\t"                  \
 120                         ".insn\n\t"                         \
 121                         ".section\t.fixup,\"ax\"\n\t"       \
 122                         "4:\tli\t%1, %3\n\t"                \
 123                         "j\t3b\n\t"                         \
 124                         ".previous\n\t"                     \
 125                         ".section\t__ex_table,\"a\"\n\t"    \
 126                         STR(PTR)"\t1b, 4b\n\t"              \
 127                         STR(PTR)"\t2b, 4b\n\t"              \
 128                         ".previous"                         \
 129                         : "=&r" (value), "=r" (res)         \
 130                         : "r" (addr), "i" (-EFAULT));       \
 131 } while(0)
 132
 133 #ifndef CONFIG_CPU_MIPSR6
 134 #define     _LoadW(addr, value, res, type)   \
 135 do {                                                        \
 136                 __asm__ __volatile__ (                      \
 137                         "1:\t"type##_lwl("%0", "(%2)")"\n"   \
 138                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 139                         "li\t%1, 0\n"                       \
 140                         "3:\n\t"                            \
 141                         ".insn\n\t"                         \
 142                         ".section\t.fixup,\"ax\"\n\t"       \
 143                         "4:\tli\t%1, %3\n\t"                \
 144                         "j\t3b\n\t"                         \
 145                         ".previous\n\t"                     \
 146                         ".section\t__ex_table,\"a\"\n\t"    \
 147                         STR(PTR)"\t1b, 4b\n\t"              \
 148                         STR(PTR)"\t2b, 4b\n\t"              \
 149                         ".previous"                         \
 150                         : "=&r" (value), "=r" (res)         \
 151                         : "r" (addr), "i" (-EFAULT));       \
 152 } while(0)
 153
 154 #else
 155 /* MIPSR6 has no lwl instruction */
 156 #define     _LoadW(addr, value, res, type) \
 157 do {                                                        \
 158                 __asm__ __volatile__ (                      \
 159                         ".set\tpush\n"                      \
 160                         ".set\tnoat\n\t"                    \
 161                         "1:"type##_lb("%0", "0(%2)")"\n\t"  \
 162                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 163                         "sll\t%0, 0x8\n\t"                  \
 164                         "or\t%0, $1\n\t"                    \
 165                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 166                         "sll\t%0, 0x8\n\t"                  \
 167                         "or\t%0, $1\n\t"                    \
 168                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 169                         "sll\t%0, 0x8\n\t"                  \
 170                         "or\t%0, $1\n\t"                    \
 171                         "li\t%1, 0\n"                       \
 172                         ".set\tpop\n"                       \
 173                         "10:\n\t"                           \
 174                         ".insn\n\t"                         \
 175                         ".section\t.fixup,\"ax\"\n\t"       \
 176                         "11:\tli\t%1, %3\n\t"               \
 177                         "j\t10b\n\t"                        \
 178                         ".previous\n\t"                     \
 179                         ".section\t__ex_table,\"a\"\n\t"    \
 180                         STR(PTR)"\t1b, 11b\n\t"             \
 181                         STR(PTR)"\t2b, 11b\n\t"             \
 182                         STR(PTR)"\t3b, 11b\n\t"             \
 183                         STR(PTR)"\t4b, 11b\n\t"             \
 184                         ".previous"                         \
 185                         : "=&r" (value), "=r" (res)         \
 186                         : "r" (addr), "i" (-EFAULT));       \
 187 } while(0)
 188
 189 #endif /* CONFIG_CPU_MIPSR6 */
 190
 191 #define     _LoadHWU(addr, value, res, type) \
 192 do {                                                        \
 193                 __asm__ __volatile__ (                      \
 194                         ".set\tnoat\n"                      \
 195                         "1:\t"type##_lbu("%0", "0(%2)")"\n" \
 196                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 197                         "sll\t%0, 0x8\n\t"                  \
 198                         "or\t%0, $1\n\t"                    \
 199                         "li\t%1, 0\n"                       \
 200                         "3:\n\t"                            \
 201                         ".insn\n\t"                         \
 202                         ".set\tat\n\t"                      \
 203                         ".section\t.fixup,\"ax\"\n\t"       \
 204                         "4:\tli\t%1, %3\n\t"                \
 205                         "j\t3b\n\t"                         \
 206                         ".previous\n\t"                     \
 207                         ".section\t__ex_table,\"a\"\n\t"    \
 208                         STR(PTR)"\t1b, 4b\n\t"              \
 209                         STR(PTR)"\t2b, 4b\n\t"              \
 210                         ".previous"                         \
 211                         : "=&r" (value), "=r" (res)         \
 212                         : "r" (addr), "i" (-EFAULT));       \
 213 } while(0)
 214
 215 #ifndef CONFIG_CPU_MIPSR6
 216 #define     _LoadWU(addr, value, res, type)  \
 217 do {                                                        \
 218                 __asm__ __volatile__ (                      \
 219                         "1:\t"type##_lwl("%0", "(%2)")"\n"  \
 220                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 221                         "dsll\t%0, %0, 32\n\t"              \
 222                         "dsrl\t%0, %0, 32\n\t"              \
 223                         "li\t%1, 0\n"                       \
 224                         "3:\n\t"                            \
 225                         ".insn\n\t"                         \
 226                         "\t.section\t.fixup,\"ax\"\n\t"     \
 227                         "4:\tli\t%1, %3\n\t"                \
 228                         "j\t3b\n\t"                         \
 229                         ".previous\n\t"                     \
 230                         ".section\t__ex_table,\"a\"\n\t"    \
 231                         STR(PTR)"\t1b, 4b\n\t"              \
 232                         STR(PTR)"\t2b, 4b\n\t"              \
 233                         ".previous"                         \
 234                         : "=&r" (value), "=r" (res)         \
 235                         : "r" (addr), "i" (-EFAULT));       \
 236 } while(0)
 237
 238 #define     _LoadDW(addr, value, res)  \
 239 do {                                                        \
 240                 __asm__ __volatile__ (                      \
 241                         "1:\tldl\t%0, (%2)\n"               \
 242                         "2:\tldr\t%0, 7(%2)\n\t"            \
 243                         "li\t%1, 0\n"                       \
 244                         "3:\n\t"                            \
 245                         ".insn\n\t"                         \
 246                         "\t.section\t.fixup,\"ax\"\n\t"     \
 247                         "4:\tli\t%1, %3\n\t"                \
 248                         "j\t3b\n\t"                         \
 249                         ".previous\n\t"                     \
 250                         ".section\t__ex_table,\"a\"\n\t"    \
 251                         STR(PTR)"\t1b, 4b\n\t"              \
 252                         STR(PTR)"\t2b, 4b\n\t"              \
 253                         ".previous"                         \
 254                         : "=&r" (value), "=r" (res)         \
 255                         : "r" (addr), "i" (-EFAULT));       \
 256 } while(0)
 257
 258 #else
 259 /* MIPSR6 has not lwl and ldl instructions */
 260 #define     _LoadWU(addr, value, res, type) \
 261 do {                                                        \
 262                 __asm__ __volatile__ (                      \
 263                         ".set\tpush\n\t"                    \
 264                         ".set\tnoat\n\t"                    \
 265                         "1:"type##_lbu("%0", "0(%2)")"\n\t" \
 266                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 267                         "sll\t%0, 0x8\n\t"                  \
 268                         "or\t%0, $1\n\t"                    \
 269                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 270                         "sll\t%0, 0x8\n\t"                  \
 271                         "or\t%0, $1\n\t"                    \
 272                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 273                         "sll\t%0, 0x8\n\t"                  \
 274                         "or\t%0, $1\n\t"                    \
 275                         "li\t%1, 0\n"                       \
 276                         ".set\tpop\n"                       \
 277                         "10:\n\t"                           \
 278                         ".insn\n\t"                         \
 279                         ".section\t.fixup,\"ax\"\n\t"       \
 280                         "11:\tli\t%1, %3\n\t"               \
 281                         "j\t10b\n\t"                        \
 282                         ".previous\n\t"                     \
 283                         ".section\t__ex_table,\"a\"\n\t"    \
 284                         STR(PTR)"\t1b, 11b\n\t"             \
 285                         STR(PTR)"\t2b, 11b\n\t"             \
 286                         STR(PTR)"\t3b, 11b\n\t"             \
 287                         STR(PTR)"\t4b, 11b\n\t"             \
 288                         ".previous"                         \
 289                         : "=&r" (value), "=r" (res)         \
 290                         : "r" (addr), "i" (-EFAULT));       \
 291 } while(0)
 292
 293 #define     _LoadDW(addr, value, res)  \
 294 do {                                                        \
 295                 __asm__ __volatile__ (                      \
 296                         ".set\tpush\n\t"                    \
 297                         ".set\tnoat\n\t"                    \
 298                         "1:lb\t%0, 0(%2)\n\t"               \
 299                         "2:lbu\t $1, 1(%2)\n\t"             \
 300                         "dsll\t%0, 0x8\n\t"                 \
 301                         "or\t%0, $1\n\t"                    \
 302                         "3:lbu\t$1, 2(%2)\n\t"              \
 303                         "dsll\t%0, 0x8\n\t"                 \
 304                         "or\t%0, $1\n\t"                    \
 305                         "4:lbu\t$1, 3(%2)\n\t"              \
 306                         "dsll\t%0, 0x8\n\t"                 \
 307                         "or\t%0, $1\n\t"                    \
 308                         "5:lbu\t$1, 4(%2)\n\t"              \
 309                         "dsll\t%0, 0x8\n\t"                 \
 310                         "or\t%0, $1\n\t"                    \
 311                         "6:lbu\t$1, 5(%2)\n\t"              \
 312                         "dsll\t%0, 0x8\n\t"                 \
 313                         "or\t%0, $1\n\t"                    \
 314                         "7:lbu\t$1, 6(%2)\n\t"              \
 315                         "dsll\t%0, 0x8\n\t"                 \
 316                         "or\t%0, $1\n\t"                    \
 317                         "8:lbu\t$1, 7(%2)\n\t"              \
 318                         "dsll\t%0, 0x8\n\t"                 \
 319                         "or\t%0, $1\n\t"                    \
 320                         "li\t%1, 0\n"                       \
 321                         ".set\tpop\n\t"                     \
 322                         "10:\n\t"                           \
 323                         ".insn\n\t"                         \
 324                         ".section\t.fixup,\"ax\"\n\t"       \
 325                         "11:\tli\t%1, %3\n\t"               \
 326                         "j\t10b\n\t"                        \
 327                         ".previous\n\t"                     \
 328                         ".section\t__ex_table,\"a\"\n\t"    \
 329                         STR(PTR)"\t1b, 11b\n\t"             \
 330                         STR(PTR)"\t2b, 11b\n\t"             \
 331                         STR(PTR)"\t3b, 11b\n\t"             \
 332                         STR(PTR)"\t4b, 11b\n\t"             \
 333                         STR(PTR)"\t5b, 11b\n\t"             \
 334                         STR(PTR)"\t6b, 11b\n\t"             \
 335                         STR(PTR)"\t7b, 11b\n\t"             \
 336                         STR(PTR)"\t8b, 11b\n\t"             \
 337                         ".previous"                         \
 338                         : "=&r" (value), "=r" (res)         \
 339                         : "r" (addr), "i" (-EFAULT));       \
 340 } while(0)
 341
 342 #endif /* CONFIG_CPU_MIPSR6 */
 343
 344
 345 #define     _StoreHW(addr, value, res, type) \
 346 do {                                                        \
 347                 __asm__ __volatile__ (                      \
 348                         ".set\tnoat\n"                      \
 349                         "1:\t"type##_sb("%1", "1(%2)")"\n"  \
 350                         "srl\t$1, %1, 0x8\n"                \
 351                         "2:\t"type##_sb("$1", "0(%2)")"\n"  \
 352                         ".set\tat\n\t"                      \
 353                         "li\t%0, 0\n"                       \
 354                         "3:\n\t"                            \
 355                         ".insn\n\t"                         \
 356                         ".section\t.fixup,\"ax\"\n\t"       \
 357                         "4:\tli\t%0, %3\n\t"                \
 358                         "j\t3b\n\t"                         \
 359                         ".previous\n\t"                     \
 360                         ".section\t__ex_table,\"a\"\n\t"    \
 361                         STR(PTR)"\t1b, 4b\n\t"              \
 362                         STR(PTR)"\t2b, 4b\n\t"              \
 363                         ".previous"                         \
 364                         : "=r" (res)                        \
 365                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 366 } while(0)
 367
 368 #ifndef CONFIG_CPU_MIPSR6
 369 #define     _StoreW(addr, value, res, type)  \
 370 do {                                                        \
 371                 __asm__ __volatile__ (                      \
 372                         "1:\t"type##_swl("%1", "(%2)")"\n"  \
 373                         "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
 374                         "li\t%0, 0\n"                       \
 375                         "3:\n\t"                            \
 376                         ".insn\n\t"                         \
 377                         ".section\t.fixup,\"ax\"\n\t"       \
 378                         "4:\tli\t%0, %3\n\t"                \
 379                         "j\t3b\n\t"                         \
 380                         ".previous\n\t"                     \
 381                         ".section\t__ex_table,\"a\"\n\t"    \
 382                         STR(PTR)"\t1b, 4b\n\t"              \
 383                         STR(PTR)"\t2b, 4b\n\t"              \
 384                         ".previous"                         \
 385                 : "=r" (res)                                \
 386                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 387 } while(0)
 388
 389 #define     _StoreDW(addr, value, res) \
 390 do {                                                        \
 391                 __asm__ __volatile__ (                      \
 392                         "1:\tsdl\t%1,(%2)\n"                \
 393                         "2:\tsdr\t%1, 7(%2)\n\t"            \
 394                         "li\t%0, 0\n"                       \
 395                         "3:\n\t"                            \
 396                         ".insn\n\t"                         \
 397                         ".section\t.fixup,\"ax\"\n\t"       \
 398                         "4:\tli\t%0, %3\n\t"                \
 399                         "j\t3b\n\t"                         \
 400                         ".previous\n\t"                     \
 401                         ".section\t__ex_table,\"a\"\n\t"    \
 402                         STR(PTR)"\t1b, 4b\n\t"              \
 403                         STR(PTR)"\t2b, 4b\n\t"              \
 404                         ".previous"                         \
 405                 : "=r" (res)                                \
 406                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 407 } while(0)
 408
 409 #else
 410 /* MIPSR6 has no swl and sdl instructions */
 411 #define     _StoreW(addr, value, res, type)  \
 412 do {                                                        \
 413                 __asm__ __volatile__ (                      \
 414                         ".set\tpush\n\t"                    \
 415                         ".set\tnoat\n\t"                    \
 416                         "1:"type##_sb("%1", "3(%2)")"\n\t"  \
 417                         "srl\t$1, %1, 0x8\n\t"              \
 418                         "2:"type##_sb("$1", "2(%2)")"\n\t"  \
 419                         "srl\t$1, $1,  0x8\n\t"             \
 420                         "3:"type##_sb("$1", "1(%2)")"\n\t"  \
 421                         "srl\t$1, $1, 0x8\n\t"              \
 422                         "4:"type##_sb("$1", "0(%2)")"\n\t"  \
 423                         ".set\tpop\n\t"                     \
 424                         "li\t%0, 0\n"                       \
 425                         "10:\n\t"                           \
 426                         ".insn\n\t"                         \
 427                         ".section\t.fixup,\"ax\"\n\t"       \
 428                         "11:\tli\t%0, %3\n\t"               \
 429                         "j\t10b\n\t"                        \
 430                         ".previous\n\t"                     \
 431                         ".section\t__ex_table,\"a\"\n\t"    \
 432                         STR(PTR)"\t1b, 11b\n\t"             \
 433                         STR(PTR)"\t2b, 11b\n\t"             \
 434                         STR(PTR)"\t3b, 11b\n\t"             \
 435                         STR(PTR)"\t4b, 11b\n\t"             \
 436                         ".previous"                         \
 437                 : "=&r" (res)                               \
 438                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 439                 : "memory");                                \
 440 } while(0)
 441
 442 #define     _StoreDW(addr, value, res) \
 443 do {                                                        \
 444                 __asm__ __volatile__ (                      \
 445                         ".set\tpush\n\t"                    \
 446                         ".set\tnoat\n\t"                    \
 447                         "1:sb\t%1, 7(%2)\n\t"               \
 448                         "dsrl\t$1, %1, 0x8\n\t"             \
 449                         "2:sb\t$1, 6(%2)\n\t"               \
 450                         "dsrl\t$1, $1, 0x8\n\t"             \
 451                         "3:sb\t$1, 5(%2)\n\t"               \
 452                         "dsrl\t$1, $1, 0x8\n\t"             \
 453                         "4:sb\t$1, 4(%2)\n\t"               \
 454                         "dsrl\t$1, $1, 0x8\n\t"             \
 455                         "5:sb\t$1, 3(%2)\n\t"               \
 456                         "dsrl\t$1, $1, 0x8\n\t"             \
 457                         "6:sb\t$1, 2(%2)\n\t"               \
 458                         "dsrl\t$1, $1, 0x8\n\t"             \
 459                         "7:sb\t$1, 1(%2)\n\t"               \
 460                         "dsrl\t$1, $1, 0x8\n\t"             \
 461                         "8:sb\t$1, 0(%2)\n\t"               \
 462                         "dsrl\t$1, $1, 0x8\n\t"             \
 463                         ".set\tpop\n\t"                     \
 464                         "li\t%0, 0\n"                       \
 465                         "10:\n\t"                           \
 466                         ".insn\n\t"                         \
 467                         ".section\t.fixup,\"ax\"\n\t"       \
 468                         "11:\tli\t%0, %3\n\t"               \
 469                         "j\t10b\n\t"                        \
 470                         ".previous\n\t"                     \
 471                         ".section\t__ex_table,\"a\"\n\t"    \
 472                         STR(PTR)"\t1b, 11b\n\t"             \
 473                         STR(PTR)"\t2b, 11b\n\t"             \
 474                         STR(PTR)"\t3b, 11b\n\t"             \
 475                         STR(PTR)"\t4b, 11b\n\t"             \
 476                         STR(PTR)"\t5b, 11b\n\t"             \
 477                         STR(PTR)"\t6b, 11b\n\t"             \
 478                         STR(PTR)"\t7b, 11b\n\t"             \
 479                         STR(PTR)"\t8b, 11b\n\t"             \
 480                         ".previous"                         \
 481                 : "=&r" (res)                               \
 482                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 483                 : "memory");                                \
 484 } while(0)
 485
 486 #endif /* CONFIG_CPU_MIPSR6 */
 487
 488 #else /* __BIG_ENDIAN */
 489
 490 #define     _LoadHW(addr, value, res, type)  \
 491 do {                                                        \
 492                 __asm__ __volatile__ (".set\tnoat\n"        \
 493                         "1:\t"type##_lb("%0", "1(%2)")"\n"  \
 494                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 495                         "sll\t%0, 0x8\n\t"                  \
 496                         "or\t%0, $1\n\t"                    \
 497                         "li\t%1, 0\n"                       \
 498                         "3:\t.set\tat\n\t"                  \
 499                         ".insn\n\t"                         \
 500                         ".section\t.fixup,\"ax\"\n\t"       \
 501                         "4:\tli\t%1, %3\n\t"                \
 502                         "j\t3b\n\t"                         \
 503                         ".previous\n\t"                     \
 504                         ".section\t__ex_table,\"a\"\n\t"    \
 505                         STR(PTR)"\t1b, 4b\n\t"              \
 506                         STR(PTR)"\t2b, 4b\n\t"              \
 507                         ".previous"                         \
 508                         : "=&r" (value), "=r" (res)         \
 509                         : "r" (addr), "i" (-EFAULT));       \
 510 } while(0)
 511
 512 #ifndef CONFIG_CPU_MIPSR6
 513 #define     _LoadW(addr, value, res, type)   \
 514 do {                                                        \
 515                 __asm__ __volatile__ (                      \
 516                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 517                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 518                         "li\t%1, 0\n"                       \
 519                         "3:\n\t"                            \
 520                         ".insn\n\t"                         \
 521                         ".section\t.fixup,\"ax\"\n\t"       \
 522                         "4:\tli\t%1, %3\n\t"                \
 523                         "j\t3b\n\t"                         \
 524                         ".previous\n\t"                     \
 525                         ".section\t__ex_table,\"a\"\n\t"    \
 526                         STR(PTR)"\t1b, 4b\n\t"              \
 527                         STR(PTR)"\t2b, 4b\n\t"              \
 528                         ".previous"                         \
 529                         : "=&r" (value), "=r" (res)         \
 530                         : "r" (addr), "i" (-EFAULT));       \
 531 } while(0)
 532
 533 #else
 534 /* MIPSR6 has no lwl instruction */
 535 #define     _LoadW(addr, value, res, type) \
 536 do {                                                        \
 537                 __asm__ __volatile__ (                      \
 538                         ".set\tpush\n"                      \
 539                         ".set\tnoat\n\t"                    \
 540                         "1:"type##_lb("%0", "3(%2)")"\n\t"  \
 541                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 542                         "sll\t%0, 0x8\n\t"                  \
 543                         "or\t%0, $1\n\t"                    \
 544                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 545                         "sll\t%0, 0x8\n\t"                  \
 546                         "or\t%0, $1\n\t"                    \
 547                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 548                         "sll\t%0, 0x8\n\t"                  \
 549                         "or\t%0, $1\n\t"                    \
 550                         "li\t%1, 0\n"                       \
 551                         ".set\tpop\n"                       \
 552                         "10:\n\t"                           \
 553                         ".insn\n\t"                         \
 554                         ".section\t.fixup,\"ax\"\n\t"       \
 555                         "11:\tli\t%1, %3\n\t"               \
 556                         "j\t10b\n\t"                        \
 557                         ".previous\n\t"                     \
 558                         ".section\t__ex_table,\"a\"\n\t"    \
 559                         STR(PTR)"\t1b, 11b\n\t"             \
 560                         STR(PTR)"\t2b, 11b\n\t"             \
 561                         STR(PTR)"\t3b, 11b\n\t"             \
 562                         STR(PTR)"\t4b, 11b\n\t"             \
 563                         ".previous"                         \
 564                         : "=&r" (value), "=r" (res)         \
 565                         : "r" (addr), "i" (-EFAULT));       \
 566 } while(0)
 567
 568 #endif /* CONFIG_CPU_MIPSR6 */
 569
 570
 571 #define     _LoadHWU(addr, value, res, type) \
 572 do {                                                        \
 573                 __asm__ __volatile__ (                      \
 574                         ".set\tnoat\n"                      \
 575                         "1:\t"type##_lbu("%0", "1(%2)")"\n" \
 576                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 577                         "sll\t%0, 0x8\n\t"                  \
 578                         "or\t%0, $1\n\t"                    \
 579                         "li\t%1, 0\n"                       \
 580                         "3:\n\t"                            \
 581                         ".insn\n\t"                         \
 582                         ".set\tat\n\t"                      \
 583                         ".section\t.fixup,\"ax\"\n\t"       \
 584                         "4:\tli\t%1, %3\n\t"                \
 585                         "j\t3b\n\t"                         \
 586                         ".previous\n\t"                     \
 587                         ".section\t__ex_table,\"a\"\n\t"    \
 588                         STR(PTR)"\t1b, 4b\n\t"              \
 589                         STR(PTR)"\t2b, 4b\n\t"              \
 590                         ".previous"                         \
 591                         : "=&r" (value), "=r" (res)         \
 592                         : "r" (addr), "i" (-EFAULT));       \
 593 } while(0)
 594
 595 #ifndef CONFIG_CPU_MIPSR6
 596 #define     _LoadWU(addr, value, res, type)  \
 597 do {                                                        \
 598                 __asm__ __volatile__ (                      \
 599                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 600                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 601                         "dsll\t%0, %0, 32\n\t"              \
 602                         "dsrl\t%0, %0, 32\n\t"              \
 603                         "li\t%1, 0\n"                       \
 604                         "3:\n\t"                            \
 605                         ".insn\n\t"                         \
 606                         "\t.section\t.fixup,\"ax\"\n\t"     \
 607                         "4:\tli\t%1, %3\n\t"                \
 608                         "j\t3b\n\t"                         \
 609                         ".previous\n\t"                     \
 610                         ".section\t__ex_table,\"a\"\n\t"    \
 611                         STR(PTR)"\t1b, 4b\n\t"              \
 612                         STR(PTR)"\t2b, 4b\n\t"              \
 613                         ".previous"                         \
 614                         : "=&r" (value), "=r" (res)         \
 615                         : "r" (addr), "i" (-EFAULT));       \
 616 } while(0)
 617
 618 #define     _LoadDW(addr, value, res)  \
 619 do {                                                        \
 620                 __asm__ __volatile__ (                      \
 621                         "1:\tldl\t%0, 7(%2)\n"              \
 622                         "2:\tldr\t%0, (%2)\n\t"             \
 623                         "li\t%1, 0\n"                       \
 624                         "3:\n\t"                            \
 625                         ".insn\n\t"                         \
 626                         "\t.section\t.fixup,\"ax\"\n\t"     \
 627                         "4:\tli\t%1, %3\n\t"                \
 628                         "j\t3b\n\t"                         \
 629                         ".previous\n\t"                     \
 630                         ".section\t__ex_table,\"a\"\n\t"    \
 631                         STR(PTR)"\t1b, 4b\n\t"              \
 632                         STR(PTR)"\t2b, 4b\n\t"              \
 633                         ".previous"                         \
 634                         : "=&r" (value), "=r" (res)         \
 635                         : "r" (addr), "i" (-EFAULT));       \
 636 } while(0)
 637
 638 #else
 639 /* MIPSR6 has not lwl and ldl instructions */
 640 #define     _LoadWU(addr, value, res, type) \
 641 do {                                                        \
 642                 __asm__ __volatile__ (                      \
 643                         ".set\tpush\n\t"                    \
 644                         ".set\tnoat\n\t"                    \
 645                         "1:"type##_lbu("%0", "3(%2)")"\n\t" \
 646                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 647                         "sll\t%0, 0x8\n\t"                  \
 648                         "or\t%0, $1\n\t"                    \
 649                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 650                         "sll\t%0, 0x8\n\t"                  \
 651                         "or\t%0, $1\n\t"                    \
 652                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 653                         "sll\t%0, 0x8\n\t"                  \
 654                         "or\t%0, $1\n\t"                    \
 655                         "li\t%1, 0\n"                       \
 656                         ".set\tpop\n"                       \
 657                         "10:\n\t"                           \
 658                         ".insn\n\t"                         \
 659                         ".section\t.fixup,\"ax\"\n\t"       \
 660                         "11:\tli\t%1, %3\n\t"               \
 661                         "j\t10b\n\t"                        \
 662                         ".previous\n\t"                     \
 663                         ".section\t__ex_table,\"a\"\n\t"    \
 664                         STR(PTR)"\t1b, 11b\n\t"             \
 665                         STR(PTR)"\t2b, 11b\n\t"             \
 666                         STR(PTR)"\t3b, 11b\n\t"             \
 667                         STR(PTR)"\t4b, 11b\n\t"             \
 668                         ".previous"                         \
 669                         : "=&r" (value), "=r" (res)         \
 670                         : "r" (addr), "i" (-EFAULT));       \
 671 } while(0)
 672
 673 #define     _LoadDW(addr, value, res)  \
 674 do {                                                        \
 675                 __asm__ __volatile__ (                      \
 676                         ".set\tpush\n\t"                    \
 677                         ".set\tnoat\n\t"                    \
 678                         "1:lb\t%0, 7(%2)\n\t"               \
 679                         "2:lbu\t$1, 6(%2)\n\t"              \
 680                         "dsll\t%0, 0x8\n\t"                 \
 681                         "or\t%0, $1\n\t"                    \
 682                         "3:lbu\t$1, 5(%2)\n\t"              \
 683                         "dsll\t%0, 0x8\n\t"                 \
 684                         "or\t%0, $1\n\t"                    \
 685                         "4:lbu\t$1, 4(%2)\n\t"              \
 686                         "dsll\t%0, 0x8\n\t"                 \
 687                         "or\t%0, $1\n\t"                    \
 688                         "5:lbu\t$1, 3(%2)\n\t"              \
 689                         "dsll\t%0, 0x8\n\t"                 \
 690                         "or\t%0, $1\n\t"                    \
 691                         "6:lbu\t$1, 2(%2)\n\t"              \
 692                         "dsll\t%0, 0x8\n\t"                 \
 693                         "or\t%0, $1\n\t"                    \
 694                         "7:lbu\t$1, 1(%2)\n\t"              \
 695                         "dsll\t%0, 0x8\n\t"                 \
 696                         "or\t%0, $1\n\t"                    \
 697                         "8:lbu\t$1, 0(%2)\n\t"              \
 698                         "dsll\t%0, 0x8\n\t"                 \
 699                         "or\t%0, $1\n\t"                    \
 700                         "li\t%1, 0\n"                       \
 701                         ".set\tpop\n\t"                     \
 702                         "10:\n\t"                           \
 703                         ".insn\n\t"                         \
 704                         ".section\t.fixup,\"ax\"\n\t"       \
 705                         "11:\tli\t%1, %3\n\t"               \
 706                         "j\t10b\n\t"                        \
 707                         ".previous\n\t"                     \
 708                         ".section\t__ex_table,\"a\"\n\t"    \
 709                         STR(PTR)"\t1b, 11b\n\t"             \
 710                         STR(PTR)"\t2b, 11b\n\t"             \
 711                         STR(PTR)"\t3b, 11b\n\t"             \
 712                         STR(PTR)"\t4b, 11b\n\t"             \
 713                         STR(PTR)"\t5b, 11b\n\t"             \
 714                         STR(PTR)"\t6b, 11b\n\t"             \
 715                         STR(PTR)"\t7b, 11b\n\t"             \
 716                         STR(PTR)"\t8b, 11b\n\t"             \
 717                         ".previous"                         \
 718                         : "=&r" (value), "=r" (res)         \
 719                         : "r" (addr), "i" (-EFAULT));       \
 720 } while(0)
 721 #endif /* CONFIG_CPU_MIPSR6 */
 722
 723 #define     _StoreHW(addr, value, res, type) \
 724 do {                                                        \
 725                 __asm__ __volatile__ (                      \
 726                         ".set\tnoat\n"                      \
 727                         "1:\t"type##_sb("%1", "0(%2)")"\n"  \
 728                         "srl\t$1,%1, 0x8\n"                 \
 729                         "2:\t"type##_sb("$1", "1(%2)")"\n"  \
 730                         ".set\tat\n\t"                      \
 731                         "li\t%0, 0\n"                       \
 732                         "3:\n\t"                            \
 733                         ".insn\n\t"                         \
 734                         ".section\t.fixup,\"ax\"\n\t"       \
 735                         "4:\tli\t%0, %3\n\t"                \
 736                         "j\t3b\n\t"                         \
 737                         ".previous\n\t"                     \
 738                         ".section\t__ex_table,\"a\"\n\t"    \
 739                         STR(PTR)"\t1b, 4b\n\t"              \
 740                         STR(PTR)"\t2b, 4b\n\t"              \
 741                         ".previous"                         \
 742                         : "=r" (res)                        \
 743                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 744 } while(0)
 745
 746 #ifndef CONFIG_CPU_MIPSR6
 747 #define     _StoreW(addr, value, res, type)  \
 748 do {                                                        \
 749                 __asm__ __volatile__ (                      \
 750                         "1:\t"type##_swl("%1", "3(%2)")"\n" \
 751                         "2:\t"type##_swr("%1", "(%2)")"\n\t"\
 752                         "li\t%0, 0\n"                       \
 753                         "3:\n\t"                            \
 754                         ".insn\n\t"                         \
 755                         ".section\t.fixup,\"ax\"\n\t"       \
 756                         "4:\tli\t%0, %3\n\t"                \
 757                         "j\t3b\n\t"                         \
 758                         ".previous\n\t"                     \
 759                         ".section\t__ex_table,\"a\"\n\t"    \
 760                         STR(PTR)"\t1b, 4b\n\t"              \
 761                         STR(PTR)"\t2b, 4b\n\t"              \
 762                         ".previous"                         \
 763                 : "=r" (res)                                \
 764                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 765 } while(0)
 766
 767 #define     _StoreDW(addr, value, res) \
 768 do {                                                        \
 769                 __asm__ __volatile__ (                      \
 770                         "1:\tsdl\t%1, 7(%2)\n"              \
 771                         "2:\tsdr\t%1, (%2)\n\t"             \
 772                         "li\t%0, 0\n"                       \
 773                         "3:\n\t"                            \
 774                         ".insn\n\t"                         \
 775                         ".section\t.fixup,\"ax\"\n\t"       \
 776                         "4:\tli\t%0, %3\n\t"                \
 777                         "j\t3b\n\t"                         \
 778                         ".previous\n\t"                     \
 779                         ".section\t__ex_table,\"a\"\n\t"    \
 780                         STR(PTR)"\t1b, 4b\n\t"              \
 781                         STR(PTR)"\t2b, 4b\n\t"              \
 782                         ".previous"                         \
 783                 : "=r" (res)                                \
 784                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 785 } while(0)
 786
 787 #else
 788 /* MIPSR6 has no swl and sdl instructions */
 789 #define     _StoreW(addr, value, res, type)  \
 790 do {                                                        \
 791                 __asm__ __volatile__ (                      \
 792                         ".set\tpush\n\t"                    \
 793                         ".set\tnoat\n\t"                    \
 794                         "1:"type##_sb("%1", "0(%2)")"\n\t"  \
 795                         "srl\t$1, %1, 0x8\n\t"              \
 796                         "2:"type##_sb("$1", "1(%2)")"\n\t"  \
 797                         "srl\t$1, $1,  0x8\n\t"             \
 798                         "3:"type##_sb("$1", "2(%2)")"\n\t"  \
 799                         "srl\t$1, $1, 0x8\n\t"              \
 800                         "4:"type##_sb("$1", "3(%2)")"\n\t"  \
 801                         ".set\tpop\n\t"                     \
 802                         "li\t%0, 0\n"                       \
 803                         "10:\n\t"                           \
 804                         ".insn\n\t"                         \
 805                         ".section\t.fixup,\"ax\"\n\t"       \
 806                         "11:\tli\t%0, %3\n\t"               \
 807                         "j\t10b\n\t"                        \
 808                         ".previous\n\t"                     \
 809                         ".section\t__ex_table,\"a\"\n\t"    \
 810                         STR(PTR)"\t1b, 11b\n\t"             \
 811                         STR(PTR)"\t2b, 11b\n\t"             \
 812                         STR(PTR)"\t3b, 11b\n\t"             \
 813                         STR(PTR)"\t4b, 11b\n\t"             \
 814                         ".previous"                         \
 815                 : "=&r" (res)                               \
 816                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 817                 : "memory");                                \
 818 } while(0)
 819
 820 #define     _StoreDW(addr, value, res) \
 821 do {                                                        \
 822                 __asm__ __volatile__ (                      \
 823                         ".set\tpush\n\t"                    \
 824                         ".set\tnoat\n\t"                    \
 825                         "1:sb\t%1, 0(%2)\n\t"               \
 826                         "dsrl\t$1, %1, 0x8\n\t"             \
 827                         "2:sb\t$1, 1(%2)\n\t"               \
 828                         "dsrl\t$1, $1, 0x8\n\t"             \
 829                         "3:sb\t$1, 2(%2)\n\t"               \
 830                         "dsrl\t$1, $1, 0x8\n\t"             \
 831                         "4:sb\t$1, 3(%2)\n\t"               \
 832                         "dsrl\t$1, $1, 0x8\n\t"             \
 833                         "5:sb\t$1, 4(%2)\n\t"               \
 834                         "dsrl\t$1, $1, 0x8\n\t"             \
 835                         "6:sb\t$1, 5(%2)\n\t"               \
 836                         "dsrl\t$1, $1, 0x8\n\t"             \
 837                         "7:sb\t$1, 6(%2)\n\t"               \
 838                         "dsrl\t$1, $1, 0x8\n\t"             \
 839                         "8:sb\t$1, 7(%2)\n\t"               \
 840                         "dsrl\t$1, $1, 0x8\n\t"             \
 841                         ".set\tpop\n\t"                     \
 842                         "li\t%0, 0\n"                       \
 843                         "10:\n\t"                           \
 844                         ".insn\n\t"                         \
 845                         ".section\t.fixup,\"ax\"\n\t"       \
 846                         "11:\tli\t%0, %3\n\t"               \
 847                         "j\t10b\n\t"                        \
 848                         ".previous\n\t"                     \
 849                         ".section\t__ex_table,\"a\"\n\t"    \
 850                         STR(PTR)"\t1b, 11b\n\t"             \
 851                         STR(PTR)"\t2b, 11b\n\t"             \
 852                         STR(PTR)"\t3b, 11b\n\t"             \
 853                         STR(PTR)"\t4b, 11b\n\t"             \
 854                         STR(PTR)"\t5b, 11b\n\t"             \
 855                         STR(PTR)"\t6b, 11b\n\t"             \
 856                         STR(PTR)"\t7b, 11b\n\t"             \
 857                         STR(PTR)"\t8b, 11b\n\t"             \
 858                         ".previous"                         \
 859                 : "=&r" (res)                               \
 860                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 861                 : "memory");                                \
 862 } while(0)
 863
 864 #endif /* CONFIG_CPU_MIPSR6 */
 865 #endif
 866
 867 #define LoadHWU(addr, value, res)       _LoadHWU(addr, value, res, kernel)
 868 #define LoadHWUE(addr, value, res)      _LoadHWU(addr, value, res, user)
 869 #define LoadWU(addr, value, res)        _LoadWU(addr, value, res, kernel)
 870 #define LoadWUE(addr, value, res)       _LoadWU(addr, value, res, user)
 871 #define LoadHW(addr, value, res)        _LoadHW(addr, value, res, kernel)
 872 #define LoadHWE(addr, value, res)       _LoadHW(addr, value, res, user)
 873 #define LoadW(addr, value, res)         _LoadW(addr, value, res, kernel)
 874 #define LoadWE(addr, value, res)        _LoadW(addr, value, res, user)
 875 #define LoadDW(addr, value, res)        _LoadDW(addr, value, res)
 876
 877 #define StoreHW(addr, value, res)       _StoreHW(addr, value, res, kernel)
 878 #define StoreHWE(addr, value, res)      _StoreHW(addr, value, res, user)
 879 #define StoreW(addr, value, res)        _StoreW(addr, value, res, kernel)
 880 #define StoreWE(addr, value, res)       _StoreW(addr, value, res, user)
 881 #define StoreDW(addr, value, res)       _StoreDW(addr, value, res)
 882
 883 static void emulate_load_store_insn(struct pt_regs *regs,
 884         void __user *addr, unsigned int __user *pc)
 885 {
 886         union mips_instruction insn;
 887         unsigned long value;
 888         unsigned int res, preempted;
 889         unsigned long origpc;
 890         unsigned long orig31;
 891         void __user *fault_addr = NULL;
 892 #ifdef  CONFIG_EVA
 893         mm_segment_t seg;
 894 #endif
 895         union fpureg *fpr;
 896         enum msa_2b_fmt df;
 897         unsigned int wd;
 898         origpc = (unsigned long)pc;
 899         orig31 = regs->regs[31];
 900
 901         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 902
 903         /*
 904          * This load never faults.
 905          */
 906         __get_user(insn.word, pc);
 907
 908         switch (insn.i_format.opcode) {
 909                 /*
 910                  * These are instructions that a compiler doesn't generate.  We
 911                  * can assume therefore that the code is MIPS-aware and
 912                  * really buggy.  Emulating these instructions would break the
 913                  * semantics anyway.
 914                  */
 915         case ll_op:
 916         case lld_op:
 917         case sc_op:
 918         case scd_op:
 919
 920                 /*
 921                  * For these instructions the only way to create an address
 922                  * error is an attempted access to kernel/supervisor address
 923                  * space.
 924                  */
 925         case ldl_op:
 926         case ldr_op:
 927         case lwl_op:
 928         case lwr_op:
 929         case sdl_op:
 930         case sdr_op:
 931         case swl_op:
 932         case swr_op:
 933         case lb_op:
 934         case lbu_op:
 935         case sb_op:
 936                 goto sigbus;
 937
 938                 /*
 939                  * The remaining opcodes are the ones that are really of
 940                  * interest.
 941                  */
 942 #ifdef CONFIG_EVA
 943         case spec3_op:
 944                 /*
 945                  * we can land here only from kernel accessing user memory,
 946                  * so we need to "switch" the address limit to user space, so
 947                  * address check can work properly.
 948                  */
 949                 seg = get_fs();
 950                 set_fs(USER_DS);
 951                 switch (insn.spec3_format.func) {
 952                 case lhe_op:
 953                         if (!access_ok(VERIFY_READ, addr, 2)) {
 954                                 set_fs(seg);
 955                                 goto sigbus;
 956                         }
 957                         LoadHWE(addr, value, res);
 958                         if (res) {
 959                                 set_fs(seg);
 960                                 goto fault;
 961                         }
 962                         compute_return_epc(regs);
 963                         regs->regs[insn.spec3_format.rt] = value;
 964                         break;
 965                 case lwe_op:
 966                         if (!access_ok(VERIFY_READ, addr, 4)) {
 967                                 set_fs(seg);
 968                                 goto sigbus;
 969                         }
 970                                 LoadWE(addr, value, res);
 971                         if (res) {
 972                                 set_fs(seg);
 973                                 goto fault;
 974                         }
 975                         compute_return_epc(regs);
 976                         regs->regs[insn.spec3_format.rt] = value;
 977                         break;
 978                 case lhue_op:
 979                         if (!access_ok(VERIFY_READ, addr, 2)) {
 980                                 set_fs(seg);
 981                                 goto sigbus;
 982                         }
 983                         LoadHWUE(addr, value, res);
 984                         if (res) {
 985                                 set_fs(seg);
 986                                 goto fault;
 987                         }
 988                         compute_return_epc(regs);
 989                         regs->regs[insn.spec3_format.rt] = value;
 990                         break;
 991                 case she_op:
 992                         if (!access_ok(VERIFY_WRITE, addr, 2)) {
 993                                 set_fs(seg);
 994                                 goto sigbus;
 995                         }
 996                         compute_return_epc(regs);
 997                         value = regs->regs[insn.spec3_format.rt];
 998                         StoreHWE(addr, value, res);
 999                         if (res) {
1000                                 set_fs(seg);
1001                                 goto fault;
1002                         }
1003                         break;
1004                 case swe_op:
1005                         if (!access_ok(VERIFY_WRITE, addr, 4)) {
1006                                 set_fs(seg);
1007                                 goto sigbus;
1008                         }
1009                         compute_return_epc(regs);
1010                         value = regs->regs[insn.spec3_format.rt];
1011                         StoreWE(addr, value, res);
1012                         if (res) {
1013                                 set_fs(seg);
1014                                 goto fault;
1015                         }
1016                         break;
1017                 default:
1018                         set_fs(seg);
1019                         goto sigill;
1020                 }
1021                 set_fs(seg);
1022                 break;
1023 #endif
1024         case lh_op:
1025                 if (!access_ok(VERIFY_READ, addr, 2))
1026                         goto sigbus;
1027
1028                 if (config_enabled(CONFIG_EVA)) {
1029                         if (segment_eq(get_fs(), get_ds()))
1030                                 LoadHW(addr, value, res);
1031                         else
1032                                 LoadHWE(addr, value, res);
1033                 } else {
1034                         LoadHW(addr, value, res);
1035                 }
1036
1037                 if (res)
1038                         goto fault;
1039                 compute_return_epc(regs);
1040                 regs->regs[insn.i_format.rt] = value;
1041                 break;
1042
1043         case lw_op:
1044                 if (!access_ok(VERIFY_READ, addr, 4))
1045                         goto sigbus;
1046
1047                 if (config_enabled(CONFIG_EVA)) {
1048                         if (segment_eq(get_fs(), get_ds()))
1049                                 LoadW(addr, value, res);
1050                         else
1051                                 LoadWE(addr, value, res);
1052                 } else {
1053                         LoadW(addr, value, res);
1054                 }
1055
1056                 if (res)
1057                         goto fault;
1058                 compute_return_epc(regs);
1059                 regs->regs[insn.i_format.rt] = value;
1060                 break;
1061
1062         case lhu_op:
1063                 if (!access_ok(VERIFY_READ, addr, 2))
1064                         goto sigbus;
1065
1066                 if (config_enabled(CONFIG_EVA)) {
1067                         if (segment_eq(get_fs(), get_ds()))
1068                                 LoadHWU(addr, value, res);
1069                         else
1070                                 LoadHWUE(addr, value, res);
1071                 } else {
1072                         LoadHWU(addr, value, res);
1073                 }
1074
1075                 if (res)
1076                         goto fault;
1077                 compute_return_epc(regs);
1078                 regs->regs[insn.i_format.rt] = value;
1079                 break;
1080
1081         case lwu_op:
1082 #ifdef CONFIG_64BIT
1083                 /*
1084                  * A 32-bit kernel might be running on a 64-bit processor.  But
1085                  * if we're on a 32-bit processor and an i-cache incoherency
1086                  * or race makes us see a 64-bit instruction here the sdl/sdr
1087                  * would blow up, so for now we don't handle unaligned 64-bit
1088                  * instructions on 32-bit kernels.
1089                  */
1090                 if (!access_ok(VERIFY_READ, addr, 4))
1091                         goto sigbus;
1092
1093                 LoadWU(addr, value, res);
1094                 if (res)
1095                         goto fault;
1096                 compute_return_epc(regs);
1097                 regs->regs[insn.i_format.rt] = value;
1098                 break;
1099 #endif /* CONFIG_64BIT */
1100
1101                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1102                 goto sigill;
1103
1104         case ld_op:
1105 #ifdef CONFIG_64BIT
1106                 /*
1107                  * A 32-bit kernel might be running on a 64-bit processor.  But
1108                  * if we're on a 32-bit processor and an i-cache incoherency
1109                  * or race makes us see a 64-bit instruction here the sdl/sdr
1110                  * would blow up, so for now we don't handle unaligned 64-bit
1111                  * instructions on 32-bit kernels.
1112                  */
1113                 if (!access_ok(VERIFY_READ, addr, 8))
1114                         goto sigbus;
1115
1116                 LoadDW(addr, value, res);
1117                 if (res)
1118                         goto fault;
1119                 compute_return_epc(regs);
1120                 regs->regs[insn.i_format.rt] = value;
1121                 break;
1122 #endif /* CONFIG_64BIT */
1123
1124                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1125                 goto sigill;
1126
1127         case sh_op:
1128                 if (!access_ok(VERIFY_WRITE, addr, 2))
1129                         goto sigbus;
1130
1131                 compute_return_epc(regs);
1132                 value = regs->regs[insn.i_format.rt];
1133
1134                 if (config_enabled(CONFIG_EVA)) {
1135                         if (segment_eq(get_fs(), get_ds()))
1136                                 StoreHW(addr, value, res);
1137                         else
1138                                 StoreHWE(addr, value, res);
1139                 } else {
1140                         StoreHW(addr, value, res);
1141                 }
1142
1143                 if (res)
1144                         goto fault;
1145                 break;
1146
1147         case sw_op:
1148                 if (!access_ok(VERIFY_WRITE, addr, 4))
1149                         goto sigbus;
1150
1151                 compute_return_epc(regs);
1152                 value = regs->regs[insn.i_format.rt];
1153
1154                 if (config_enabled(CONFIG_EVA)) {
1155                         if (segment_eq(get_fs(), get_ds()))
1156                                 StoreW(addr, value, res);
1157                         else
1158                                 StoreWE(addr, value, res);
1159                 } else {
1160                         StoreW(addr, value, res);
1161                 }
1162
1163                 if (res)
1164                         goto fault;
1165                 break;
1166
1167         case sd_op:
1168 #ifdef CONFIG_64BIT
1169                 /*
1170                  * A 32-bit kernel might be running on a 64-bit processor.  But
1171                  * if we're on a 32-bit processor and an i-cache incoherency
1172                  * or race makes us see a 64-bit instruction here the sdl/sdr
1173                  * would blow up, so for now we don't handle unaligned 64-bit
1174                  * instructions on 32-bit kernels.
1175                  */
1176                 if (!access_ok(VERIFY_WRITE, addr, 8))
1177                         goto sigbus;
1178
1179                 compute_return_epc(regs);
1180                 value = regs->regs[insn.i_format.rt];
1181                 StoreDW(addr, value, res);
1182                 if (res)
1183                         goto fault;
1184                 break;
1185 #endif /* CONFIG_64BIT */
1186
1187                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1188                 goto sigill;
1189
1190         case lwc1_op:
1191         case ldc1_op:
1192         case swc1_op:
1193         case sdc1_op:
1194                 die_if_kernel("Unaligned FP access in kernel code", regs);
1195                 BUG_ON(!used_math());
1196
1197                 lose_fpu(1);    /* Save FPU state for the emulator. */
1198                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1199                                                &fault_addr);
1200                 own_fpu(1);     /* Restore FPU state. */
1201
1202                 /* Signal if something went wrong. */
1203                 process_fpemu_return(res, fault_addr, 0);
1204
1205                 if (res == 0)
1206                         break;
1207                 return;
1208
1209         case msa_op:
1210                 if (!cpu_has_msa)
1211                         goto sigill;
1212
1213                 /*
1214                  * If we've reached this point then userland should have taken
1215                  * the MSA disabled exception & initialised vector context at
1216                  * some point in the past.
1217                  */
1218                 BUG_ON(!thread_msa_context_live());
1219
1220                 df = insn.msa_mi10_format.df;
1221                 wd = insn.msa_mi10_format.wd;
1222                 fpr = &current->thread.fpu.fpr[wd];
1223
1224                 switch (insn.msa_mi10_format.func) {
1225                 case msa_ld_op:
1226                         if (!access_ok(VERIFY_READ, addr, sizeof(*fpr)))
1227                                 goto sigbus;
1228
1229                         do {
1230                                 /*
1231                                  * If we have live MSA context keep track of
1232                                  * whether we get preempted in order to avoid
1233                                  * the register context we load being clobbered
1234                                  * by the live context as it's saved during
1235                                  * preemption. If we don't have live context
1236                                  * then it can't be saved to clobber the value
1237                                  * we load.
1238                                  */
1239                                 preempted = test_thread_flag(TIF_USEDMSA);
1240
1241                                 res = __copy_from_user_inatomic(fpr, addr,
1242                                                                 sizeof(*fpr));
1243                                 if (res)
1244                                         goto fault;
1245
1246                                 /*
1247                                  * Update the hardware register if it is in use
1248                                  * by the task in this quantum, in order to
1249                                  * avoid having to save & restore the whole
1250                                  * vector context.
1251                                  */
1252                                 preempt_disable();
1253                                 if (test_thread_flag(TIF_USEDMSA)) {
1254                                         write_msa_wr(wd, fpr, df);
1255                                         preempted = 0;
1256                                 }
1257                                 preempt_enable();
1258                         } while (preempted);
1259                         break;
1260
1261                 case msa_st_op:
1262                         if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr)))
1263                                 goto sigbus;
1264
1265                         /*
1266                          * Update from the hardware register if it is in use by
1267                          * the task in this quantum, in order to avoid having to
1268                          * save & restore the whole vector context.
1269                          */
1270                         preempt_disable();
1271                         if (test_thread_flag(TIF_USEDMSA))
1272                                 read_msa_wr(wd, fpr, df);
1273                         preempt_enable();
1274
1275                         res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1276                         if (res)
1277                                 goto fault;
1278                         break;
1279
1280                 default:
1281                         goto sigbus;
1282                 }
1283
1284                 compute_return_epc(regs);
1285                 break;
1286
1287 #ifndef CONFIG_CPU_MIPSR6
1288         /*
1289          * COP2 is available to implementor for application specific use.
1290          * It's up to applications to register a notifier chain and do
1291          * whatever they have to do, including possible sending of signals.
1292          *
1293          * This instruction has been reallocated in Release 6
1294          */
1295         case lwc2_op:
1296                 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1297                 break;
1298
1299         case ldc2_op:
1300                 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1301                 break;
1302
1303         case swc2_op:
1304                 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1305                 break;
1306
1307         case sdc2_op:
1308                 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1309                 break;
1310 #endif
1311         default:
1312                 /*
1313                  * Pheeee...  We encountered an yet unknown instruction or
1314                  * cache coherence problem.  Die sucker, die ...
1315                  */
1316                 goto sigill;
1317         }
1318
1319 #ifdef CONFIG_DEBUG_FS
1320         unaligned_instructions++;
1321 #endif
1322
1323         return;
1324
1325 fault:
1326         /* roll back jump/branch */
1327         regs->cp0_epc = origpc;
1328         regs->regs[31] = orig31;
1329         /* Did we have an exception handler installed? */
1330         if (fixup_exception(regs))
1331                 return;
1332
1333         die_if_kernel("Unhandled kernel unaligned access", regs);
1334         force_sig(SIGSEGV, current);
1335
1336         return;
1337
1338 sigbus:
1339         die_if_kernel("Unhandled kernel unaligned access", regs);
1340         force_sig(SIGBUS, current);
1341
1342         return;
1343
1344 sigill:
1345         die_if_kernel
1346             ("Unhandled kernel unaligned access or invalid instruction", regs);
1347         force_sig(SIGILL, current);
1348 }
1349
1350 /* Recode table from 16-bit register notation to 32-bit GPR. */
1351 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1352
1353 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1354 const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1355
1356 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1357                                          void __user *addr)
1358 {
1359         unsigned long value;
1360         unsigned int res;
1361         int i;
1362         unsigned int reg = 0, rvar;
1363         unsigned long orig31;
1364         u16 __user *pc16;
1365         u16 halfword;
1366         unsigned int word;
1367         unsigned long origpc, contpc;
1368         union mips_instruction insn;
1369         struct mm_decoded_insn mminsn;
1370         void __user *fault_addr = NULL;
1371
1372         origpc = regs->cp0_epc;
1373         orig31 = regs->regs[31];
1374
1375         mminsn.micro_mips_mode = 1;
1376
1377         /*
1378          * This load never faults.
1379          */
1380         pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1381         __get_user(halfword, pc16);
1382         pc16++;
1383         contpc = regs->cp0_epc + 2;
1384         word = ((unsigned int)halfword << 16);
1385         mminsn.pc_inc = 2;
1386
1387         if (!mm_insn_16bit(halfword)) {
1388                 __get_user(halfword, pc16);
1389                 pc16++;
1390                 contpc = regs->cp0_epc + 4;
1391                 mminsn.pc_inc = 4;
1392                 word |= halfword;
1393         }
1394         mminsn.insn = word;
1395
1396         if (get_user(halfword, pc16))
1397                 goto fault;
1398         mminsn.next_pc_inc = 2;
1399         word = ((unsigned int)halfword << 16);
1400
1401         if (!mm_insn_16bit(halfword)) {
1402                 pc16++;
1403                 if (get_user(halfword, pc16))
1404                         goto fault;
1405                 mminsn.next_pc_inc = 4;
1406                 word |= halfword;
1407         }
1408         mminsn.next_insn = word;
1409
1410         insn = (union mips_instruction)(mminsn.insn);
1411         if (mm_isBranchInstr(regs, mminsn, &contpc))
1412                 insn = (union mips_instruction)(mminsn.next_insn);
1413
1414         /*  Parse instruction to find what to do */
1415
1416         switch (insn.mm_i_format.opcode) {
1417
1418         case mm_pool32a_op:
1419                 switch (insn.mm_x_format.func) {
1420                 case mm_lwxs_op:
1421                         reg = insn.mm_x_format.rd;
1422                         goto loadW;
1423                 }
1424
1425                 goto sigbus;
1426
1427         case mm_pool32b_op:
1428                 switch (insn.mm_m_format.func) {
1429                 case mm_lwp_func:
1430                         reg = insn.mm_m_format.rd;
1431                         if (reg == 31)
1432                                 goto sigbus;
1433
1434                         if (!access_ok(VERIFY_READ, addr, 8))
1435                                 goto sigbus;
1436
1437                         LoadW(addr, value, res);
1438                         if (res)
1439                                 goto fault;
1440                         regs->regs[reg] = value;
1441                         addr += 4;
1442                         LoadW(addr, value, res);
1443                         if (res)
1444                                 goto fault;
1445                         regs->regs[reg + 1] = value;
1446                         goto success;
1447
1448                 case mm_swp_func:
1449                         reg = insn.mm_m_format.rd;
1450                         if (reg == 31)
1451                                 goto sigbus;
1452
1453                         if (!access_ok(VERIFY_WRITE, addr, 8))
1454                                 goto sigbus;
1455
1456                         value = regs->regs[reg];
1457                         StoreW(addr, value, res);
1458                         if (res)
1459                                 goto fault;
1460                         addr += 4;
1461                         value = regs->regs[reg + 1];
1462                         StoreW(addr, value, res);
1463                         if (res)
1464                                 goto fault;
1465                         goto success;
1466
1467                 case mm_ldp_func:
1468 #ifdef CONFIG_64BIT
1469                         reg = insn.mm_m_format.rd;
1470                         if (reg == 31)
1471                                 goto sigbus;
1472
1473                         if (!access_ok(VERIFY_READ, addr, 16))
1474                                 goto sigbus;
1475
1476                         LoadDW(addr, value, res);
1477                         if (res)
1478                                 goto fault;
1479                         regs->regs[reg] = value;
1480                         addr += 8;
1481                         LoadDW(addr, value, res);
1482                         if (res)
1483                                 goto fault;
1484                         regs->regs[reg + 1] = value;
1485                         goto success;
1486 #endif /* CONFIG_64BIT */
1487
1488                         goto sigill;
1489
1490                 case mm_sdp_func:
1491 #ifdef CONFIG_64BIT
1492                         reg = insn.mm_m_format.rd;
1493                         if (reg == 31)
1494                                 goto sigbus;
1495
1496                         if (!access_ok(VERIFY_WRITE, addr, 16))
1497                                 goto sigbus;
1498
1499                         value = regs->regs[reg];
1500                         StoreDW(addr, value, res);
1501                         if (res)
1502                                 goto fault;
1503                         addr += 8;
1504                         value = regs->regs[reg + 1];
1505                         StoreDW(addr, value, res);
1506                         if (res)
1507                                 goto fault;
1508                         goto success;
1509 #endif /* CONFIG_64BIT */
1510
1511                         goto sigill;
1512
1513                 case mm_lwm32_func:
1514                         reg = insn.mm_m_format.rd;
1515                         rvar = reg & 0xf;
1516                         if ((rvar > 9) || !reg)
1517                                 goto sigill;
1518                         if (reg & 0x10) {
1519                                 if (!access_ok
1520                                     (VERIFY_READ, addr, 4 * (rvar + 1)))
1521                                         goto sigbus;
1522                         } else {
1523                                 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1524                                         goto sigbus;
1525                         }
1526                         if (rvar == 9)
1527                                 rvar = 8;
1528                         for (i = 16; rvar; rvar--, i++) {
1529                                 LoadW(addr, value, res);
1530                                 if (res)
1531                                         goto fault;
1532                                 addr += 4;
1533                                 regs->regs[i] = value;
1534                         }
1535                         if ((reg & 0xf) == 9) {
1536                                 LoadW(addr, value, res);
1537                                 if (res)
1538                                         goto fault;
1539                                 addr += 4;
1540                                 regs->regs[30] = value;
1541                         }
1542                         if (reg & 0x10) {
1543                                 LoadW(addr, value, res);
1544                                 if (res)
1545                                         goto fault;
1546                                 regs->regs[31] = value;
1547                         }
1548                         goto success;
1549
1550                 case mm_swm32_func:
1551                         reg = insn.mm_m_format.rd;
1552                         rvar = reg & 0xf;
1553                         if ((rvar > 9) || !reg)
1554                                 goto sigill;
1555                         if (reg & 0x10) {
1556                                 if (!access_ok
1557                                     (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1558                                         goto sigbus;
1559                         } else {
1560                                 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1561                                         goto sigbus;
1562                         }
1563                         if (rvar == 9)
1564                                 rvar = 8;
1565                         for (i = 16; rvar; rvar--, i++) {
1566                                 value = regs->regs[i];
1567                                 StoreW(addr, value, res);
1568                                 if (res)
1569                                         goto fault;
1570                                 addr += 4;
1571                         }
1572                         if ((reg & 0xf) == 9) {
1573                                 value = regs->regs[30];
1574                                 StoreW(addr, value, res);
1575                                 if (res)
1576                                         goto fault;
1577                                 addr += 4;
1578                         }
1579                         if (reg & 0x10) {
1580                                 value = regs->regs[31];
1581                                 StoreW(addr, value, res);
1582                                 if (res)
1583                                         goto fault;
1584                         }
1585                         goto success;
1586
1587                 case mm_ldm_func:
1588 #ifdef CONFIG_64BIT
1589                         reg = insn.mm_m_format.rd;
1590                         rvar = reg & 0xf;
1591                         if ((rvar > 9) || !reg)
1592                                 goto sigill;
1593                         if (reg & 0x10) {
1594                                 if (!access_ok
1595                                     (VERIFY_READ, addr, 8 * (rvar + 1)))
1596                                         goto sigbus;
1597                         } else {
1598                                 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1599                                         goto sigbus;
1600                         }
1601                         if (rvar == 9)
1602                                 rvar = 8;
1603
1604                         for (i = 16; rvar; rvar--, i++) {
1605                                 LoadDW(addr, value, res);
1606                                 if (res)
1607                                         goto fault;
1608                                 addr += 4;
1609                                 regs->regs[i] = value;
1610                         }
1611                         if ((reg & 0xf) == 9) {
1612                                 LoadDW(addr, value, res);
1613                                 if (res)
1614                                         goto fault;
1615                                 addr += 8;
1616                                 regs->regs[30] = value;
1617                         }
1618                         if (reg & 0x10) {
1619                                 LoadDW(addr, value, res);
1620                                 if (res)
1621                                         goto fault;
1622                                 regs->regs[31] = value;
1623                         }
1624                         goto success;
1625 #endif /* CONFIG_64BIT */
1626
1627                         goto sigill;
1628
1629                 case mm_sdm_func:
1630 #ifdef CONFIG_64BIT
1631                         reg = insn.mm_m_format.rd;
1632                         rvar = reg & 0xf;
1633                         if ((rvar > 9) || !reg)
1634                                 goto sigill;
1635                         if (reg & 0x10) {
1636                                 if (!access_ok
1637                                     (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1638                                         goto sigbus;
1639                         } else {
1640                                 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1641                                         goto sigbus;
1642                         }
1643                         if (rvar == 9)
1644                                 rvar = 8;
1645
1646                         for (i = 16; rvar; rvar--, i++) {
1647                                 value = regs->regs[i];
1648                                 StoreDW(addr, value, res);
1649                                 if (res)
1650                                         goto fault;
1651                                 addr += 8;
1652                         }
1653                         if ((reg & 0xf) == 9) {
1654                                 value = regs->regs[30];
1655                                 StoreDW(addr, value, res);
1656                                 if (res)
1657                                         goto fault;
1658                                 addr += 8;
1659                         }
1660                         if (reg & 0x10) {
1661                                 value = regs->regs[31];
1662                                 StoreDW(addr, value, res);
1663                                 if (res)
1664                                         goto fault;
1665                         }
1666                         goto success;
1667 #endif /* CONFIG_64BIT */
1668
1669                         goto sigill;
1670
1671                         /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1672                 }
1673
1674                 goto sigbus;
1675
1676         case mm_pool32c_op:
1677                 switch (insn.mm_m_format.func) {
1678                 case mm_lwu_func:
1679                         reg = insn.mm_m_format.rd;
1680                         goto loadWU;
1681                 }
1682
1683                 /*  LL,SC,LLD,SCD are not serviced */
1684                 goto sigbus;
1685
1686         case mm_pool32f_op:
1687                 switch (insn.mm_x_format.func) {
1688                 case mm_lwxc1_func:
1689                 case mm_swxc1_func:
1690                 case mm_ldxc1_func:
1691                 case mm_sdxc1_func:
1692                         goto fpu_emul;
1693                 }
1694
1695                 goto sigbus;
1696
1697         case mm_ldc132_op:
1698         case mm_sdc132_op:
1699         case mm_lwc132_op:
1700         case mm_swc132_op:
1701 fpu_emul:
1702                 /* roll back jump/branch */
1703                 regs->cp0_epc = origpc;
1704                 regs->regs[31] = orig31;
1705
1706                 die_if_kernel("Unaligned FP access in kernel code", regs);
1707                 BUG_ON(!used_math());
1708                 BUG_ON(!is_fpu_owner());
1709
1710                 lose_fpu(1);    /* save the FPU state for the emulator */
1711                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1712                                                &fault_addr);
1713                 own_fpu(1);     /* restore FPU state */
1714
1715                 /* If something went wrong, signal */
1716                 process_fpemu_return(res, fault_addr, 0);
1717
1718                 if (res == 0)
1719                         goto success;
1720                 return;
1721
1722         case mm_lh32_op:
1723                 reg = insn.mm_i_format.rt;
1724                 goto loadHW;
1725
1726         case mm_lhu32_op:
1727                 reg = insn.mm_i_format.rt;
1728                 goto loadHWU;
1729
1730         case mm_lw32_op:
1731                 reg = insn.mm_i_format.rt;
1732                 goto loadW;
1733
1734         case mm_sh32_op:
1735                 reg = insn.mm_i_format.rt;
1736                 goto storeHW;
1737
1738         case mm_sw32_op:
1739                 reg = insn.mm_i_format.rt;
1740                 goto storeW;
1741
1742         case mm_ld32_op:
1743                 reg = insn.mm_i_format.rt;
1744                 goto loadDW;
1745
1746         case mm_sd32_op:
1747                 reg = insn.mm_i_format.rt;
1748                 goto storeDW;
1749
1750         case mm_pool16c_op:
1751                 switch (insn.mm16_m_format.func) {
1752                 case mm_lwm16_op:
1753                         reg = insn.mm16_m_format.rlist;
1754                         rvar = reg + 1;
1755                         if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1756                                 goto sigbus;
1757
1758                         for (i = 16; rvar; rvar--, i++) {
1759                                 LoadW(addr, value, res);
1760                                 if (res)
1761                                         goto fault;
1762                                 addr += 4;
1763                                 regs->regs[i] = value;
1764                         }
1765                         LoadW(addr, value, res);
1766                         if (res)
1767                                 goto fault;
1768                         regs->regs[31] = value;
1769
1770                         goto success;
1771
1772                 case mm_swm16_op:
1773                         reg = insn.mm16_m_format.rlist;
1774                         rvar = reg + 1;
1775                         if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1776                                 goto sigbus;
1777
1778                         for (i = 16; rvar; rvar--, i++) {
1779                                 value = regs->regs[i];
1780                                 StoreW(addr, value, res);
1781                                 if (res)
1782                                         goto fault;
1783                                 addr += 4;
1784                         }
1785                         value = regs->regs[31];
1786                         StoreW(addr, value, res);
1787                         if (res)
1788                                 goto fault;
1789
1790                         goto success;
1791
1792                 }
1793
1794                 goto sigbus;
1795
1796         case mm_lhu16_op:
1797                 reg = reg16to32[insn.mm16_rb_format.rt];
1798                 goto loadHWU;
1799
1800         case mm_lw16_op:
1801                 reg = reg16to32[insn.mm16_rb_format.rt];
1802                 goto loadW;
1803
1804         case mm_sh16_op:
1805                 reg = reg16to32st[insn.mm16_rb_format.rt];
1806                 goto storeHW;
1807
1808         case mm_sw16_op:
1809                 reg = reg16to32st[insn.mm16_rb_format.rt];
1810                 goto storeW;
1811
1812         case mm_lwsp16_op:
1813                 reg = insn.mm16_r5_format.rt;
1814                 goto loadW;
1815
1816         case mm_swsp16_op:
1817                 reg = insn.mm16_r5_format.rt;
1818                 goto storeW;
1819
1820         case mm_lwgp16_op:
1821                 reg = reg16to32[insn.mm16_r3_format.rt];
1822                 goto loadW;
1823
1824         default:
1825                 goto sigill;
1826         }
1827
1828 loadHW:
1829         if (!access_ok(VERIFY_READ, addr, 2))
1830                 goto sigbus;
1831
1832         LoadHW(addr, value, res);
1833         if (res)
1834                 goto fault;
1835         regs->regs[reg] = value;
1836         goto success;
1837
1838 loadHWU:
1839         if (!access_ok(VERIFY_READ, addr, 2))
1840                 goto sigbus;
1841
1842         LoadHWU(addr, value, res);
1843         if (res)
1844                 goto fault;
1845         regs->regs[reg] = value;
1846         goto success;
1847
1848 loadW:
1849         if (!access_ok(VERIFY_READ, addr, 4))
1850                 goto sigbus;
1851
1852         LoadW(addr, value, res);
1853         if (res)
1854                 goto fault;
1855         regs->regs[reg] = value;
1856         goto success;
1857
1858 loadWU:
1859 #ifdef CONFIG_64BIT
1860         /*
1861          * A 32-bit kernel might be running on a 64-bit processor.  But
1862          * if we're on a 32-bit processor and an i-cache incoherency
1863          * or race makes us see a 64-bit instruction here the sdl/sdr
1864          * would blow up, so for now we don't handle unaligned 64-bit
1865          * instructions on 32-bit kernels.
1866          */
1867         if (!access_ok(VERIFY_READ, addr, 4))
1868                 goto sigbus;
1869
1870         LoadWU(addr, value, res);
1871         if (res)
1872                 goto fault;
1873         regs->regs[reg] = value;
1874         goto success;
1875 #endif /* CONFIG_64BIT */
1876
1877         /* Cannot handle 64-bit instructions in 32-bit kernel */
1878         goto sigill;
1879
1880 loadDW:
1881 #ifdef CONFIG_64BIT
1882         /*
1883          * A 32-bit kernel might be running on a 64-bit processor.  But
1884          * if we're on a 32-bit processor and an i-cache incoherency
1885          * or race makes us see a 64-bit instruction here the sdl/sdr
1886          * would blow up, so for now we don't handle unaligned 64-bit
1887          * instructions on 32-bit kernels.
1888          */
1889         if (!access_ok(VERIFY_READ, addr, 8))
1890                 goto sigbus;
1891
1892         LoadDW(addr, value, res);
1893         if (res)
1894                 goto fault;
1895         regs->regs[reg] = value;
1896         goto success;
1897 #endif /* CONFIG_64BIT */
1898
1899         /* Cannot handle 64-bit instructions in 32-bit kernel */
1900         goto sigill;
1901
1902 storeHW:
1903         if (!access_ok(VERIFY_WRITE, addr, 2))
1904                 goto sigbus;
1905
1906         value = regs->regs[reg];
1907         StoreHW(addr, value, res);
1908         if (res)
1909                 goto fault;
1910         goto success;
1911
1912 storeW:
1913         if (!access_ok(VERIFY_WRITE, addr, 4))
1914                 goto sigbus;
1915
1916         value = regs->regs[reg];
1917         StoreW(addr, value, res);
1918         if (res)
1919                 goto fault;
1920         goto success;
1921
1922 storeDW:
1923 #ifdef CONFIG_64BIT
1924         /*
1925          * A 32-bit kernel might be running on a 64-bit processor.  But
1926          * if we're on a 32-bit processor and an i-cache incoherency
1927          * or race makes us see a 64-bit instruction here the sdl/sdr
1928          * would blow up, so for now we don't handle unaligned 64-bit
1929          * instructions on 32-bit kernels.
1930          */
1931         if (!access_ok(VERIFY_WRITE, addr, 8))
1932                 goto sigbus;
1933
1934         value = regs->regs[reg];
1935         StoreDW(addr, value, res);
1936         if (res)
1937                 goto fault;
1938         goto success;
1939 #endif /* CONFIG_64BIT */
1940
1941         /* Cannot handle 64-bit instructions in 32-bit kernel */
1942         goto sigill;
1943
1944 success:
1945         regs->cp0_epc = contpc; /* advance or branch */
1946
1947 #ifdef CONFIG_DEBUG_FS
1948         unaligned_instructions++;
1949 #endif
1950         return;
1951
1952 fault:
1953         /* roll back jump/branch */
1954         regs->cp0_epc = origpc;
1955         regs->regs[31] = orig31;
1956         /* Did we have an exception handler installed? */
1957         if (fixup_exception(regs))
1958                 return;
1959
1960         die_if_kernel("Unhandled kernel unaligned access", regs);
1961         force_sig(SIGSEGV, current);
1962
1963         return;
1964
1965 sigbus:
1966         die_if_kernel("Unhandled kernel unaligned access", regs);
1967         force_sig(SIGBUS, current);
1968
1969         return;
1970
1971 sigill:
1972         die_if_kernel
1973             ("Unhandled kernel unaligned access or invalid instruction", regs);
1974         force_sig(SIGILL, current);
1975 }
1976
1977 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1978 {
1979         unsigned long value;
1980         unsigned int res;
1981         int reg;
1982         unsigned long orig31;
1983         u16 __user *pc16;
1984         unsigned long origpc;
1985         union mips16e_instruction mips16inst, oldinst;
1986
1987         origpc = regs->cp0_epc;
1988         orig31 = regs->regs[31];
1989         pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1990         /*
1991          * This load never faults.
1992          */
1993         __get_user(mips16inst.full, pc16);
1994         oldinst = mips16inst;
1995
1996         /* skip EXTEND instruction */
1997         if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1998                 pc16++;
1999                 __get_user(mips16inst.full, pc16);
2000         } else if (delay_slot(regs)) {
2001                 /*  skip jump instructions */
2002                 /*  JAL/JALX are 32 bits but have OPCODE in first short int */
2003                 if (mips16inst.ri.opcode == MIPS16e_jal_op)
2004                         pc16++;
2005                 pc16++;
2006                 if (get_user(mips16inst.full, pc16))
2007                         goto sigbus;
2008         }
2009
2010         switch (mips16inst.ri.opcode) {
2011         case MIPS16e_i64_op:    /* I64 or RI64 instruction */
2012                 switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
2013                 case MIPS16e_ldpc_func:
2014                 case MIPS16e_ldsp_func:
2015                         reg = reg16to32[mips16inst.ri64.ry];
2016                         goto loadDW;
2017
2018                 case MIPS16e_sdsp_func:
2019                         reg = reg16to32[mips16inst.ri64.ry];
2020                         goto writeDW;
2021
2022                 case MIPS16e_sdrasp_func:
2023                         reg = 29;       /* GPRSP */
2024                         goto writeDW;
2025                 }
2026
2027                 goto sigbus;
2028
2029         case MIPS16e_swsp_op:
2030         case MIPS16e_lwpc_op:
2031         case MIPS16e_lwsp_op:
2032                 reg = reg16to32[mips16inst.ri.rx];
2033                 break;
2034
2035         case MIPS16e_i8_op:
2036                 if (mips16inst.i8.func != MIPS16e_swrasp_func)
2037                         goto sigbus;
2038                 reg = 29;       /* GPRSP */
2039                 break;
2040
2041         default:
2042                 reg = reg16to32[mips16inst.rri.ry];
2043                 break;
2044         }
2045
2046         switch (mips16inst.ri.opcode) {
2047
2048         case MIPS16e_lb_op:
2049         case MIPS16e_lbu_op:
2050         case MIPS16e_sb_op:
2051                 goto sigbus;
2052
2053         case MIPS16e_lh_op:
2054                 if (!access_ok(VERIFY_READ, addr, 2))
2055                         goto sigbus;
2056
2057                 LoadHW(addr, value, res);
2058                 if (res)
2059                         goto fault;
2060                 MIPS16e_compute_return_epc(regs, &oldinst);
2061                 regs->regs[reg] = value;
2062                 break;
2063
2064         case MIPS16e_lhu_op:
2065                 if (!access_ok(VERIFY_READ, addr, 2))
2066                         goto sigbus;
2067
2068                 LoadHWU(addr, value, res);
2069                 if (res)
2070                         goto fault;
2071                 MIPS16e_compute_return_epc(regs, &oldinst);
2072                 regs->regs[reg] = value;
2073                 break;
2074
2075         case MIPS16e_lw_op:
2076         case MIPS16e_lwpc_op:
2077         case MIPS16e_lwsp_op:
2078                 if (!access_ok(VERIFY_READ, addr, 4))
2079                         goto sigbus;
2080
2081                 LoadW(addr, value, res);
2082                 if (res)
2083                         goto fault;
2084                 MIPS16e_compute_return_epc(regs, &oldinst);
2085                 regs->regs[reg] = value;
2086                 break;
2087
2088         case MIPS16e_lwu_op:
2089 #ifdef CONFIG_64BIT
2090                 /*
2091                  * A 32-bit kernel might be running on a 64-bit processor.  But
2092                  * if we're on a 32-bit processor and an i-cache incoherency
2093                  * or race makes us see a 64-bit instruction here the sdl/sdr
2094                  * would blow up, so for now we don't handle unaligned 64-bit
2095                  * instructions on 32-bit kernels.
2096                  */
2097                 if (!access_ok(VERIFY_READ, addr, 4))
2098                         goto sigbus;
2099
2100                 LoadWU(addr, value, res);
2101                 if (res)
2102                         goto fault;
2103                 MIPS16e_compute_return_epc(regs, &oldinst);
2104                 regs->regs[reg] = value;
2105                 break;
2106 #endif /* CONFIG_64BIT */
2107
2108                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2109                 goto sigill;
2110
2111         case MIPS16e_ld_op:
2112 loadDW:
2113 #ifdef CONFIG_64BIT
2114                 /*
2115                  * A 32-bit kernel might be running on a 64-bit processor.  But
2116                  * if we're on a 32-bit processor and an i-cache incoherency
2117                  * or race makes us see a 64-bit instruction here the sdl/sdr
2118                  * would blow up, so for now we don't handle unaligned 64-bit
2119                  * instructions on 32-bit kernels.
2120                  */
2121                 if (!access_ok(VERIFY_READ, addr, 8))
2122                         goto sigbus;
2123
2124                 LoadDW(addr, value, res);
2125                 if (res)
2126                         goto fault;
2127                 MIPS16e_compute_return_epc(regs, &oldinst);
2128                 regs->regs[reg] = value;
2129                 break;
2130 #endif /* CONFIG_64BIT */
2131
2132                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2133                 goto sigill;
2134
2135         case MIPS16e_sh_op:
2136                 if (!access_ok(VERIFY_WRITE, addr, 2))
2137                         goto sigbus;
2138
2139                 MIPS16e_compute_return_epc(regs, &oldinst);
2140                 value = regs->regs[reg];
2141                 StoreHW(addr, value, res);
2142                 if (res)
2143                         goto fault;
2144                 break;
2145
2146         case MIPS16e_sw_op:
2147         case MIPS16e_swsp_op:
2148         case MIPS16e_i8_op:     /* actually - MIPS16e_swrasp_func */
2149                 if (!access_ok(VERIFY_WRITE, addr, 4))
2150                         goto sigbus;
2151
2152                 MIPS16e_compute_return_epc(regs, &oldinst);
2153                 value = regs->regs[reg];
2154                 StoreW(addr, value, res);
2155                 if (res)
2156                         goto fault;
2157                 break;
2158
2159         case MIPS16e_sd_op:
2160 writeDW:
2161 #ifdef CONFIG_64BIT
2162                 /*
2163                  * A 32-bit kernel might be running on a 64-bit processor.  But
2164                  * if we're on a 32-bit processor and an i-cache incoherency
2165                  * or race makes us see a 64-bit instruction here the sdl/sdr
2166                  * would blow up, so for now we don't handle unaligned 64-bit
2167                  * instructions on 32-bit kernels.
2168                  */
2169                 if (!access_ok(VERIFY_WRITE, addr, 8))
2170                         goto sigbus;
2171
2172                 MIPS16e_compute_return_epc(regs, &oldinst);
2173                 value = regs->regs[reg];
2174                 StoreDW(addr, value, res);
2175                 if (res)
2176                         goto fault;
2177                 break;
2178 #endif /* CONFIG_64BIT */
2179
2180                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2181                 goto sigill;
2182
2183         default:
2184                 /*
2185                  * Pheeee...  We encountered an yet unknown instruction or
2186                  * cache coherence problem.  Die sucker, die ...
2187                  */
2188                 goto sigill;
2189         }
2190
2191 #ifdef CONFIG_DEBUG_FS
2192         unaligned_instructions++;
2193 #endif
2194
2195         return;
2196
2197 fault:
2198         /* roll back jump/branch */
2199         regs->cp0_epc = origpc;
2200         regs->regs[31] = orig31;
2201         /* Did we have an exception handler installed? */
2202         if (fixup_exception(regs))
2203                 return;
2204
2205         die_if_kernel("Unhandled kernel unaligned access", regs);
2206         force_sig(SIGSEGV, current);
2207
2208         return;
2209
2210 sigbus:
2211         die_if_kernel("Unhandled kernel unaligned access", regs);
2212         force_sig(SIGBUS, current);
2213
2214         return;
2215
2216 sigill:
2217         die_if_kernel
2218             ("Unhandled kernel unaligned access or invalid instruction", regs);
2219         force_sig(SIGILL, current);
2220 }
2221
2222 asmlinkage void do_ade(struct pt_regs *regs)
2223 {
2224         enum ctx_state prev_state;
2225         unsigned int __user *pc;
2226         mm_segment_t seg;
2227
2228         prev_state = exception_enter();
2229         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2230                         1, regs, regs->cp0_badvaddr);
2231         /*
2232          * Did we catch a fault trying to load an instruction?
2233          */
2234         if (regs->cp0_badvaddr == regs->cp0_epc)
2235                 goto sigbus;
2236
2237         if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2238                 goto sigbus;
2239         if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2240                 goto sigbus;
2241
2242         /*
2243          * Do branch emulation only if we didn't forward the exception.
2244          * This is all so but ugly ...
2245          */
2246
2247         /*
2248          * Are we running in microMIPS mode?
2249          */
2250         if (get_isa16_mode(regs->cp0_epc)) {
2251                 /*
2252                  * Did we catch a fault trying to load an instruction in
2253                  * 16-bit mode?
2254                  */
2255                 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2256                         goto sigbus;
2257                 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2258                         show_registers(regs);
2259
2260                 if (cpu_has_mmips) {
2261                         seg = get_fs();
2262                         if (!user_mode(regs))
2263                                 set_fs(KERNEL_DS);
2264                         emulate_load_store_microMIPS(regs,
2265                                 (void __user *)regs->cp0_badvaddr);
2266                         set_fs(seg);
2267
2268                         return;
2269                 }
2270
2271                 if (cpu_has_mips16) {
2272                         seg = get_fs();
2273                         if (!user_mode(regs))
2274                                 set_fs(KERNEL_DS);
2275                         emulate_load_store_MIPS16e(regs,
2276                                 (void __user *)regs->cp0_badvaddr);
2277                         set_fs(seg);
2278
2279                         return;
2280         }
2281
2282                 goto sigbus;
2283         }
2284
2285         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2286                 show_registers(regs);
2287         pc = (unsigned int __user *)exception_epc(regs);
2288
2289         seg = get_fs();
2290         if (!user_mode(regs))
2291                 set_fs(KERNEL_DS);
2292         emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2293         set_fs(seg);
2294
2295         return;
2296
2297 sigbus:
2298         die_if_kernel("Kernel unaligned instruction access", regs);
2299         force_sig(SIGBUS, current);
2300
2301         /*
2302          * XXX On return from the signal handler we should advance the epc
2303          */
2304         exception_exit(prev_state);
2305 }
2306
2307 #ifdef CONFIG_DEBUG_FS
2308 static int __init debugfs_unaligned(void)
2309 {
2310         struct dentry *d;
2311
2312         if (!mips_debugfs_dir)
2313                 return -ENODEV;
2314         d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2315                                mips_debugfs_dir, &unaligned_instructions);
2316         if (!d)
2317                 return -ENOMEM;
2318         d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2319                                mips_debugfs_dir, &unaligned_action);
2320         if (!d)
2321                 return -ENOMEM;
2322         return 0;
2323 }
2324 arch_initcall(debugfs_unaligned);
2325 #endif