From 52f993b8e89487ec9ee15a7fb4979e0f09a45b27 Mon Sep 17 00:00:00 2001 From: Yunhong Jiang Date: Wed, 8 Mar 2017 23:13:28 -0800 Subject: [PATCH] Upgrade to 4.4.50-rt62 The current kernel is based on rt kernel v4.4.6-rt14. We will upgrade it to 4.4.50-rt62. The command to achieve it is: a) Clone a git repo from git://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-stable-rt.git b) Get the diff between this two changesets: git diff 640eca2901f3435e616157b11379d3223a44b391 705619beeea1b0b48219a683fd1a901a86fdaf5e where the two commits are: [yjiang5@jnakajim-build linux-stable-rt]$ git show --oneline --name-only 640eca2901f3435e616157b11379d3223a44b391 640eca2901f3 v4.4.6-rt14 localversion-rt [yjiang5@jnakajim-build linux-stable-rt]$ git show --oneline --name-only 705619beeea1b0b48219a683fd1a901a86fdaf5e 705619beeea1 Linux 4.4.50-rt62 localversion-rt c) One patch has been backported thus revert the patch before applying. filterdiff -p1 -x scripts/package/Makefile ~/tmp/v4.4.6-rt14-4.4.50-rt62.diff |patch -p1 --dry-run Upstream status: backport Change-Id: I244d57a32f6066e5a5b9915f9fbf99e7bbca6e01 Signed-off-by: Yunhong Jiang --- .../ABI/testing/sysfs-bus-iio-proximity-as3935 | 2 +- .../devicetree/bindings/ata/ahci-platform.txt | 4 + .../devicetree/bindings/clock/imx31-clock.txt | 2 +- .../devicetree/bindings/clock/imx35-clock.txt | 1 + .../bindings/iio/adc/rockchip-saradc.txt | 7 + .../bindings/pinctrl/img,pistachio-pinctrl.txt | 12 +- .../bindings/regulator/qcom,spmi-regulator.txt | 6 +- kernel/Documentation/filesystems/proc.txt | 9 +- kernel/Documentation/kernel-parameters.txt | 11 +- kernel/Documentation/mic/mpssd/mpssd.c | 4 +- kernel/Documentation/module-signing.txt | 6 + kernel/Documentation/pinctrl.txt | 2 +- kernel/Documentation/scsi/scsi_eh.txt | 8 +- kernel/Documentation/serial/tty.txt | 3 - kernel/Documentation/sysctl/fs.txt | 23 + kernel/Documentation/virtual/kvm/api.txt | 1 + kernel/Documentation/x86/pat.txt | 32 ++ kernel/MAINTAINERS | 100 ++--- kernel/Makefile | 38 +- kernel/arch/alpha/include/asm/uaccess.h | 19 +- kernel/arch/arc/Kconfig | 2 +- kernel/arch/arc/Makefile | 16 +- kernel/arch/arc/include/asm/arcregs.h | 6 - kernel/arch/arc/include/asm/bitops.h | 15 - kernel/arch/arc/include/asm/cacheflush.h | 6 +- kernel/arch/arc/include/asm/delay.h | 11 +- kernel/arch/arc/include/asm/entry.h | 4 +- kernel/arch/arc/include/asm/io.h | 45 +- kernel/arch/arc/include/asm/irqflags-compact.h | 2 +- kernel/arch/arc/include/asm/pgtable.h | 5 +- kernel/arch/arc/include/asm/uaccess.h | 11 +- kernel/arch/arc/kernel/setup.c | 4 - kernel/arch/arc/kernel/signal.c | 8 +- kernel/arch/arc/kernel/stacktrace.c | 2 +- kernel/arch/arc/kernel/time.c | 19 +- kernel/arch/arc/kernel/unaligned.c | 3 +- kernel/arch/arc/mm/cache.c | 22 +- kernel/arch/arm/boot/compressed/head.S | 2 +- kernel/arch/arm/boot/dts/am43x-epos-evm.dts | 5 + kernel/arch/arm/boot/dts/armada-375.dtsi | 2 +- kernel/arch/arm/boot/dts/armada-385-linksys.dtsi | 12 +- kernel/arch/arm/boot/dts/armada-390.dtsi | 3 + .../arch/arm/boot/dts/armada-xp-linksys-mamba.dts | 4 +- kernel/arch/arm/boot/dts/at91-sama5d3_xplained.dts | 1 + kernel/arch/arm/boot/dts/at91-sama5d4_xplained.dts | 1 + kernel/arch/arm/boot/dts/at91sam9x5.dtsi | 2 +- kernel/arch/arm/boot/dts/da850-evm.dts | 1 + kernel/arch/arm/boot/dts/exynos4210-trats.dts | 2 + kernel/arch/arm/boot/dts/imx31.dtsi | 18 +- .../arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi | 4 +- kernel/arch/arm/boot/dts/imx6qdl.dtsi | 2 +- kernel/arch/arm/boot/dts/kirkwood-ib62x0.dts | 2 +- kernel/arch/arm/boot/dts/omap3-overo-base.dtsi | 4 +- .../boot/dts/omap3-overo-chestnut43-common.dtsi | 2 - .../arch/arm/boot/dts/omap3-overo-tobi-common.dtsi | 2 - .../arm/boot/dts/omap3-overo-tobiduo-common.dtsi | 3 - kernel/arch/arm/boot/dts/pxa3xx.dtsi | 2 +- kernel/arch/arm/boot/dts/qcom-apq8064.dtsi | 76 +++- kernel/arch/arm/boot/dts/r8a7794.dtsi | 2 +- kernel/arch/arm/boot/dts/sama5d2-pinfunc.h | 4 +- kernel/arch/arm/boot/dts/stih407-family.dtsi | 10 +- kernel/arch/arm/boot/dts/stih410.dtsi | 12 +- kernel/arch/arm/boot/dts/sun4i-a10-a1000.dts | 1 + kernel/arch/arm/boot/dts/sun4i-a10-hackberry.dts | 1 + kernel/arch/arm/boot/dts/sun4i-a10-jesurun-q5.dts | 1 + kernel/arch/arm/boot/dts/sun5i-a10s-wobo-i5.dts | 1 + kernel/arch/arm/boot/dts/sun5i-a13.dtsi | 2 +- kernel/arch/arm/boot/dts/sun5i-r8-chip.dts | 2 +- kernel/arch/arm/common/sa1111.c | 22 +- kernel/arch/arm/crypto/aes-ce-glue.c | 7 +- kernel/arch/arm/crypto/ghash-ce-glue.c | 24 ++ kernel/arch/arm/include/asm/cputype.h | 3 + kernel/arch/arm/include/asm/dma-mapping.h | 2 +- kernel/arch/arm/include/asm/floppy.h | 2 +- kernel/arch/arm/include/asm/pgtable-2level.h | 1 + kernel/arch/arm/include/asm/pgtable-3level.h | 5 +- kernel/arch/arm/include/asm/pgtable.h | 1 - kernel/arch/arm/kernel/devtree.c | 14 +- kernel/arch/arm/kernel/entry-armv.S | 6 +- kernel/arch/arm/kernel/hw_breakpoint.c | 16 + kernel/arch/arm/kernel/patch.c | 6 +- kernel/arch/arm/kernel/ptrace.c | 4 +- kernel/arch/arm/kernel/smp_tlb.c | 7 + kernel/arch/arm/kernel/sys_oabi-compat.c | 8 +- kernel/arch/arm/kvm/arm.c | 2 - kernel/arch/arm/kvm/mmu.c | 18 +- kernel/arch/arm/mach-cns3xxx/pcie.c | 6 +- kernel/arch/arm/mach-davinci/da850.c | 12 +- kernel/arch/arm/mach-exynos/Kconfig | 1 + kernel/arch/arm/mach-exynos/pm_domains.c | 2 +- kernel/arch/arm/mach-imx/mach-imx6ul.c | 2 +- kernel/arch/arm/mach-imx/pm-imx6.c | 4 +- kernel/arch/arm/mach-mvebu/coherency.c | 22 +- kernel/arch/arm/mach-omap2/cpuidle34xx.c | 69 ++- kernel/arch/arm/mach-omap2/io.c | 1 + kernel/arch/arm/mach-omap2/omap-mpuss-lowpower.c | 5 +- kernel/arch/arm/mach-omap2/omap_hwmod.c | 8 +- .../mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c | 1 + kernel/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c | 12 + kernel/arch/arm/mach-omap2/prcm43xx.h | 1 + kernel/arch/arm/mach-prima2/Kconfig | 1 + kernel/arch/arm/mach-pxa/idp.c | 3 +- kernel/arch/arm/mach-pxa/pxa_cplds_irqs.c | 24 +- kernel/arch/arm/mach-pxa/xcep.c | 3 +- kernel/arch/arm/mach-realview/core.c | 3 +- kernel/arch/arm/mach-s3c64xx/dev-audio.c | 41 +- kernel/arch/arm/mach-s3c64xx/include/mach/dma.h | 52 +-- kernel/arch/arm/mach-sa1100/clock.c | 5 +- kernel/arch/arm/mach-sa1100/generic.c | 4 + kernel/arch/arm/mach-sa1100/generic.h | 2 + kernel/arch/arm/mach-sa1100/pleb.c | 2 +- .../arm/mach-shmobile/regulator-quirk-rcar-gen2.c | 62 ++- kernel/arch/arm/mach-socfpga/headsmp.S | 1 + kernel/arch/arm/mach-ux500/pm.c | 4 +- kernel/arch/arm/mach-zynq/common.c | 2 +- kernel/arch/arm/mm/fault.c | 4 +- kernel/arch/arm/mm/fault.h | 4 + kernel/arch/arm/plat-samsung/devs.c | 11 +- kernel/arch/arm/xen/enlighten.c | 3 +- kernel/arch/arm64/Kconfig | 20 + kernel/arch/arm64/boot/dts/rockchip/rk3368.dtsi | 4 +- kernel/arch/arm64/crypto/aes-ce-ccm-core.S | 53 +-- kernel/arch/arm64/crypto/aes-ce-cipher.c | 25 +- kernel/arch/arm64/crypto/aes-ce.S | 1 + kernel/arch/arm64/crypto/aes-glue.c | 2 +- kernel/arch/arm64/crypto/aes-modes.S | 91 ++-- kernel/arch/arm64/crypto/aes-neon.S | 25 +- kernel/arch/arm64/crypto/ghash-ce-core.S | 6 +- kernel/arch/arm64/crypto/sha1-ce-core.S | 4 +- kernel/arch/arm64/crypto/sha2-ce-core.S | 4 +- kernel/arch/arm64/include/asm/arch_gicv3.h | 1 + kernel/arch/arm64/include/asm/cpufeature.h | 5 +- kernel/arch/arm64/include/asm/elf.h | 5 +- kernel/arch/arm64/include/asm/futex.h | 2 + kernel/arch/arm64/include/asm/kvm_arm.h | 2 - kernel/arch/arm64/include/asm/opcodes.h | 4 + kernel/arch/arm64/include/asm/percpu.h | 120 +++--- kernel/arch/arm64/include/asm/pgtable-hwdef.h | 1 - kernel/arch/arm64/include/asm/pgtable.h | 47 ++- kernel/arch/arm64/include/asm/processor.h | 2 +- kernel/arch/arm64/include/asm/ptrace.h | 35 +- kernel/arch/arm64/include/asm/spinlock.h | 10 + kernel/arch/arm64/include/uapi/asm/auxvec.h | 2 + kernel/arch/arm64/include/uapi/asm/ptrace.h | 1 + kernel/arch/arm64/kernel/asm-offsets.c | 1 + kernel/arch/arm64/kernel/cpu_errata.c | 9 + kernel/arch/arm64/kernel/cpufeature.c | 10 +- kernel/arch/arm64/kernel/cpuinfo.c | 11 +- kernel/arch/arm64/kernel/debug-monitors.c | 7 +- kernel/arch/arm64/kernel/entry.S | 17 +- kernel/arch/arm64/kernel/head.S | 3 +- kernel/arch/arm64/kernel/ptrace.c | 97 ++++- kernel/arch/arm64/kernel/signal.c | 4 +- kernel/arch/arm64/kernel/signal32.c | 2 +- kernel/arch/arm64/kernel/smp.c | 3 +- kernel/arch/arm64/kernel/suspend.c | 9 + kernel/arch/arm64/kernel/traps.c | 28 +- kernel/arch/arm64/kvm/hyp-init.S | 13 +- kernel/arch/arm64/kvm/inject_fault.c | 2 +- kernel/arch/arm64/mm/fault.c | 62 ++- kernel/arch/arm64/mm/mmu.c | 8 +- kernel/arch/arm64/mm/proc.S | 14 + kernel/arch/avr32/include/asm/uaccess.h | 11 +- kernel/arch/avr32/kernel/avr32_ksyms.c | 2 +- kernel/arch/avr32/lib/copy_user.S | 8 +- kernel/arch/avr32/mach-at32ap/pio.c | 2 +- kernel/arch/blackfin/include/asm/uaccess.h | 9 +- kernel/arch/blackfin/mach-bf561/boards/cm_bf561.c | 3 +- kernel/arch/blackfin/mach-bf561/boards/ezkit.c | 3 +- kernel/arch/cris/boot/rescue/Makefile | 8 + kernel/arch/cris/include/asm/uaccess.h | 71 ++-- kernel/arch/frv/include/asm/uaccess.h | 12 +- kernel/arch/h8300/include/asm/thread_info.h | 4 - kernel/arch/h8300/kernel/signal.c | 2 +- kernel/arch/hexagon/include/asm/uaccess.h | 3 +- kernel/arch/ia64/include/asm/io.h | 1 + kernel/arch/ia64/include/asm/uaccess.h | 20 +- kernel/arch/m32r/include/asm/uaccess.h | 2 +- kernel/arch/m68k/include/asm/delay.h | 2 +- kernel/arch/metag/include/asm/atomic.h | 3 +- kernel/arch/metag/include/asm/atomic_lnkget.h | 2 +- kernel/arch/metag/include/asm/cmpxchg_lnkget.h | 2 +- kernel/arch/metag/include/asm/uaccess.h | 3 +- kernel/arch/microblaze/include/asm/uaccess.h | 11 +- kernel/arch/mips/Kconfig.debug | 36 -- kernel/arch/mips/Makefile | 4 - kernel/arch/mips/alchemy/devboards/db1000.c | 18 +- kernel/arch/mips/alchemy/devboards/db1550.c | 4 +- kernel/arch/mips/ath79/early_printk.c | 6 +- kernel/arch/mips/include/asm/asmmacro.h | 41 +- kernel/arch/mips/include/asm/cacheflush.h | 6 - kernel/arch/mips/include/asm/kvm_host.h | 10 +- .../include/asm/mach-paravirt/kernel-entry-init.h | 2 + kernel/arch/mips/include/asm/msa.h | 13 + kernel/arch/mips/include/asm/pgtable.h | 26 +- kernel/arch/mips/include/asm/processor.h | 2 +- kernel/arch/mips/include/asm/ptrace.h | 2 +- kernel/arch/mips/include/asm/switch_to.h | 2 +- kernel/arch/mips/include/asm/uaccess.h | 3 + kernel/arch/mips/include/asm/uprobes.h | 1 - kernel/arch/mips/include/asm/watch.h | 10 +- kernel/arch/mips/include/uapi/asm/siginfo.h | 22 +- kernel/arch/mips/kernel/csrc-r4k.c | 4 +- kernel/arch/mips/kernel/mips-r2-to-r6-emul.c | 107 ++--- kernel/arch/mips/kernel/pm.c | 2 +- kernel/arch/mips/kernel/process.c | 14 +- kernel/arch/mips/kernel/ptrace.c | 27 +- kernel/arch/mips/kernel/r4k_fpu.S | 10 +- kernel/arch/mips/kernel/scall64-n32.S | 2 +- kernel/arch/mips/kernel/scall64-o32.S | 2 +- kernel/arch/mips/kernel/setup.c | 3 + kernel/arch/mips/kernel/signal.c | 15 +- kernel/arch/mips/kernel/smp.c | 7 +- kernel/arch/mips/kernel/traps.c | 8 +- kernel/arch/mips/kernel/unaligned.c | 51 ++- kernel/arch/mips/kernel/uprobes.c | 25 +- kernel/arch/mips/kernel/vdso.c | 8 +- kernel/arch/mips/kernel/watch.c | 5 +- kernel/arch/mips/kvm/emulate.c | 224 ++++++---- kernel/arch/mips/kvm/interrupt.h | 1 + kernel/arch/mips/kvm/locore.S | 1 + kernel/arch/mips/kvm/mips.c | 15 +- kernel/arch/mips/kvm/tlb.c | 70 ++-- kernel/arch/mips/kvm/trap_emul.c | 2 +- kernel/arch/mips/lib/ashldi3.c | 2 +- kernel/arch/mips/lib/ashrdi3.c | 2 +- kernel/arch/mips/lib/bswapdi.c | 2 +- kernel/arch/mips/lib/bswapsi.c | 2 +- kernel/arch/mips/lib/cmpdi2.c | 2 +- kernel/arch/mips/lib/lshrdi3.c | 2 +- kernel/arch/mips/lib/ucmpdi2.c | 2 +- kernel/arch/mips/loongson64/loongson-3/hpet.c | 14 +- kernel/arch/mips/loongson64/loongson-3/numa.c | 6 +- kernel/arch/mips/math-emu/cp1emu.c | 8 +- kernel/arch/mips/mm/cache.c | 41 +- kernel/arch/mips/mm/uasm-mips.c | 2 +- kernel/arch/mips/mti-malta/malta-setup.c | 8 +- kernel/arch/mips/vdso/Makefile | 6 +- kernel/arch/mn10300/include/asm/uaccess.h | 1 + kernel/arch/mn10300/lib/usercopy.c | 4 +- kernel/arch/nios2/include/asm/uaccess.h | 13 +- kernel/arch/openrisc/include/asm/uaccess.h | 35 +- kernel/arch/parisc/include/asm/bitops.h | 8 +- kernel/arch/parisc/include/asm/pgtable.h | 14 +- kernel/arch/parisc/include/asm/uaccess.h | 7 +- kernel/arch/parisc/include/uapi/asm/bitsperlong.h | 2 - kernel/arch/parisc/include/uapi/asm/errno.h | 4 +- kernel/arch/parisc/include/uapi/asm/swab.h | 5 +- kernel/arch/parisc/kernel/asm-offsets.c | 1 + kernel/arch/parisc/kernel/cache.c | 40 +- kernel/arch/parisc/kernel/pacache.S | 49 +-- kernel/arch/parisc/kernel/parisc_ksyms.c | 10 +- kernel/arch/parisc/kernel/pci-dma.c | 2 +- kernel/arch/parisc/kernel/setup.c | 12 + kernel/arch/parisc/kernel/syscall.S | 11 +- kernel/arch/parisc/kernel/traps.c | 3 + kernel/arch/parisc/kernel/unaligned.c | 10 +- kernel/arch/parisc/kernel/vmlinux.lds.S | 7 +- kernel/arch/parisc/lib/fixup.S | 6 + kernel/arch/parisc/mm/fault.c | 1 + kernel/arch/powerpc/boot/ps3-head.S | 5 - kernel/arch/powerpc/boot/ps3.c | 8 +- kernel/arch/powerpc/include/asm/icswx.h | 1 + kernel/arch/powerpc/include/asm/kvm_host.h | 1 + kernel/arch/powerpc/include/asm/reg.h | 7 +- kernel/arch/powerpc/include/asm/uaccess.h | 21 +- kernel/arch/powerpc/include/asm/word-at-a-time.h | 2 +- kernel/arch/powerpc/include/uapi/asm/cputable.h | 1 + kernel/arch/powerpc/include/uapi/asm/kvm.h | 1 + kernel/arch/powerpc/kernel/asm-offsets.c | 1 + kernel/arch/powerpc/kernel/eeh.c | 4 +- kernel/arch/powerpc/kernel/eeh_driver.c | 40 +- kernel/arch/powerpc/kernel/entry_64.S | 12 +- kernel/arch/powerpc/kernel/exceptions-64s.S | 16 +- kernel/arch/powerpc/kernel/ibmebus.c | 16 +- kernel/arch/powerpc/kernel/idle_power7.S | 2 +- kernel/arch/powerpc/kernel/misc_32.S | 2 +- kernel/arch/powerpc/kernel/nvram_64.c | 6 +- kernel/arch/powerpc/kernel/process.c | 28 +- kernel/arch/powerpc/kernel/prom.c | 26 +- kernel/arch/powerpc/kernel/prom_init.c | 15 +- kernel/arch/powerpc/kernel/ptrace.c | 4 +- kernel/arch/powerpc/kernel/tm.S | 61 ++- kernel/arch/powerpc/kernel/vdso64/datapage.S | 2 +- kernel/arch/powerpc/kernel/vdso64/gettimeofday.S | 2 +- kernel/arch/powerpc/kvm/book3s_emulate.c | 2 + kernel/arch/powerpc/kvm/book3s_hv.c | 6 + kernel/arch/powerpc/kvm/book3s_hv_rm_mmu.c | 2 + kernel/arch/powerpc/kvm/book3s_hv_rmhandlers.S | 466 +++++++++++---------- kernel/arch/powerpc/kvm/booke.c | 2 +- kernel/arch/powerpc/lib/copyuser_64.S | 2 +- kernel/arch/powerpc/mm/copro_fault.c | 2 + kernel/arch/powerpc/mm/hugetlbpage.c | 4 +- kernel/arch/powerpc/mm/slb_low.S | 7 +- .../arch/powerpc/platforms/powernv/eeh-powernv.c | 9 +- kernel/arch/powerpc/platforms/powernv/opal-dump.c | 7 +- kernel/arch/powerpc/platforms/powernv/opal-elog.c | 7 +- kernel/arch/powerpc/platforms/powernv/pci.c | 4 +- .../arch/powerpc/platforms/pseries/eeh_pseries.c | 51 ++- kernel/arch/powerpc/platforms/pseries/iommu.c | 24 +- kernel/arch/powerpc/platforms/pseries/lpar.c | 4 +- kernel/arch/s390/crypto/prng.c | 8 +- kernel/arch/s390/hypfs/hypfs_diag.c | 6 +- kernel/arch/s390/include/asm/fpu/api.h | 2 +- kernel/arch/s390/include/asm/mmu.h | 2 +- kernel/arch/s390/include/asm/mmu_context.h | 28 +- kernel/arch/s390/include/asm/pci.h | 3 +- kernel/arch/s390/include/asm/pci_dma.h | 2 + kernel/arch/s390/include/asm/pgalloc.h | 4 +- kernel/arch/s390/include/asm/processor.h | 2 +- kernel/arch/s390/include/asm/tlbflush.h | 12 +- kernel/arch/s390/include/asm/uaccess.h | 8 +- kernel/arch/s390/kernel/entry.S | 106 +---- kernel/arch/s390/kernel/head64.S | 2 +- kernel/arch/s390/kernel/ipl.c | 7 - kernel/arch/s390/kernel/ptrace.c | 8 + kernel/arch/s390/kernel/setup.c | 1 + kernel/arch/s390/mm/init.c | 3 +- kernel/arch/s390/mm/mmap.c | 6 +- kernel/arch/s390/mm/pgtable.c | 89 ++-- kernel/arch/s390/net/bpf_jit.h | 4 +- kernel/arch/s390/net/bpf_jit_comp.c | 4 +- kernel/arch/s390/pci/pci.c | 8 +- kernel/arch/s390/pci/pci_dma.c | 19 +- kernel/arch/score/include/asm/uaccess.h | 46 +- kernel/arch/sh/include/asm/uaccess.h | 5 +- kernel/arch/sh/include/asm/uaccess_64.h | 1 + kernel/arch/sh/mm/kmap.c | 2 + kernel/arch/sparc/include/asm/head_64.h | 4 + kernel/arch/sparc/include/asm/mmu_64.h | 3 +- kernel/arch/sparc/include/asm/pgtable_64.h | 43 +- kernel/arch/sparc/include/asm/tlbflush_64.h | 3 +- kernel/arch/sparc/include/asm/ttable.h | 8 +- kernel/arch/sparc/include/asm/uaccess_32.h | 4 +- kernel/arch/sparc/include/asm/uaccess_64.h | 66 +-- kernel/arch/sparc/kernel/Makefile | 1 + kernel/arch/sparc/kernel/cherrs.S | 14 +- kernel/arch/sparc/kernel/dtlb_prot.S | 4 +- kernel/arch/sparc/kernel/entry.S | 17 + kernel/arch/sparc/kernel/fpu_traps.S | 11 +- kernel/arch/sparc/kernel/head_64.S | 60 +-- kernel/arch/sparc/kernel/jump_label.c | 23 +- kernel/arch/sparc/kernel/ktlb.S | 12 + kernel/arch/sparc/kernel/misctrap.S | 12 +- kernel/arch/sparc/kernel/pci.c | 17 + kernel/arch/sparc/kernel/rtrap_64.S | 57 +-- kernel/arch/sparc/kernel/signal32.c | 46 +- kernel/arch/sparc/kernel/signal_32.c | 41 +- kernel/arch/sparc/kernel/signal_64.c | 33 +- kernel/arch/sparc/kernel/sigutil_32.c | 9 +- kernel/arch/sparc/kernel/sigutil_64.c | 10 +- kernel/arch/sparc/kernel/sparc_ksyms_64.c | 1 - kernel/arch/sparc/kernel/spiterrs.S | 18 +- kernel/arch/sparc/kernel/syscalls.S | 36 ++ kernel/arch/sparc/kernel/tsb.S | 12 +- kernel/arch/sparc/kernel/urtt_fill.S | 98 +++++ kernel/arch/sparc/kernel/utrap.S | 3 +- kernel/arch/sparc/kernel/vmlinux.lds.S | 4 + kernel/arch/sparc/kernel/winfixup.S | 3 +- kernel/arch/sparc/lib/GENcopy_from_user.S | 4 +- kernel/arch/sparc/lib/GENcopy_to_user.S | 4 +- kernel/arch/sparc/lib/GENmemcpy.S | 48 ++- kernel/arch/sparc/lib/Makefile | 2 +- kernel/arch/sparc/lib/NG2copy_from_user.S | 8 +- kernel/arch/sparc/lib/NG2copy_to_user.S | 8 +- kernel/arch/sparc/lib/NG2memcpy.S | 228 ++++++---- kernel/arch/sparc/lib/NG4copy_from_user.S | 8 +- kernel/arch/sparc/lib/NG4copy_to_user.S | 8 +- kernel/arch/sparc/lib/NG4memcpy.S | 294 +++++++++---- kernel/arch/sparc/lib/NGcopy_from_user.S | 4 +- kernel/arch/sparc/lib/NGcopy_to_user.S | 4 +- kernel/arch/sparc/lib/NGmemcpy.S | 233 +++++++---- kernel/arch/sparc/lib/U1copy_from_user.S | 8 +- kernel/arch/sparc/lib/U1copy_to_user.S | 8 +- kernel/arch/sparc/lib/U1memcpy.S | 345 ++++++++++----- kernel/arch/sparc/lib/U3copy_from_user.S | 8 +- kernel/arch/sparc/lib/U3copy_to_user.S | 8 +- kernel/arch/sparc/lib/U3memcpy.S | 227 ++++++---- kernel/arch/sparc/lib/copy_in_user.S | 35 +- kernel/arch/sparc/lib/user_fixup.c | 71 ---- kernel/arch/sparc/mm/fault_64.c | 6 +- kernel/arch/sparc/mm/hugetlbpage.c | 37 +- kernel/arch/sparc/mm/init_64.c | 111 +++-- kernel/arch/sparc/mm/tlb.c | 29 +- kernel/arch/sparc/mm/tsb.c | 63 ++- kernel/arch/sparc/mm/ultra.S | 374 +++++++++++++---- kernel/arch/tile/include/asm/elf.h | 1 + kernel/arch/tile/include/uapi/asm/auxvec.h | 2 + kernel/arch/tile/kernel/ptrace.c | 2 +- kernel/arch/tile/kernel/time.c | 4 +- kernel/arch/um/drivers/mconsole_kern.c | 2 +- kernel/arch/um/include/asm/common.lds.S | 2 +- kernel/arch/x86/Kconfig | 27 +- kernel/arch/x86/boot/Makefile | 3 + kernel/arch/x86/boot/compressed/Makefile | 14 +- kernel/arch/x86/boot/compressed/head_32.S | 28 ++ kernel/arch/x86/boot/compressed/head_64.S | 8 + kernel/arch/x86/configs/tiny.config | 2 + kernel/arch/x86/crypto/sha-mb/sha1_mb.c | 4 +- kernel/arch/x86/entry/common.c | 23 +- kernel/arch/x86/entry/entry_32.S | 4 +- kernel/arch/x86/entry/syscalls/syscall_32.tbl | 2 +- kernel/arch/x86/include/asm/apic.h | 2 +- kernel/arch/x86/include/asm/hugetlb.h | 1 + kernel/arch/x86/include/asm/hw_irq.h | 1 + kernel/arch/x86/include/asm/kvm_host.h | 2 +- kernel/arch/x86/include/asm/microcode.h | 26 ++ kernel/arch/x86/include/asm/mtrr.h | 6 +- kernel/arch/x86/include/asm/pat.h | 2 +- kernel/arch/x86/include/asm/pci_x86.h | 2 + kernel/arch/x86/include/asm/perf_event.h | 1 + kernel/arch/x86/include/asm/preempt.h | 17 +- kernel/arch/x86/include/asm/pvclock.h | 2 + kernel/arch/x86/include/asm/tlbflush.h | 9 +- kernel/arch/x86/include/asm/uaccess.h | 10 +- kernel/arch/x86/include/asm/xen/hypervisor.h | 2 + kernel/arch/x86/kernel/acpi/boot.c | 2 + kernel/arch/x86/kernel/amd_nb.c | 4 +- kernel/arch/x86/kernel/apic/apic.c | 3 + kernel/arch/x86/kernel/apic/io_apic.c | 2 + kernel/arch/x86/kernel/apic/vector.c | 114 ++++- kernel/arch/x86/kernel/cpu/amd.c | 18 +- kernel/arch/x86/kernel/cpu/common.c | 25 +- kernel/arch/x86/kernel/cpu/mcheck/mce-genpool.c | 4 +- kernel/arch/x86/kernel/cpu/mcheck/mce.c | 2 +- kernel/arch/x86/kernel/cpu/mcheck/therm_throt.c | 3 + kernel/arch/x86/kernel/cpu/microcode/intel.c | 38 +- kernel/arch/x86/kernel/cpu/mshyperv.c | 12 + kernel/arch/x86/kernel/cpu/mtrr/generic.c | 24 +- kernel/arch/x86/kernel/cpu/mtrr/main.c | 13 +- kernel/arch/x86/kernel/cpu/mtrr/mtrr.h | 1 + kernel/arch/x86/kernel/cpu/perf_event.c | 15 +- kernel/arch/x86/kernel/cpu/perf_event.h | 3 + kernel/arch/x86/kernel/cpu/perf_event_intel.c | 31 +- kernel/arch/x86/kernel/cpu/perf_event_intel_cqm.c | 56 ++- kernel/arch/x86/kernel/cpu/perf_event_intel_ds.c | 33 +- kernel/arch/x86/kernel/cpu/perf_event_intel_pt.c | 2 + kernel/arch/x86/kernel/cpu/perf_event_intel_rapl.c | 20 +- kernel/arch/x86/kernel/cpu/perf_event_knc.c | 4 +- kernel/arch/x86/kernel/e820.c | 12 +- kernel/arch/x86/kernel/early-quirks.c | 116 ++++- kernel/arch/x86/kernel/head_32.S | 2 +- kernel/arch/x86/kernel/hpet.c | 1 + kernel/arch/x86/kernel/ioport.c | 12 +- kernel/arch/x86/kernel/kprobes/core.c | 12 + kernel/arch/x86/kernel/mcount_64.S | 3 +- kernel/arch/x86/kernel/nmi.c | 16 +- kernel/arch/x86/kernel/paravirt.c | 4 +- kernel/arch/x86/kernel/process_64.c | 12 + kernel/arch/x86/kernel/ptrace.c | 4 +- kernel/arch/x86/kernel/pvclock.c | 4 + kernel/arch/x86/kernel/reboot.c | 20 + kernel/arch/x86/kernel/sysfb_efi.c | 14 +- kernel/arch/x86/kernel/traps.c | 20 +- kernel/arch/x86/kernel/tsc_msr.c | 2 +- kernel/arch/x86/kernel/uprobes.c | 22 +- kernel/arch/x86/kvm/cpuid.c | 1 + kernel/arch/x86/kvm/emulate.c | 287 ++++++++++--- kernel/arch/x86/kvm/i8254.c | 12 +- kernel/arch/x86/kvm/ioapic.c | 2 +- kernel/arch/x86/kvm/irq_comm.c | 13 + kernel/arch/x86/kvm/lapic.c | 6 + kernel/arch/x86/kvm/lapic.h | 1 + kernel/arch/x86/kvm/mtrr.c | 3 +- kernel/arch/x86/kvm/vmx.c | 85 +++- kernel/arch/x86/kvm/x86.c | 66 ++- kernel/arch/x86/mm/kmmio.c | 88 ++-- kernel/arch/x86/mm/mmap.c | 14 +- kernel/arch/x86/mm/pat.c | 109 +++-- kernel/arch/x86/mm/tlb.c | 12 +- kernel/arch/x86/pci/acpi.c | 10 + kernel/arch/x86/pci/common.c | 26 +- kernel/arch/x86/pci/fixup.c | 7 + kernel/arch/x86/pci/intel_mid_pci.c | 21 +- kernel/arch/x86/pci/irq.c | 23 +- kernel/arch/x86/pci/xen.c | 7 +- kernel/arch/x86/purgatory/Makefile | 1 + kernel/arch/x86/xen/enlighten.c | 11 +- kernel/arch/x86/xen/mmu.c | 2 +- kernel/arch/x86/xen/setup.c | 65 ++- kernel/arch/xtensa/kernel/head.S | 2 +- kernel/arch/xtensa/mm/cache.c | 8 +- kernel/arch/xtensa/platforms/iss/console.c | 10 +- kernel/block/bio.c | 15 + kernel/block/blk-cgroup.c | 4 +- kernel/block/blk-core.c | 6 +- kernel/block/blk-map.c | 3 + kernel/block/blk-merge.c | 22 + kernel/block/blk-mq.c | 18 +- kernel/block/bsg.c | 3 + kernel/block/cfq-iosched.c | 15 +- kernel/block/genhd.c | 3 +- kernel/block/ioprio.c | 2 + kernel/block/partition-generic.c | 13 +- kernel/crypto/Makefile | 1 + kernel/crypto/ahash.c | 3 +- kernel/crypto/algapi.c | 1 + kernel/crypto/asymmetric_keys/Kconfig | 1 + kernel/crypto/asymmetric_keys/pkcs7_parser.c | 4 +- kernel/crypto/asymmetric_keys/pkcs7_trust.c | 2 + kernel/crypto/asymmetric_keys/x509_cert_parser.c | 8 +- kernel/crypto/async_tx/async_pq.c | 8 +- kernel/crypto/blkcipher.c | 3 +- kernel/crypto/cryptd.c | 9 +- kernel/crypto/crypto_user.c | 1 + kernel/crypto/echainiv.c | 115 ++--- kernel/crypto/gcm.c | 6 +- kernel/crypto/ghash-generic.c | 13 +- kernel/crypto/keywrap.c | 4 +- kernel/crypto/mcryptd.c | 19 +- kernel/crypto/scatterwalk.c | 3 +- kernel/crypto/testmgr.c | 27 +- kernel/drivers/acpi/acpi_processor.c | 52 +++ kernel/drivers/acpi/acpica/dsmethod.c | 3 + kernel/drivers/acpi/acpica/hwxface.c | 24 +- kernel/drivers/acpi/apei/ghes.c | 9 +- kernel/drivers/acpi/bus.c | 3 + kernel/drivers/acpi/cppc_acpi.c | 24 +- kernel/drivers/acpi/ec.c | 41 +- kernel/drivers/acpi/internal.h | 6 + kernel/drivers/acpi/nfit.c | 6 +- kernel/drivers/acpi/nfit.h | 4 + kernel/drivers/acpi/numa.c | 16 +- kernel/drivers/acpi/osl.c | 16 +- kernel/drivers/acpi/pci_irq.c | 17 +- kernel/drivers/acpi/resource.c | 14 +- kernel/drivers/acpi/scan.c | 6 +- kernel/drivers/acpi/sleep.c | 1 + kernel/drivers/acpi/sysfs.c | 7 +- kernel/drivers/acpi/video_detect.c | 20 + kernel/drivers/android/binder.c | 35 +- kernel/drivers/ata/ahci_platform.c | 3 + kernel/drivers/ata/ahci_xgene.c | 4 +- kernel/drivers/ata/libahci.c | 1 + kernel/drivers/ata/libata-core.c | 6 + kernel/drivers/ata/libata-eh.c | 2 +- kernel/drivers/ata/sata_mv.c | 3 + kernel/drivers/base/core.c | 39 +- kernel/drivers/base/dma-mapping.c | 2 +- kernel/drivers/base/firmware_class.c | 7 +- kernel/drivers/base/memory.c | 11 +- kernel/drivers/base/module.c | 8 +- kernel/drivers/base/platform.c | 4 +- kernel/drivers/base/power/domain.c | 2 +- kernel/drivers/base/power/main.c | 13 +- kernel/drivers/base/power/opp/core.c | 10 +- kernel/drivers/base/power/power.h | 19 +- kernel/drivers/base/power/runtime.c | 17 +- kernel/drivers/base/power/wakeirq.c | 76 +++- kernel/drivers/base/regmap/regmap-spmi.c | 2 +- kernel/drivers/bcma/bcma_private.h | 2 - kernel/drivers/block/brd.c | 2 +- kernel/drivers/block/drbd/drbd_main.c | 2 +- kernel/drivers/block/loop.c | 8 +- kernel/drivers/block/mtip32xx/mtip32xx.c | 267 +++++++++--- kernel/drivers/block/mtip32xx/mtip32xx.h | 11 +- kernel/drivers/block/nbd.c | 4 +- kernel/drivers/block/paride/pd.c | 4 +- kernel/drivers/block/paride/pt.c | 4 +- kernel/drivers/block/rbd.c | 6 +- kernel/drivers/block/zram/zram_drv.c | 11 +- kernel/drivers/bluetooth/ath3k.c | 8 + kernel/drivers/bluetooth/btusb.c | 15 +- kernel/drivers/bluetooth/hci_intel.c | 6 +- kernel/drivers/bluetooth/hci_vhci.c | 28 +- kernel/drivers/bus/arm-ccn.c | 27 +- kernel/drivers/bus/imx-weim.c | 2 +- kernel/drivers/bus/vexpress-config.c | 7 +- kernel/drivers/char/hw_random/core.c | 6 +- kernel/drivers/char/hw_random/exynos-rng.c | 19 +- kernel/drivers/char/hw_random/omap-rng.c | 16 +- kernel/drivers/char/ipmi/ipmi_msghandler.c | 8 +- kernel/drivers/char/random.c | 42 +- kernel/drivers/char/tpm/tpm-chip.c | 14 +- kernel/drivers/char/tpm/tpm-dev.c | 2 +- kernel/drivers/char/tpm/tpm-interface.c | 51 ++- kernel/drivers/char/tpm/tpm-sysfs.c | 2 +- kernel/drivers/char/tpm/tpm.h | 12 +- kernel/drivers/char/tpm/tpm2-cmd.c | 103 +++-- kernel/drivers/char/tpm/tpm_crb.c | 11 +- kernel/drivers/char/tpm/tpm_eventlog.c | 14 +- kernel/drivers/char/tpm/xen-tpmfront.c | 1 - kernel/drivers/char/virtio_console.c | 22 +- kernel/drivers/clk/at91/clk-h32mx.c | 2 +- kernel/drivers/clk/bcm/clk-bcm2835.c | 39 +- kernel/drivers/clk/clk-divider.c | 13 +- kernel/drivers/clk/clk-qoriq.c | 19 +- kernel/drivers/clk/clk-wm831x.c | 2 +- kernel/drivers/clk/clk-xgene.c | 3 +- kernel/drivers/clk/imx/clk-imx31.c | 4 +- kernel/drivers/clk/imx/clk-imx35.c | 6 +- kernel/drivers/clk/imx/clk-imx6q.c | 18 + kernel/drivers/clk/meson/clkc.c | 2 +- kernel/drivers/clk/mmp/clk-of-mmp2.c | 2 +- kernel/drivers/clk/mmp/clk-of-pxa168.c | 2 +- kernel/drivers/clk/mmp/clk-of-pxa910.c | 4 +- kernel/drivers/clk/nxp/clk-lpc18xx-ccu.c | 2 +- kernel/drivers/clk/qcom/gcc-msm8916.c | 2 + kernel/drivers/clk/qcom/gcc-msm8960.c | 4 +- kernel/drivers/clk/rockchip/clk-mmc-phase.c | 1 + kernel/drivers/clk/rockchip/clk-rk3188.c | 1 + kernel/drivers/clk/rockchip/clk-rk3368.c | 48 +-- kernel/drivers/clk/rockchip/clk.c | 13 +- kernel/drivers/clk/ti/clk-3xxx.c | 20 +- kernel/drivers/clk/ti/clock.h | 9 + kernel/drivers/clk/ti/dpll.c | 19 +- kernel/drivers/clk/ti/dpll3xxx.c | 67 +++ kernel/drivers/clk/versatile/clk-sp810.c | 4 +- kernel/drivers/clocksource/exynos_mct.c | 1 + kernel/drivers/clocksource/sun4i_timer.c | 9 +- kernel/drivers/cpufreq/cpufreq_userspace.c | 43 +- kernel/drivers/cpufreq/intel_pstate.c | 17 +- kernel/drivers/cpufreq/powernv-cpufreq.c | 8 +- kernel/drivers/cpuidle/cpuidle-arm.c | 3 +- kernel/drivers/cpuidle/cpuidle.c | 4 +- kernel/drivers/crypto/atmel-aes.c | 4 +- kernel/drivers/crypto/atmel-sha.c | 4 +- kernel/drivers/crypto/atmel-tdes.c | 4 +- kernel/drivers/crypto/caam/caamalg.c | 103 +++-- kernel/drivers/crypto/caam/caamhash.c | 1 + kernel/drivers/crypto/caam/jr.c | 2 +- kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 39 ++ kernel/drivers/crypto/ccp/ccp-crypto-aes-xts.c | 17 +- kernel/drivers/crypto/ccp/ccp-crypto-sha.c | 43 ++ kernel/drivers/crypto/ccp/ccp-crypto.h | 22 + kernel/drivers/crypto/marvell/cesa.c | 2 +- kernel/drivers/crypto/nx/nx-842-powernv.c | 12 +- kernel/drivers/crypto/nx/nx.c | 2 +- kernel/drivers/crypto/qat/qat_common/Makefile | 1 + .../drivers/crypto/qat/qat_common/adf_common_drv.h | 11 + kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c | 6 + kernel/drivers/crypto/qat/qat_common/adf_sriov.c | 26 +- kernel/drivers/crypto/qat/qat_common/qat_algs.c | 4 +- kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 10 +- kernel/drivers/crypto/talitos.c | 151 +++++-- kernel/drivers/crypto/ux500/cryp/cryp_core.c | 4 +- kernel/drivers/crypto/ux500/hash/hash_core.c | 8 +- kernel/drivers/crypto/vmx/aes_cbc.c | 4 +- kernel/drivers/crypto/vmx/aes_ctr.c | 4 +- kernel/drivers/crypto/vmx/ghash.c | 31 +- kernel/drivers/crypto/vmx/ppc-xlate.pl | 20 + kernel/drivers/dma/at_xdmac.c | 108 +++-- kernel/drivers/dma/dw/core.c | 34 +- kernel/drivers/dma/hsu/hsu.c | 2 +- kernel/drivers/dma/hsu/hsu.h | 3 + kernel/drivers/dma/ipu/ipu_irq.c | 9 +- kernel/drivers/dma/pl330.c | 11 + kernel/drivers/dma/pxa_dma.c | 39 +- kernel/drivers/dma/sh/usb-dmac.c | 19 +- kernel/drivers/edac/amd64_edac.c | 2 +- kernel/drivers/edac/edac_mc.c | 2 +- kernel/drivers/edac/edac_mc_sysfs.c | 20 +- kernel/drivers/edac/i7core_edac.c | 2 +- kernel/drivers/edac/sb_edac.c | 43 +- kernel/drivers/extcon/extcon-max77843.c | 2 +- kernel/drivers/firewire/net.c | 59 ++- kernel/drivers/firmware/efi/efi.c | 1 + kernel/drivers/firmware/efi/vars.c | 37 +- kernel/drivers/gpio/Kconfig | 1 + kernel/drivers/gpio/gpio-bcm-kona.c | 4 +- kernel/drivers/gpio/gpio-intel-mid.c | 19 - kernel/drivers/gpio/gpio-mpc8xxx.c | 2 +- kernel/drivers/gpio/gpio-pca953x.c | 2 +- kernel/drivers/gpio/gpio-sa1100.c | 2 +- kernel/drivers/gpio/gpiolib-legacy.c | 8 +- kernel/drivers/gpio/gpiolib.c | 52 +-- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.h | 6 +- .../drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c | 38 +- .../gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c | 5 + kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c | 2 +- .../drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c | 17 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 + kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c | 16 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 4 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 5 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 4 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h | 6 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 + kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c | 20 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 +- kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 5 + kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 1 + kernel/drivers/gpu/drm/amd/amdgpu/atombios_dp.c | 104 ++--- .../drivers/gpu/drm/amd/amdgpu/atombios_encoders.c | 5 + kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 3 + kernel/drivers/gpu/drm/amd/amdgpu/cz_dpm.c | 4 +- kernel/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 23 +- kernel/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 24 +- kernel/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 48 ++- kernel/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 4 +- kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 17 +- kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 23 +- kernel/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 4 +- kernel/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c | 2 +- kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c | 70 ++-- kernel/drivers/gpu/drm/ast/ast_main.c | 7 +- .../gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c | 12 +- kernel/drivers/gpu/drm/drm_atomic.c | 5 +- kernel/drivers/gpu/drm/drm_atomic_helper.c | 4 - kernel/drivers/gpu/drm/drm_cache.c | 1 + kernel/drivers/gpu/drm/drm_crtc.c | 65 ++- kernel/drivers/gpu/drm/drm_dp_helper.c | 27 +- kernel/drivers/gpu/drm/drm_dp_mst_topology.c | 43 +- kernel/drivers/gpu/drm/drm_edid.c | 8 + kernel/drivers/gpu/drm/drm_fb_helper.c | 5 +- kernel/drivers/gpu/drm/drm_gem.c | 29 +- kernel/drivers/gpu/drm/drm_ioc32.c | 4 + kernel/drivers/gpu/drm/drm_modes.c | 9 + kernel/drivers/gpu/drm/drm_prime.c | 17 +- kernel/drivers/gpu/drm/exynos/exynos_drm_core.c | 2 +- kernel/drivers/gpu/drm/gma500/cdv_intel_dp.c | 2 +- .../drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c | 2 +- kernel/drivers/gpu/drm/gma500/psb_drv.c | 3 + kernel/drivers/gpu/drm/i915/i915_drv.h | 27 +- kernel/drivers/gpu/drm/i915/i915_gem_gtt.c | 1 + kernel/drivers/gpu/drm/i915/i915_gem_shrinker.c | 2 +- kernel/drivers/gpu/drm/i915/i915_gem_stolen.c | 23 +- kernel/drivers/gpu/drm/i915/i915_reg.h | 26 +- kernel/drivers/gpu/drm/i915/intel_bios.c | 39 ++ kernel/drivers/gpu/drm/i915/intel_crt.c | 17 +- kernel/drivers/gpu/drm/i915/intel_csr.c | 3 +- kernel/drivers/gpu/drm/i915/intel_ddi.c | 22 +- kernel/drivers/gpu/drm/i915/intel_display.c | 132 +++--- kernel/drivers/gpu/drm/i915/intel_dp.c | 68 +-- kernel/drivers/gpu/drm/i915/intel_dp_mst.c | 6 +- kernel/drivers/gpu/drm/i915/intel_drv.h | 13 +- kernel/drivers/gpu/drm/i915/intel_fbdev.c | 6 +- kernel/drivers/gpu/drm/i915/intel_hdmi.c | 115 ++--- kernel/drivers/gpu/drm/i915/intel_lrc.c | 6 +- kernel/drivers/gpu/drm/i915/intel_opregion.c | 2 +- kernel/drivers/gpu/drm/i915/intel_pm.c | 67 +-- kernel/drivers/gpu/drm/i915/intel_ringbuffer.c | 30 +- kernel/drivers/gpu/drm/i915/intel_sprite.c | 2 +- kernel/drivers/gpu/drm/i915/intel_uncore.c | 6 +- kernel/drivers/gpu/drm/imx/imx-drm-core.c | 8 + kernel/drivers/gpu/drm/imx/ipuv3-crtc.c | 2 +- kernel/drivers/gpu/drm/mgag200/mgag200_mode.c | 10 +- kernel/drivers/gpu/drm/msm/msm_gem_submit.c | 27 +- kernel/drivers/gpu/drm/nouveau/dispnv04/hw.c | 3 +- kernel/drivers/gpu/drm/nouveau/nouveau_bios.c | 3 + kernel/drivers/gpu/drm/nouveau/nouveau_drm.c | 11 +- kernel/drivers/gpu/drm/nouveau/nouveau_fbcon.c | 2 + kernel/drivers/gpu/drm/nouveau/nv04_fbcon.c | 7 +- kernel/drivers/gpu/drm/nouveau/nv50_fbcon.c | 6 +- kernel/drivers/gpu/drm/nouveau/nvc0_fbcon.c | 6 +- kernel/drivers/gpu/drm/nouveau/nvkm/core/ramht.c | 6 +- .../gpu/drm/nouveau/nvkm/engine/device/base.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/disp/hdagt215.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c | 8 +- .../drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 39 +- .../drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c | 4 +- .../drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c | 4 +- .../drivers/gpu/drm/nouveau/nvkm/engine/pm/nv40.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/bios/priv.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/bios/shadow.c | 7 +- .../gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c | 1 + .../drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 2 + kernel/drivers/gpu/drm/qxl/qxl_display.c | 13 +- kernel/drivers/gpu/drm/qxl/qxl_draw.c | 2 + kernel/drivers/gpu/drm/qxl/qxl_drv.h | 2 + kernel/drivers/gpu/drm/radeon/atombios_crtc.c | 12 + kernel/drivers/gpu/drm/radeon/atombios_dp.c | 116 ++--- kernel/drivers/gpu/drm/radeon/atombios_encoders.c | 11 +- kernel/drivers/gpu/drm/radeon/evergreen.c | 154 ++++++- kernel/drivers/gpu/drm/radeon/evergreen_reg.h | 46 ++ kernel/drivers/gpu/drm/radeon/ni.c | 4 +- kernel/drivers/gpu/drm/radeon/r600_dpm.c | 15 +- kernel/drivers/gpu/drm/radeon/radeon_atombios.c | 4 +- .../drivers/gpu/drm/radeon/radeon_atpx_handler.c | 5 + kernel/drivers/gpu/drm/radeon/radeon_connectors.c | 22 +- kernel/drivers/gpu/drm/radeon/radeon_cursor.c | 73 ++-- kernel/drivers/gpu/drm/radeon/radeon_device.c | 22 + kernel/drivers/gpu/drm/radeon/radeon_dp_mst.c | 4 +- kernel/drivers/gpu/drm/radeon/radeon_legacy_crtc.c | 2 + kernel/drivers/gpu/drm/radeon/radeon_mode.h | 7 +- kernel/drivers/gpu/drm/radeon/radeon_ttm.c | 6 +- kernel/drivers/gpu/drm/radeon/si_dpm.c | 36 +- kernel/drivers/gpu/drm/radeon/sislands_smc.h | 1 + kernel/drivers/gpu/drm/ttm/ttm_bo.c | 7 +- kernel/drivers/gpu/drm/udl/udl_fb.c | 2 +- kernel/drivers/gpu/drm/udl/udl_gem.c | 2 +- kernel/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c | 25 +- kernel/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 12 + kernel/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 1 + kernel/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 16 +- kernel/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c | 53 +-- kernel/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 10 +- kernel/drivers/gpu/ipu-v3/ipu-common.c | 12 +- kernel/drivers/hid/hid-core.c | 11 +- kernel/drivers/hid/hid-corsair.c | 60 ++- kernel/drivers/hid/hid-cypress.c | 3 + kernel/drivers/hid/hid-elo.c | 2 +- kernel/drivers/hid/hid-ids.h | 2 + kernel/drivers/hid/hid-multitouch.c | 23 +- kernel/drivers/hid/hid-sony.c | 6 +- kernel/drivers/hid/i2c-hid/i2c-hid.c | 16 +- kernel/drivers/hid/uhid.c | 33 +- kernel/drivers/hid/usbhid/hid-core.c | 73 ++-- kernel/drivers/hid/usbhid/hid-quirks.c | 2 + kernel/drivers/hid/usbhid/hiddev.c | 10 +- kernel/drivers/hid/wacom_wac.c | 40 +- kernel/drivers/hv/channel.c | 27 +- kernel/drivers/hv/channel_mgmt.c | 61 ++- kernel/drivers/hv/hv.c | 10 +- kernel/drivers/hv/hv_fcopy.c | 37 +- kernel/drivers/hv/hv_kvp.c | 31 +- kernel/drivers/hv/hv_snapshot.c | 34 +- kernel/drivers/hv/hv_util.c | 10 +- kernel/drivers/hv/hv_utils_transport.c | 9 +- kernel/drivers/hv/hyperv_vmbus.h | 11 +- kernel/drivers/hv/vmbus_drv.c | 38 +- kernel/drivers/hwmon/ads7828.c | 10 + kernel/drivers/hwmon/adt7411.c | 5 +- kernel/drivers/hwmon/amc6821.c | 4 +- kernel/drivers/hwmon/dell-smm-hwmon.c | 80 +++- kernel/drivers/hwmon/ds620.c | 2 +- kernel/drivers/hwmon/g762.c | 11 +- kernel/drivers/hwmon/iio_hwmon.c | 24 +- kernel/drivers/hwmon/max1111.c | 6 + kernel/drivers/hwmon/nct7802.c | 8 +- kernel/drivers/hwmon/scpi-hwmon.c | 1 + kernel/drivers/hwtracing/intel_th/core.c | 35 +- kernel/drivers/hwtracing/intel_th/intel_th.h | 3 + kernel/drivers/hwtracing/intel_th/pci.c | 5 + kernel/drivers/hwtracing/stm/Kconfig | 1 + kernel/drivers/i2c/Kconfig | 1 - kernel/drivers/i2c/busses/i2c-cpm.c | 4 +- kernel/drivers/i2c/busses/i2c-cros-ec-tunnel.c | 2 +- kernel/drivers/i2c/busses/i2c-efm32.c | 2 +- kernel/drivers/i2c/busses/i2c-eg20t.c | 18 +- kernel/drivers/i2c/busses/i2c-exynos5.c | 24 +- kernel/drivers/i2c/busses/i2c-i801.c | 103 ++++- kernel/drivers/i2c/busses/i2c-qup.c | 3 +- kernel/drivers/i2c/busses/i2c-xgene-slimpro.c | 2 +- kernel/drivers/i2c/i2c-core.c | 4 +- kernel/drivers/i2c/i2c-dev.c | 2 +- kernel/drivers/i2c/muxes/Kconfig | 1 + kernel/drivers/i2c/muxes/i2c-mux-reg.c | 2 +- kernel/drivers/idle/intel_idle.c | 133 +++++- kernel/drivers/iio/accel/bmc150-accel-core.c | 18 +- kernel/drivers/iio/accel/kxsd9.c | 6 +- kernel/drivers/iio/adc/Kconfig | 1 + kernel/drivers/iio/adc/ad7266.c | 7 +- kernel/drivers/iio/adc/ad799x.c | 1 + kernel/drivers/iio/adc/at91_adc.c | 4 +- kernel/drivers/iio/adc/rockchip_saradc.c | 30 ++ kernel/drivers/iio/adc/ti_am335x_adc.c | 16 +- .../iio/common/hid-sensors/hid-sensor-attributes.c | 58 +-- kernel/drivers/iio/gyro/bmg160_core.c | 9 +- kernel/drivers/iio/humidity/hdc100x.c | 20 +- kernel/drivers/iio/industrialio-buffer.c | 27 +- kernel/drivers/iio/industrialio-core.c | 14 +- kernel/drivers/iio/industrialio-trigger.c | 23 +- kernel/drivers/iio/light/apds9960.c | 1 + kernel/drivers/iio/magnetometer/ak8975.c | 6 +- kernel/drivers/iio/magnetometer/st_magn.h | 1 + .../drivers/iio/orientation/hid-sensor-rotation.c | 1 + kernel/drivers/iio/pressure/st_pressure_core.c | 80 ++-- kernel/drivers/iio/proximity/as3935.c | 19 +- kernel/drivers/infiniband/core/cm.c | 130 +++++- kernel/drivers/infiniband/core/cma.c | 3 +- kernel/drivers/infiniband/core/iwpm_util.c | 1 + kernel/drivers/infiniband/core/mad.c | 2 +- kernel/drivers/infiniband/core/multicast.c | 20 +- kernel/drivers/infiniband/core/sa_query.c | 2 +- kernel/drivers/infiniband/core/ucm.c | 4 + kernel/drivers/infiniband/core/ucma.c | 3 + kernel/drivers/infiniband/core/umem.c | 4 +- kernel/drivers/infiniband/core/uverbs.h | 1 + kernel/drivers/infiniband/core/uverbs_main.c | 49 ++- kernel/drivers/infiniband/hw/cxgb4/cq.c | 2 +- kernel/drivers/infiniband/hw/cxgb4/qp.c | 6 +- kernel/drivers/infiniband/hw/mlx4/ah.c | 13 +- kernel/drivers/infiniband/hw/mlx4/cq.c | 5 +- kernel/drivers/infiniband/hw/mlx4/mad.c | 47 ++- kernel/drivers/infiniband/hw/mlx4/main.c | 29 +- kernel/drivers/infiniband/hw/mlx4/mcg.c | 14 +- kernel/drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +- kernel/drivers/infiniband/hw/mlx4/qp.c | 46 +- kernel/drivers/infiniband/hw/mlx5/cq.c | 15 +- kernel/drivers/infiniband/hw/mlx5/main.c | 18 +- kernel/drivers/infiniband/hw/mlx5/mr.c | 28 ++ kernel/drivers/infiniband/hw/mlx5/qp.c | 21 +- kernel/drivers/infiniband/hw/qib/qib_file_ops.c | 5 + kernel/drivers/infiniband/ulp/ipoib/ipoib.h | 21 +- kernel/drivers/infiniband/ulp/ipoib/ipoib_cm.c | 33 +- kernel/drivers/infiniband/ulp/ipoib/ipoib_ib.c | 23 +- kernel/drivers/infiniband/ulp/ipoib/ipoib_main.c | 60 ++- .../drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 37 +- kernel/drivers/infiniband/ulp/isert/ib_isert.c | 122 +++--- kernel/drivers/infiniband/ulp/isert/ib_isert.h | 2 +- kernel/drivers/infiniband/ulp/srp/ib_srp.c | 2 +- kernel/drivers/infiniband/ulp/srpt/ib_srpt.c | 59 +-- kernel/drivers/input/joystick/xpad.c | 354 +++++++++++----- kernel/drivers/input/keyboard/tegra-kbc.c | 2 +- kernel/drivers/input/misc/ati_remote2.c | 36 +- kernel/drivers/input/misc/drv260x.c | 1 - kernel/drivers/input/misc/ims-pcu.c | 4 + kernel/drivers/input/misc/max8997_haptic.c | 6 +- kernel/drivers/input/misc/pmic8xxx-pwrkey.c | 7 +- kernel/drivers/input/misc/powermate.c | 3 + kernel/drivers/input/misc/pwm-beeper.c | 69 ++- kernel/drivers/input/misc/uinput.c | 6 + kernel/drivers/input/mouse/elan_i2c_core.c | 79 +++- kernel/drivers/input/mouse/elantech.c | 33 +- kernel/drivers/input/mouse/synaptics.c | 5 +- kernel/drivers/input/mouse/vmmouse.c | 22 +- kernel/drivers/input/serio/i8042-io.h | 2 +- kernel/drivers/input/serio/i8042-ip22io.h | 2 +- kernel/drivers/input/serio/i8042-ppcio.h | 2 +- kernel/drivers/input/serio/i8042-sparcio.h | 2 +- kernel/drivers/input/serio/i8042-unicore32io.h | 2 +- kernel/drivers/input/serio/i8042-x86ia64io.h | 109 ++++- kernel/drivers/input/serio/i8042.c | 72 ++-- kernel/drivers/input/serio/libps2.c | 10 +- kernel/drivers/input/tablet/gtco.c | 10 +- kernel/drivers/input/touchscreen/elants_i2c.c | 4 +- kernel/drivers/input/touchscreen/sur40.c | 5 +- kernel/drivers/input/touchscreen/tsc2004.c | 7 +- kernel/drivers/input/touchscreen/tsc2005.c | 7 +- kernel/drivers/input/touchscreen/tsc200x-core.c | 15 +- kernel/drivers/input/touchscreen/tsc200x-core.h | 2 +- kernel/drivers/input/touchscreen/wacom_w8001.c | 2 +- kernel/drivers/input/touchscreen/zforce_ts.c | 4 +- kernel/drivers/iommu/amd_iommu.c | 132 +++++- kernel/drivers/iommu/amd_iommu_init.c | 14 +- kernel/drivers/iommu/amd_iommu_v2.c | 4 +- kernel/drivers/iommu/arm-smmu-v3.c | 8 +- kernel/drivers/iommu/dma-iommu.c | 7 +- kernel/drivers/iommu/dmar.c | 7 +- kernel/drivers/iommu/exynos-iommu.c | 1 + kernel/drivers/iommu/intel-iommu.c | 91 +++- kernel/drivers/iommu/intel-svm.c | 28 +- kernel/drivers/iommu/iommu.c | 3 +- kernel/drivers/irqchip/irq-atmel-aic.c | 5 +- kernel/drivers/irqchip/irq-atmel-aic5.c | 5 +- kernel/drivers/irqchip/irq-bcm7038-l1.c | 26 ++ kernel/drivers/irqchip/irq-gic-v3-its.c | 49 ++- kernel/drivers/irqchip/irq-gic-v3.c | 28 +- kernel/drivers/irqchip/irq-gic.c | 8 + kernel/drivers/irqchip/irq-mxs.c | 2 +- kernel/drivers/irqchip/irq-sunxi-nmi.c | 4 +- kernel/drivers/isdn/gigaset/ser-gigaset.c | 4 +- kernel/drivers/isdn/hardware/eicon/message.c | 3 +- kernel/drivers/lightnvm/gennvm.c | 3 +- kernel/drivers/lightnvm/rrpc.c | 26 +- kernel/drivers/mcb/mcb-parse.c | 2 +- kernel/drivers/md/bcache/super.c | 48 ++- kernel/drivers/md/dm-cache-metadata.c | 122 ++++-- kernel/drivers/md/dm-cache-metadata.h | 4 +- kernel/drivers/md/dm-cache-target.c | 12 +- kernel/drivers/md/dm-crypt.c | 38 +- kernel/drivers/md/dm-flakey.c | 34 +- kernel/drivers/md/dm-log-writes.c | 10 +- kernel/drivers/md/dm-mpath.c | 6 +- kernel/drivers/md/dm-raid1.c | 1 + kernel/drivers/md/dm-snap.c | 9 + kernel/drivers/md/dm-table.c | 36 +- kernel/drivers/md/dm-thin-metadata.c | 5 +- kernel/drivers/md/dm.c | 43 +- kernel/drivers/md/md.c | 16 +- kernel/drivers/md/multipath.c | 4 +- .../md/persistent-data/dm-space-map-metadata.c | 14 +- kernel/drivers/md/raid1.c | 7 +- kernel/drivers/md/raid10.c | 7 +- kernel/drivers/md/raid5.c | 60 ++- kernel/drivers/md/raid5.h | 4 +- kernel/drivers/media/dvb-core/dvb_ringbuffer.c | 74 +++- kernel/drivers/media/dvb-frontends/Kconfig | 2 +- kernel/drivers/media/dvb-frontends/mb86a20s.c | 104 ++--- kernel/drivers/media/i2c/Kconfig | 1 + kernel/drivers/media/i2c/adv7511.c | 21 +- kernel/drivers/media/pci/bt8xx/bttv-driver.c | 26 +- kernel/drivers/media/pci/cx23885/cx23885-dvb.c | 6 +- kernel/drivers/media/pci/saa7134/saa7134-video.c | 18 +- kernel/drivers/media/pci/solo6x10/solo6x10.h | 3 + kernel/drivers/media/platform/am437x/am437x-vpfe.c | 2 +- kernel/drivers/media/platform/blackfin/ppi.c | 2 + kernel/drivers/media/platform/coda/coda-bit.c | 2 +- kernel/drivers/media/platform/coda/coda-common.c | 10 +- kernel/drivers/media/platform/s5p-mfc/s5p_mfc.c | 11 + kernel/drivers/media/platform/vsp1/vsp1_sru.c | 1 + kernel/drivers/media/rc/ir-rc5-decoder.c | 2 +- kernel/drivers/media/rc/ite-cir.c | 2 + kernel/drivers/media/usb/airspy/airspy.c | 3 +- kernel/drivers/media/usb/au0828/au0828-core.c | 2 +- kernel/drivers/media/usb/au0828/au0828-input.c | 4 +- kernel/drivers/media/usb/au0828/au0828-video.c | 63 ++- kernel/drivers/media/usb/au0828/au0828.h | 9 +- kernel/drivers/media/usb/cx231xx/cx231xx-avcore.c | 5 +- kernel/drivers/media/usb/cx231xx/cx231xx-cards.c | 2 +- kernel/drivers/media/usb/cx231xx/cx231xx-core.c | 3 +- kernel/drivers/media/usb/dvb-usb/dib0700_core.c | 5 +- kernel/drivers/media/usb/em28xx/em28xx-i2c.c | 5 +- kernel/drivers/media/usb/gspca/cpia1.c | 2 +- kernel/drivers/media/usb/gspca/konica.c | 2 +- kernel/drivers/media/usb/gspca/t613.c | 2 +- kernel/drivers/media/usb/pwc/pwc-if.c | 6 + kernel/drivers/media/usb/usbtv/usbtv-audio.c | 5 + .../drivers/media/usb/usbvision/usbvision-video.c | 16 +- kernel/drivers/media/usb/uvc/uvc_driver.c | 20 + kernel/drivers/media/usb/uvc/uvc_v4l2.c | 39 +- kernel/drivers/media/usb/uvc/uvcvideo.h | 12 + .../drivers/media/v4l2-core/v4l2-compat-ioctl32.c | 24 +- kernel/drivers/media/v4l2-core/videobuf2-core.c | 18 +- kernel/drivers/media/v4l2-core/videobuf2-memops.c | 2 +- kernel/drivers/media/v4l2-core/videobuf2-v4l2.c | 6 + kernel/drivers/memory/omap-gpmc.c | 2 +- kernel/drivers/memstick/host/rtsx_usb_ms.c | 6 + kernel/drivers/mfd/Kconfig | 1 + kernel/drivers/mfd/atmel-hlcdc.c | 5 +- kernel/drivers/mfd/intel-lpss.c | 17 +- kernel/drivers/mfd/intel_soc_pmic_core.c | 1 + kernel/drivers/mfd/mfd-core.c | 2 + kernel/drivers/mfd/omap-usb-tll.c | 13 +- kernel/drivers/mfd/qcom_rpm.c | 55 ++- kernel/drivers/mfd/rtsx_usb.c | 10 +- kernel/drivers/misc/Kconfig | 2 +- kernel/drivers/misc/ad525x_dpot.c | 2 +- kernel/drivers/misc/cxl/Makefile | 2 +- kernel/drivers/misc/cxl/api.c | 6 +- kernel/drivers/misc/cxl/context.c | 15 +- kernel/drivers/misc/cxl/cxl.h | 15 + kernel/drivers/misc/cxl/fault.c | 129 ++++-- kernel/drivers/misc/cxl/file.c | 25 +- kernel/drivers/misc/cxl/irq.c | 1 - kernel/drivers/misc/cxl/pci.c | 1 + kernel/drivers/misc/genwqe/card_utils.c | 12 +- kernel/drivers/misc/mei/amthif.c | 4 +- kernel/drivers/misc/mei/bus-fixup.c | 2 +- kernel/drivers/misc/mei/bus.c | 28 +- kernel/drivers/misc/mei/client.c | 6 +- kernel/drivers/misc/mei/hbm.c | 3 +- kernel/drivers/misc/mei/hw-me-regs.h | 4 + kernel/drivers/misc/mei/hw-me.c | 10 +- kernel/drivers/misc/mei/hw-txe.c | 6 +- kernel/drivers/misc/mei/interrupt.c | 6 +- kernel/drivers/misc/mei/main.c | 2 +- kernel/drivers/misc/mei/mei_dev.h | 2 + kernel/drivers/misc/mei/pci-me.c | 7 +- kernel/drivers/misc/mic/scif/scif_rma.c | 7 +- kernel/drivers/mmc/card/block.c | 46 +- kernel/drivers/mmc/card/mmc_test.c | 2 +- kernel/drivers/mmc/card/queue.h | 2 +- kernel/drivers/mmc/core/core.c | 4 +- kernel/drivers/mmc/core/mmc.c | 7 + kernel/drivers/mmc/host/Kconfig | 1 + kernel/drivers/mmc/host/dw_mmc-pltfm.c | 5 +- kernel/drivers/mmc/host/dw_mmc.c | 2 +- kernel/drivers/mmc/host/mmc_spi.c | 6 + kernel/drivers/mmc/host/mxs-mmc.c | 10 +- kernel/drivers/mmc/host/pxamci.c | 16 +- kernel/drivers/mmc/host/rtsx_usb_sdmmc.c | 7 +- kernel/drivers/mmc/host/sdhci-acpi.c | 87 +++- kernel/drivers/mmc/host/sdhci-pci-core.c | 30 +- kernel/drivers/mmc/host/sdhci-pci.h | 3 + kernel/drivers/mmc/host/sdhci.c | 81 ++-- kernel/drivers/mmc/host/sdhci.h | 21 +- kernel/drivers/mmc/host/sh_mmcif.c | 84 ++-- kernel/drivers/mtd/maps/pmcmsp-flash.c | 6 +- kernel/drivers/mtd/maps/sa1100-flash.c | 4 +- kernel/drivers/mtd/nand/Kconfig | 2 +- kernel/drivers/mtd/nand/brcmnand/brcmnand.c | 34 +- kernel/drivers/mtd/nand/davinci_nand.c | 3 + kernel/drivers/mtd/nand/nand_base.c | 12 +- kernel/drivers/mtd/onenand/onenand_base.c | 3 +- kernel/drivers/mtd/spi-nor/spi-nor.c | 46 +- kernel/drivers/mtd/ubi/build.c | 13 +- kernel/drivers/mtd/ubi/eba.c | 43 +- kernel/drivers/mtd/ubi/fastmap.c | 18 +- kernel/drivers/mtd/ubi/ubi.h | 2 + kernel/drivers/mtd/ubi/vmt.c | 25 +- kernel/drivers/mtd/ubi/wl.c | 21 +- kernel/drivers/net/bonding/bond_main.c | 72 ++-- kernel/drivers/net/bonding/bond_netlink.c | 6 +- kernel/drivers/net/can/at91_can.c | 5 +- kernel/drivers/net/can/c_can/c_can.c | 38 +- kernel/drivers/net/can/c_can/c_can_pci.c | 1 + kernel/drivers/net/can/dev.c | 92 +++- kernel/drivers/net/can/flexcan.c | 13 +- kernel/drivers/net/can/m_can/m_can.c | 2 +- kernel/drivers/net/can/ti_hecc.c | 16 +- .../drivers/net/can/usb/peak_usb/pcan_usb_core.c | 6 +- kernel/drivers/net/dsa/bcm_sf2.c | 4 + kernel/drivers/net/dsa/bcm_sf2.h | 2 +- kernel/drivers/net/ethernet/atheros/alx/main.c | 7 +- kernel/drivers/net/ethernet/atheros/atlx/atl2.c | 2 +- kernel/drivers/net/ethernet/broadcom/bcmsysport.c | 25 +- kernel/drivers/net/ethernet/broadcom/bgmac.c | 7 +- .../net/ethernet/broadcom/bnx2x/bnx2x_main.c | 4 + .../drivers/net/ethernet/broadcom/genet/bcmgenet.c | 12 +- kernel/drivers/net/ethernet/broadcom/tg3.c | 10 +- kernel/drivers/net/ethernet/cadence/macb.c | 13 +- kernel/drivers/net/ethernet/cadence/macb.h | 2 +- .../net/ethernet/cavium/liquidio/lio_main.c | 2 +- kernel/drivers/net/ethernet/cavium/thunder/nic.h | 9 + .../drivers/net/ethernet/cavium/thunder/nic_main.c | 6 + .../drivers/net/ethernet/cavium/thunder/nic_reg.h | 3 +- .../net/ethernet/cavium/thunder/nicvf_ethtool.c | 5 +- .../net/ethernet/cavium/thunder/nicvf_main.c | 12 +- .../net/ethernet/cavium/thunder/nicvf_queues.c | 12 +- .../net/ethernet/cavium/thunder/nicvf_queues.h | 3 +- .../net/ethernet/cavium/thunder/thunder_bgx.c | 91 ++-- .../net/ethernet/cavium/thunder/thunder_bgx.h | 2 + kernel/drivers/net/ethernet/freescale/fec_main.c | 20 +- .../net/ethernet/hisilicon/hns/hns_dsaf_reg.h | 8 +- kernel/drivers/net/ethernet/intel/e1000/e1000.h | 7 +- .../drivers/net/ethernet/intel/e1000/e1000_main.c | 5 +- kernel/drivers/net/ethernet/intel/e1000e/netdev.c | 6 +- kernel/drivers/net/ethernet/intel/fm10k/fm10k.h | 2 +- .../drivers/net/ethernet/intel/fm10k/fm10k_main.c | 8 +- .../drivers/net/ethernet/intel/fm10k/fm10k_pci.c | 65 ++- .../drivers/net/ethernet/intel/fm10k/fm10k_type.h | 1 + kernel/drivers/net/ethernet/intel/fm10k/fm10k_vf.c | 16 +- kernel/drivers/net/ethernet/intel/i40e/i40e.h | 2 + .../drivers/net/ethernet/intel/i40e/i40e_common.c | 4 +- .../drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 14 +- kernel/drivers/net/ethernet/intel/i40e/i40e_main.c | 134 ++++-- kernel/drivers/net/ethernet/intel/i40e/i40e_txrx.c | 73 ++-- .../net/ethernet/intel/i40e/i40e_virtchnl_pf.c | 23 +- .../drivers/net/ethernet/intel/i40evf/i40e_txrx.c | 182 ++++---- .../drivers/net/ethernet/intel/i40evf/i40e_txrx.h | 2 + .../net/ethernet/intel/i40evf/i40evf_ethtool.c | 40 +- .../net/ethernet/intel/i40evf/i40evf_main.c | 6 + .../net/ethernet/intel/i40evf/i40evf_virtchnl.c | 32 +- .../drivers/net/ethernet/intel/igb/e1000_82575.c | 1 + kernel/drivers/net/ethernet/intel/igb/e1000_i210.c | 27 ++ kernel/drivers/net/ethernet/intel/igb/e1000_i210.h | 1 + kernel/drivers/net/ethernet/intel/igb/e1000_regs.h | 1 + kernel/drivers/net/ethernet/intel/igb/igb.h | 2 + kernel/drivers/net/ethernet/intel/igb/igb_main.c | 17 +- .../drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 3 +- kernel/drivers/net/ethernet/jme.c | 26 +- kernel/drivers/net/ethernet/marvell/mvneta.c | 4 +- kernel/drivers/net/ethernet/marvell/mvpp2.c | 59 +-- kernel/drivers/net/ethernet/marvell/sky2.c | 13 + .../drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 2 +- kernel/drivers/net/ethernet/mellanox/mlx4/en_rx.c | 7 +- kernel/drivers/net/ethernet/mellanox/mlx4/en_tx.c | 6 +- .../net/ethernet/mellanox/mlx4/resource_tracker.c | 3 +- .../drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 85 ++-- .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 12 + .../net/ethernet/mellanox/mlx5/core/en_main.c | 27 +- .../drivers/net/ethernet/mellanox/mlx5/core/main.c | 10 +- .../drivers/net/ethernet/mellanox/mlx5/core/port.c | 10 +- kernel/drivers/net/ethernet/mellanox/mlxsw/pci.h | 8 +- .../drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 1 + .../drivers/net/ethernet/mellanox/mlxsw/spectrum.h | 2 + .../ethernet/mellanox/mlxsw/spectrum_switchdev.c | 9 +- .../drivers/net/ethernet/mellanox/mlxsw/switchx2.c | 1 + .../drivers/net/ethernet/neterion/vxge/vxge-main.c | 31 +- kernel/drivers/net/ethernet/qlogic/qed/qed_spq.c | 7 +- kernel/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h | 3 +- .../net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c | 24 +- .../net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c | 3 +- .../drivers/net/ethernet/qlogic/qlge/qlge_main.c | 11 + kernel/drivers/net/ethernet/qualcomm/qca_spi.c | 2 +- kernel/drivers/net/ethernet/renesas/ravb_main.c | 13 + kernel/drivers/net/ethernet/renesas/sh_eth.c | 12 +- kernel/drivers/net/ethernet/rocker/rocker.c | 14 +- kernel/drivers/net/ethernet/sfc/ef10.c | 16 + kernel/drivers/net/ethernet/smsc/smc91x.c | 9 +- kernel/drivers/net/ethernet/smsc/smc91x.h | 65 ++- .../net/ethernet/stmicro/stmmac/stmmac_main.c | 22 +- kernel/drivers/net/ethernet/ti/cpmac.c | 5 +- kernel/drivers/net/geneve.c | 57 ++- kernel/drivers/net/hyperv/netvsc_drv.c | 3 + kernel/drivers/net/ieee802154/atusb.c | 31 +- kernel/drivers/net/irda/irtty-sir.c | 10 - kernel/drivers/net/loopback.c | 1 + kernel/drivers/net/macvtap.c | 13 +- kernel/drivers/net/phy/bcm63xx.c | 21 +- kernel/drivers/net/phy/phy.c | 6 +- kernel/drivers/net/ppp/ppp_generic.c | 37 +- kernel/drivers/net/rionet.c | 4 +- kernel/drivers/net/team/team.c | 9 +- kernel/drivers/net/tun.c | 28 +- kernel/drivers/net/usb/asix_common.c | 2 +- kernel/drivers/net/usb/cdc_ether.c | 16 +- kernel/drivers/net/usb/cdc_mbim.c | 9 +- kernel/drivers/net/usb/cdc_ncm.c | 33 +- kernel/drivers/net/usb/qmi_wwan.c | 14 +- kernel/drivers/net/usb/r8152.c | 89 +++- kernel/drivers/net/usb/usbnet.c | 7 + kernel/drivers/net/virtio_net.c | 5 + kernel/drivers/net/vrf.c | 21 +- kernel/drivers/net/vxlan.c | 66 +-- kernel/drivers/net/wan/farsync.c | 2 +- kernel/drivers/net/wireless/ath/ath10k/core.c | 8 +- kernel/drivers/net/wireless/ath/ath10k/debug.c | 7 +- kernel/drivers/net/wireless/ath/ath10k/mac.c | 13 +- kernel/drivers/net/wireless/ath/ath10k/spectral.c | 2 +- kernel/drivers/net/wireless/ath/ath5k/led.c | 2 +- kernel/drivers/net/wireless/ath/ath9k/ar5008_phy.c | 8 +- kernel/drivers/net/wireless/ath/ath9k/ar9002_phy.c | 5 - .../drivers/net/wireless/ath/ath9k/ar9003_eeprom.c | 2 +- kernel/drivers/net/wireless/ath/ath9k/eeprom.c | 7 +- kernel/drivers/net/wireless/ath/ath9k/init.c | 11 +- kernel/drivers/net/wireless/ath/ath9k/main.c | 8 +- kernel/drivers/net/wireless/ath/ath9k/pci.c | 15 +- .../net/wireless/brcm80211/brcmfmac/bcmsdh.c | 4 +- .../net/wireless/brcm80211/brcmfmac/cfg80211.c | 7 +- .../drivers/net/wireless/brcm80211/brcmsmac/dma.c | 4 +- .../drivers/net/wireless/brcm80211/brcmsmac/stf.c | 2 +- kernel/drivers/net/wireless/iwlegacy/3945.c | 3 +- kernel/drivers/net/wireless/iwlwifi/dvm/calib.c | 2 +- kernel/drivers/net/wireless/iwlwifi/mvm/fw.c | 7 +- kernel/drivers/net/wireless/iwlwifi/mvm/mac80211.c | 10 +- kernel/drivers/net/wireless/iwlwifi/mvm/mvm.h | 3 + kernel/drivers/net/wireless/iwlwifi/mvm/sf.c | 2 +- kernel/drivers/net/wireless/iwlwifi/pcie/drv.c | 79 ++-- kernel/drivers/net/wireless/iwlwifi/pcie/trans.c | 4 +- kernel/drivers/net/wireless/iwlwifi/pcie/tx.c | 4 +- kernel/drivers/net/wireless/mac80211_hwsim.c | 1 + kernel/drivers/net/wireless/mwifiex/cfg80211.c | 13 +- kernel/drivers/net/wireless/mwifiex/join.c | 3 +- kernel/drivers/net/wireless/mwifiex/sta_ioctl.c | 1 + kernel/drivers/net/wireless/realtek/rtlwifi/base.c | 12 +- .../realtek/rtlwifi/btcoexist/halbtc8723b2ant.c | 9 +- .../realtek/rtlwifi/btcoexist/halbtcoutsrc.c | 27 +- .../realtek/rtlwifi/btcoexist/halbtcoutsrc.h | 2 +- .../wireless/realtek/rtlwifi/btcoexist/rtl_btc.c | 5 +- kernel/drivers/net/wireless/realtek/rtlwifi/core.c | 9 +- kernel/drivers/net/wireless/realtek/rtlwifi/pci.c | 16 +- kernel/drivers/net/wireless/realtek/rtlwifi/ps.c | 36 +- kernel/drivers/net/wireless/realtek/rtlwifi/regd.c | 4 +- .../net/wireless/realtek/rtlwifi/rtl8723be/hw.c | 5 + .../net/wireless/realtek/rtlwifi/rtl8723be/sw.c | 3 + kernel/drivers/net/wireless/realtek/rtlwifi/wifi.h | 3 + kernel/drivers/net/xen-netfront.c | 3 +- kernel/drivers/nfc/fdp/fdp.c | 4 +- kernel/drivers/nfc/mei_phy.c | 2 +- kernel/drivers/nvdimm/bus.c | 10 +- kernel/drivers/nvdimm/pfn_devs.c | 2 +- kernel/drivers/nvme/host/pci.c | 71 ++-- kernel/drivers/nvmem/mxs-ocotp.c | 4 +- kernel/drivers/of/base.c | 41 +- kernel/drivers/of/dynamic.c | 2 +- kernel/drivers/of/irq.c | 19 +- kernel/drivers/of/of_private.h | 3 + kernel/drivers/of/of_reserved_mem.c | 12 +- kernel/drivers/pci/hotplug/rpadlpar_core.c | 10 +- kernel/drivers/pci/msi.c | 2 + kernel/drivers/pci/pci-sysfs.c | 18 +- kernel/drivers/pci/pci.c | 4 + kernel/drivers/pci/pcie/aer/aer_inject.c | 14 - kernel/drivers/pci/pcie/aspm.c | 19 +- kernel/drivers/pci/probe.c | 54 ++- kernel/drivers/pci/quirks.c | 23 +- kernel/drivers/pcmcia/db1xxx_ss.c | 11 +- kernel/drivers/perf/arm_pmu.c | 1 + kernel/drivers/pinctrl/bcm/pinctrl-bcm2835.c | 2 +- kernel/drivers/pinctrl/freescale/pinctrl-imx.c | 21 +- kernel/drivers/pinctrl/intel/pinctrl-broxton.c | 2 +- kernel/drivers/pinctrl/intel/pinctrl-cherryview.c | 97 +++-- .../drivers/pinctrl/mediatek/pinctrl-mtk-common.c | 14 +- kernel/drivers/pinctrl/meson/pinctrl-meson.c | 2 +- kernel/drivers/pinctrl/nomadik/pinctrl-nomadik.c | 2 +- kernel/drivers/pinctrl/pinctrl-amd.c | 20 - kernel/drivers/pinctrl/pinctrl-at91-pio4.c | 4 +- kernel/drivers/pinctrl/pinctrl-pistachio.c | 36 +- kernel/drivers/pinctrl/pinctrl-single.c | 9 +- .../drivers/pinctrl/samsung/pinctrl-exynos5440.c | 15 +- kernel/drivers/pinctrl/sh-pfc/core.c | 4 +- kernel/drivers/pinctrl/sh-pfc/pinctrl.c | 3 +- kernel/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c | 4 +- kernel/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c | 5 +- kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.c | 17 +- kernel/drivers/pinctrl/sunxi/pinctrl-sunxi.h | 21 +- .../pinctrl/uniphier/pinctrl-uniphier-core.c | 6 + kernel/drivers/platform/chrome/cros_ec_dev.c | 8 +- kernel/drivers/platform/chrome/cros_ec_proto.c | 17 + kernel/drivers/platform/x86/asus-nb-wmi.c | 9 + kernel/drivers/platform/x86/dell-rbtn.c | 56 +++ kernel/drivers/platform/x86/hp-wmi.c | 7 +- kernel/drivers/platform/x86/ideapad-laptop.c | 14 + kernel/drivers/platform/x86/intel_mid_powerbtn.c | 2 +- kernel/drivers/platform/x86/toshiba-wmi.c | 26 +- kernel/drivers/platform/x86/toshiba_acpi.c | 2 +- kernel/drivers/pnp/quirks.c | 2 + kernel/drivers/power/bq24257_charger.c | 12 +- kernel/drivers/power/max17042_battery.c | 15 +- kernel/drivers/power/power_supply_core.c | 27 +- kernel/drivers/power/reset/hisi-reboot.c | 5 +- kernel/drivers/power/tps65217_charger.c | 1 + kernel/drivers/pps/clients/pps_parport.c | 2 +- kernel/drivers/pwm/core.c | 4 +- kernel/drivers/pwm/pwm-brcmstb.c | 4 +- kernel/drivers/pwm/pwm-fsl-ftm.c | 58 ++- kernel/drivers/pwm/pwm-lpc32xx.c | 55 +-- kernel/drivers/pwm/sysfs.c | 20 + kernel/drivers/regulator/anatop-regulator.c | 2 +- kernel/drivers/regulator/axp20x-regulator.c | 4 +- kernel/drivers/regulator/core.c | 22 +- kernel/drivers/regulator/qcom_smd-regulator.c | 17 +- kernel/drivers/regulator/qcom_spmi-regulator.c | 7 +- kernel/drivers/regulator/s2mps11.c | 28 +- kernel/drivers/regulator/s5m8767.c | 13 +- kernel/drivers/regulator/stw481x-vmmc.c | 3 +- kernel/drivers/regulator/tps65910-regulator.c | 6 + kernel/drivers/remoteproc/remoteproc_core.c | 15 +- kernel/drivers/rtc/rtc-ds1685.c | 8 +- kernel/drivers/rtc/rtc-hym8563.c | 2 +- kernel/drivers/rtc/rtc-max77686.c | 2 +- kernel/drivers/rtc/rtc-omap.c | 6 +- kernel/drivers/rtc/rtc-rx8025.c | 1 - kernel/drivers/rtc/rtc-s3c.c | 2 + kernel/drivers/rtc/rtc-vr41xx.c | 13 +- kernel/drivers/s390/block/dasd.c | 10 +- kernel/drivers/s390/char/con3270.c | 11 +- kernel/drivers/s390/char/sclp_ctl.c | 12 +- kernel/drivers/s390/char/vmlogrdr.c | 2 +- kernel/drivers/s390/cio/chp.c | 21 +- kernel/drivers/s390/cio/chp.h | 2 +- kernel/drivers/s390/cio/chsc.c | 63 +-- kernel/drivers/s390/cio/cmf.c | 29 +- kernel/drivers/s390/net/qeth_l2_main.c | 2 + kernel/drivers/s390/net/qeth_l3_main.c | 2 + kernel/drivers/s390/scsi/zfcp_dbf.c | 177 +++++++- kernel/drivers/s390/scsi/zfcp_dbf.h | 53 ++- kernel/drivers/s390/scsi/zfcp_erp.c | 69 ++- kernel/drivers/s390/scsi/zfcp_ext.h | 10 +- kernel/drivers/s390/scsi/zfcp_fsf.c | 18 +- kernel/drivers/s390/scsi/zfcp_fsf.h | 5 +- kernel/drivers/s390/scsi/zfcp_reqlist.h | 30 +- kernel/drivers/s390/scsi/zfcp_scsi.c | 67 ++- kernel/drivers/scsi/53c700.c | 4 +- kernel/drivers/scsi/aacraid/aacraid.h | 3 + kernel/drivers/scsi/aacraid/commctrl.c | 13 +- kernel/drivers/scsi/aacraid/comminit.c | 28 ++ kernel/drivers/scsi/aacraid/commsup.c | 49 ++- kernel/drivers/scsi/aacraid/linit.c | 12 +- kernel/drivers/scsi/aacraid/src.c | 30 +- kernel/drivers/scsi/aic7xxx/aic7xxx_osm.c | 1 + kernel/drivers/scsi/arcmsr/arcmsr_hba.c | 47 ++- kernel/drivers/scsi/be2iscsi/be_main.c | 1 + kernel/drivers/scsi/constants.c | 5 +- kernel/drivers/scsi/cxlflash/common.h | 2 + kernel/drivers/scsi/cxlflash/main.c | 69 ++- kernel/drivers/scsi/cxlflash/main.h | 4 +- kernel/drivers/scsi/cxlflash/superpipe.c | 19 +- kernel/drivers/scsi/cxlflash/vlun.c | 2 + kernel/drivers/scsi/device_handler/Kconfig | 8 +- kernel/drivers/scsi/fcoe/fcoe.c | 2 +- kernel/drivers/scsi/fnic/fnic_fcs.c | 8 +- kernel/drivers/scsi/hpsa.c | 78 +++- kernel/drivers/scsi/ibmvscsi/ibmvfc.c | 1 - kernel/drivers/scsi/ipr.c | 11 +- kernel/drivers/scsi/lpfc/lpfc_crtn.h | 1 + kernel/drivers/scsi/lpfc/lpfc_els.c | 373 ++++++++--------- kernel/drivers/scsi/lpfc/lpfc_hbadisc.c | 20 +- kernel/drivers/scsi/lpfc/lpfc_init.c | 14 +- kernel/drivers/scsi/lpfc/lpfc_mbox.c | 10 +- kernel/drivers/scsi/lpfc/lpfc_nportdisc.c | 134 +++--- kernel/drivers/scsi/lpfc/lpfc_scsi.c | 6 +- kernel/drivers/scsi/lpfc/lpfc_sli.c | 23 +- kernel/drivers/scsi/megaraid/megaraid_sas.h | 4 +- kernel/drivers/scsi/megaraid/megaraid_sas_base.c | 300 +++++-------- kernel/drivers/scsi/megaraid/megaraid_sas_fusion.c | 12 +- kernel/drivers/scsi/mpt3sas/mpt3sas_base.c | 46 +- kernel/drivers/scsi/mpt3sas/mpt3sas_base.h | 5 +- kernel/drivers/scsi/mpt3sas/mpt3sas_ctl.c | 2 +- kernel/drivers/scsi/mpt3sas/mpt3sas_scsih.c | 41 +- kernel/drivers/scsi/mvsas/mv_94xx.c | 2 +- kernel/drivers/scsi/qla1280.c | 2 +- kernel/drivers/scsi/qla2xxx/qla_os.c | 18 +- kernel/drivers/scsi/qla2xxx/qla_target.c | 2 +- kernel/drivers/scsi/scsi_common.c | 12 +- kernel/drivers/scsi/scsi_debug.c | 1 + kernel/drivers/scsi/scsi_devinfo.c | 11 +- kernel/drivers/scsi/scsi_error.c | 4 +- kernel/drivers/scsi/scsi_lib.c | 7 +- kernel/drivers/scsi/scsi_scan.c | 3 +- kernel/drivers/scsi/scsi_sysfs.c | 17 +- kernel/drivers/scsi/sd.c | 38 +- kernel/drivers/scsi/sd.h | 12 +- kernel/drivers/scsi/sg.c | 6 +- kernel/drivers/scsi/storvsc_drv.c | 5 +- kernel/drivers/soc/qcom/spm.c | 2 +- kernel/drivers/soc/rockchip/pm_domains.c | 1 + kernel/drivers/spi/spi-fsl-dspi.c | 1 - kernel/drivers/spi/spi-orion.c | 83 ++-- kernel/drivers/spi/spi-pxa2xx.c | 11 +- kernel/drivers/spi/spi-rockchip.c | 7 +- kernel/drivers/spi/spi-sh-msiof.c | 3 + kernel/drivers/spi/spi-sun4i.c | 23 +- kernel/drivers/spi/spi-sun6i.c | 10 +- kernel/drivers/spi/spi-ti-qspi.c | 45 +- kernel/drivers/ssb/pci.c | 1 + kernel/drivers/staging/android/ion/ion.c | 4 +- kernel/drivers/staging/android/ion/ion_test.c | 4 +- .../drivers/staging/comedi/drivers/comedi_test.c | 46 +- .../drivers/staging/comedi/drivers/daqboard2000.c | 2 +- kernel/drivers/staging/comedi/drivers/das1800.c | 22 +- kernel/drivers/staging/comedi/drivers/dt282x.c | 65 +-- .../drivers/staging/comedi/drivers/ni_mio_common.c | 32 +- kernel/drivers/staging/comedi/drivers/ni_tiocmd.c | 2 +- kernel/drivers/staging/fbtft/fbtft-core.c | 4 +- kernel/drivers/staging/iio/accel/sca3000_core.c | 2 +- kernel/drivers/staging/iio/adc/ad7192.c | 2 +- kernel/drivers/staging/iio/adc/ad7606_core.c | 2 +- .../staging/iio/impedance-analyzer/ad5933.c | 17 +- .../staging/lustre/lustre/llite/llite_internal.h | 2 - kernel/drivers/staging/nvec/nvec_ps2.c | 8 +- kernel/drivers/staging/rdma/hfi1/TODO | 2 +- kernel/drivers/staging/rdma/hfi1/file_ops.c | 6 + kernel/drivers/staging/rdma/ipath/ipath_file_ops.c | 5 + kernel/drivers/staging/rtl8188eu/core/rtw_cmd.c | 4 +- kernel/drivers/target/iscsi/iscsi_target.c | 22 +- kernel/drivers/target/iscsi/iscsi_target_login.c | 5 +- kernel/drivers/target/iscsi/iscsi_target_tpg.c | 1 - kernel/drivers/target/target_core_device.c | 18 +- kernel/drivers/target/target_core_file.c | 3 +- kernel/drivers/target/target_core_iblock.c | 3 +- kernel/drivers/target/target_core_internal.h | 1 + kernel/drivers/target/target_core_sbc.c | 10 +- kernel/drivers/target/target_core_transport.c | 108 +++-- kernel/drivers/target/target_core_user.c | 2 - kernel/drivers/target/target_core_xcopy.c | 36 +- kernel/drivers/thermal/cpu_cooling.c | 16 +- kernel/drivers/thermal/rockchip_thermal.c | 11 +- kernel/drivers/thermal/thermal_core.c | 13 +- kernel/drivers/thermal/thermal_hwmon.c | 2 +- kernel/drivers/thermal/x86_pkg_temp_thermal.c | 2 +- kernel/drivers/thunderbolt/eeprom.c | 1 + kernel/drivers/tty/n_gsm.c | 4 +- kernel/drivers/tty/n_hdlc.c | 4 +- kernel/drivers/tty/n_tty.c | 70 ++-- kernel/drivers/tty/pty.c | 67 ++- kernel/drivers/tty/serial/8250/8250_dw.c | 2 +- kernel/drivers/tty/serial/8250/8250_mid.c | 38 +- kernel/drivers/tty/serial/8250/8250_pci.c | 142 +++++++ kernel/drivers/tty/serial/8250/8250_port.c | 22 +- kernel/drivers/tty/serial/atmel_serial.c | 58 ++- kernel/drivers/tty/serial/msm_serial.c | 2 +- kernel/drivers/tty/serial/samsung.c | 22 +- kernel/drivers/tty/serial/sh-sci.c | 39 -- kernel/drivers/tty/serial/sunhv.c | 6 - kernel/drivers/tty/serial/ucc_uart.c | 3 + kernel/drivers/tty/sysrq.c | 4 +- kernel/drivers/tty/tty_buffer.c | 34 +- kernel/drivers/tty/tty_ldisc.c | 7 + kernel/drivers/tty/vt/keyboard.c | 32 +- kernel/drivers/tty/vt/vt.c | 13 +- kernel/drivers/uio/uio_dmem_genirq.c | 2 +- kernel/drivers/usb/chipidea/core.c | 1 + kernel/drivers/usb/chipidea/udc.c | 18 +- kernel/drivers/usb/class/cdc-acm.c | 11 +- kernel/drivers/usb/class/cdc-acm.h | 1 - kernel/drivers/usb/class/usbtmc.c | 3 +- kernel/drivers/usb/common/common.c | 1 + kernel/drivers/usb/common/usb-otg-fsm.c | 2 + kernel/drivers/usb/core/config.c | 107 ++++- kernel/drivers/usb/core/devices.c | 10 +- kernel/drivers/usb/core/devio.c | 25 +- kernel/drivers/usb/core/driver.c | 46 +- kernel/drivers/usb/core/hcd-pci.c | 11 +- kernel/drivers/usb/core/hcd.c | 6 +- kernel/drivers/usb/core/hub.c | 207 ++++----- kernel/drivers/usb/core/quirks.c | 30 +- kernel/drivers/usb/core/urb.c | 3 +- kernel/drivers/usb/core/usb.h | 2 +- kernel/drivers/usb/dwc2/core.h | 27 ++ kernel/drivers/usb/dwc3/core.h | 4 +- kernel/drivers/usb/dwc3/dwc3-exynos.c | 19 +- kernel/drivers/usb/dwc3/dwc3-pci.c | 4 + kernel/drivers/usb/dwc3/ep0.c | 50 ++- kernel/drivers/usb/dwc3/gadget.c | 35 +- kernel/drivers/usb/gadget/composite.c | 23 +- kernel/drivers/usb/gadget/function/f_fs.c | 30 +- .../drivers/usb/gadget/function/f_mass_storage.c | 36 +- .../drivers/usb/gadget/function/f_mass_storage.h | 2 - kernel/drivers/usb/gadget/function/f_uac2.c | 15 +- kernel/drivers/usb/gadget/function/u_ether.c | 7 - kernel/drivers/usb/gadget/function/uvc_video.c | 2 +- kernel/drivers/usb/gadget/legacy/acm_ms.c | 4 - kernel/drivers/usb/gadget/legacy/inode.c | 36 +- kernel/drivers/usb/gadget/legacy/mass_storage.c | 4 - kernel/drivers/usb/gadget/legacy/multi.c | 12 - kernel/drivers/usb/gadget/legacy/nokia.c | 7 - kernel/drivers/usb/gadget/udc/dummy_hcd.c | 6 +- kernel/drivers/usb/gadget/udc/fsl_qe_udc.c | 9 +- kernel/drivers/usb/gadget/udc/udc-core.c | 2 +- kernel/drivers/usb/host/ehci-hcd.c | 4 +- kernel/drivers/usb/host/ehci-tegra.c | 2 +- kernel/drivers/usb/host/ohci-hcd.c | 2 +- kernel/drivers/usb/host/ohci-q.c | 3 +- kernel/drivers/usb/host/uhci-pci.c | 4 + kernel/drivers/usb/host/xhci-hub.c | 44 +- kernel/drivers/usb/host/xhci-mem.c | 65 ++- kernel/drivers/usb/host/xhci-pci.c | 24 +- kernel/drivers/usb/host/xhci-plat.c | 3 + kernel/drivers/usb/host/xhci-ring.c | 241 ++++++----- kernel/drivers/usb/host/xhci.c | 63 ++- kernel/drivers/usb/host/xhci.h | 9 +- kernel/drivers/usb/misc/iowarrior.c | 6 + kernel/drivers/usb/misc/legousbtower.c | 35 +- kernel/drivers/usb/misc/usbtest.c | 11 +- kernel/drivers/usb/musb/blackfin.c | 1 + kernel/drivers/usb/musb/musb_core.c | 3 +- kernel/drivers/usb/musb/musb_core.h | 7 + kernel/drivers/usb/musb/musb_dsps.c | 12 + kernel/drivers/usb/musb/musb_host.c | 41 +- kernel/drivers/usb/musb/musbhsdma.h | 2 +- kernel/drivers/usb/phy/phy-am335x-control.c | 2 + kernel/drivers/usb/renesas_usbhs/fifo.c | 26 +- kernel/drivers/usb/renesas_usbhs/mod.c | 11 +- kernel/drivers/usb/renesas_usbhs/mod_gadget.c | 24 +- kernel/drivers/usb/serial/ch341.c | 84 ++-- kernel/drivers/usb/serial/cp210x.c | 13 +- kernel/drivers/usb/serial/cyberjack.c | 10 + kernel/drivers/usb/serial/cypress_m8.c | 11 +- kernel/drivers/usb/serial/digi_acceleport.c | 19 + kernel/drivers/usb/serial/ftdi_sio.c | 12 +- kernel/drivers/usb/serial/ftdi_sio_ids.h | 31 +- kernel/drivers/usb/serial/garmin_gps.c | 1 + kernel/drivers/usb/serial/io_edgeport.c | 61 ++- kernel/drivers/usb/serial/io_ti.c | 22 +- kernel/drivers/usb/serial/iuu_phoenix.c | 11 + kernel/drivers/usb/serial/keyspan.c | 4 + kernel/drivers/usb/serial/keyspan_pda.c | 14 + kernel/drivers/usb/serial/kl5kusb105.c | 44 +- kernel/drivers/usb/serial/kobil_sct.c | 12 + kernel/drivers/usb/serial/mct_u232.c | 9 +- kernel/drivers/usb/serial/mos7720.c | 54 +-- kernel/drivers/usb/serial/mos7840.c | 16 +- kernel/drivers/usb/serial/mxuport.c | 10 + kernel/drivers/usb/serial/omninet.c | 13 + kernel/drivers/usb/serial/option.c | 199 ++++++++- kernel/drivers/usb/serial/oti6858.c | 16 + kernel/drivers/usb/serial/pl2303.c | 9 + kernel/drivers/usb/serial/pl2303.h | 1 + kernel/drivers/usb/serial/qcserial.c | 1 + kernel/drivers/usb/serial/quatech2.c | 5 +- kernel/drivers/usb/serial/spcp8x5.c | 14 + kernel/drivers/usb/serial/ti_usb_3410_5052.c | 7 + kernel/drivers/usb/serial/usb-serial-simple.c | 3 +- kernel/drivers/usb/serial/usb-serial.c | 7 +- kernel/drivers/usb/storage/transport.c | 7 +- kernel/drivers/usb/storage/uas.c | 22 +- kernel/drivers/usb/storage/unusual_devs.h | 7 + kernel/drivers/usb/storage/unusual_uas.h | 7 + kernel/drivers/usb/storage/usb.c | 5 +- kernel/drivers/usb/usbip/usbip_common.c | 11 + kernel/drivers/uwb/lc-rc.c | 16 +- kernel/drivers/uwb/pal.c | 2 + kernel/drivers/vfio/pci/vfio_pci_intrs.c | 85 ++-- kernel/drivers/vhost/scsi.c | 6 +- kernel/drivers/video/fbdev/Kconfig | 1 - kernel/drivers/video/fbdev/core/fbcmap.c | 26 +- kernel/drivers/video/fbdev/da8xx-fb.c | 7 +- kernel/drivers/video/fbdev/efifb.c | 6 +- kernel/drivers/virtio/virtio_balloon.c | 22 +- kernel/drivers/virtio/virtio_pci_modern.c | 11 +- kernel/drivers/virtio/virtio_ring.c | 16 +- kernel/drivers/vme/bridges/vme_ca91cx42.c | 2 +- kernel/drivers/w1/masters/omap_hdq.c | 2 - kernel/drivers/watchdog/rc32434_wdt.c | 2 +- kernel/drivers/xen/balloon.c | 40 +- kernel/drivers/xen/events/events_base.c | 30 +- kernel/drivers/xen/evtchn.c | 20 +- kernel/drivers/xen/gntdev.c | 2 +- kernel/drivers/xen/xen-acpi-processor.c | 35 +- kernel/drivers/xen/xen-pciback/conf_space.c | 6 +- kernel/drivers/xen/xenbus/xenbus_dev_frontend.c | 14 +- kernel/drivers/xen/xenbus/xenbus_xs.c | 3 - kernel/fs/9p/acl.c | 40 +- kernel/fs/9p/vfs_file.c | 6 +- kernel/fs/affs/super.c | 5 +- kernel/fs/aio.c | 9 +- kernel/fs/attr.c | 15 + kernel/fs/autofs4/autofs_i.h | 8 +- kernel/fs/autofs4/expire.c | 80 ++-- kernel/fs/autofs4/root.c | 2 +- kernel/fs/block_dev.c | 9 +- kernel/fs/btrfs/acl.c | 6 +- kernel/fs/btrfs/async-thread.c | 14 + kernel/fs/btrfs/async-thread.h | 1 + kernel/fs/btrfs/compression.c | 4 +- kernel/fs/btrfs/ctree.c | 5 +- kernel/fs/btrfs/ctree.h | 5 + kernel/fs/btrfs/delayed-inode.c | 6 +- kernel/fs/btrfs/disk-io.c | 3 +- kernel/fs/btrfs/extent-tree.c | 58 ++- kernel/fs/btrfs/extent_io.c | 15 +- kernel/fs/btrfs/file.c | 39 +- kernel/fs/btrfs/inode.c | 9 +- kernel/fs/btrfs/ioctl.c | 21 +- kernel/fs/btrfs/qgroup.c | 18 +- kernel/fs/btrfs/qgroup.h | 3 +- kernel/fs/btrfs/relocation.c | 27 +- kernel/fs/btrfs/super.c | 2 +- kernel/fs/btrfs/transaction.h | 2 +- kernel/fs/btrfs/tree-log.c | 161 ++++++- kernel/fs/ceph/acl.c | 6 +- kernel/fs/ceph/file.c | 3 +- kernel/fs/ceph/inode.c | 9 +- kernel/fs/ceph/mds_client.c | 9 +- kernel/fs/cifs/cifs_debug.c | 57 +-- kernel/fs/cifs/cifs_debug.h | 2 +- kernel/fs/cifs/cifs_fs_sb.h | 4 + kernel/fs/cifs/cifs_unicode.c | 33 +- kernel/fs/cifs/cifs_unicode.h | 2 + kernel/fs/cifs/cifsencrypt.c | 16 +- kernel/fs/cifs/cifsfs.c | 23 +- kernel/fs/cifs/cifsglob.h | 37 +- kernel/fs/cifs/cifsproto.h | 3 + kernel/fs/cifs/cifssmb.c | 4 +- kernel/fs/cifs/connect.c | 91 +++- kernel/fs/cifs/dir.c | 44 +- kernel/fs/cifs/file.c | 66 +-- kernel/fs/cifs/inode.c | 22 +- kernel/fs/cifs/misc.c | 15 +- kernel/fs/cifs/ntlmssp.h | 2 +- kernel/fs/cifs/readdir.c | 7 +- kernel/fs/cifs/sess.c | 203 +++++---- kernel/fs/cifs/smb2file.c | 2 +- kernel/fs/cifs/smb2glob.h | 11 + kernel/fs/cifs/smb2inode.c | 14 +- kernel/fs/cifs/smb2misc.c | 16 +- kernel/fs/cifs/smb2ops.c | 42 +- kernel/fs/cifs/smb2pdu.c | 107 ++++- kernel/fs/cifs/smb2pdu.h | 2 +- kernel/fs/cifs/smb2proto.h | 3 + kernel/fs/coredump.c | 33 +- kernel/fs/dcache.c | 36 +- kernel/fs/debugfs/inode.c | 2 +- kernel/fs/devpts/inode.c | 49 ++- kernel/fs/dlm/lowcomms.c | 8 +- kernel/fs/ecryptfs/file.c | 86 +++- kernel/fs/exec.c | 32 +- kernel/fs/ext2/acl.c | 12 +- kernel/fs/ext4/acl.c | 12 +- kernel/fs/ext4/balloc.c | 3 + kernel/fs/ext4/crypto_policy.c | 3 + kernel/fs/ext4/ext4.h | 37 ++ kernel/fs/ext4/ext4_jbd2.h | 14 +- kernel/fs/ext4/extents.c | 148 ++++--- kernel/fs/ext4/file.c | 66 ++- kernel/fs/ext4/ialloc.c | 59 ++- kernel/fs/ext4/inline.c | 4 +- kernel/fs/ext4/inode.c | 170 ++++++-- kernel/fs/ext4/ioctl.c | 6 + kernel/fs/ext4/mballoc.c | 78 ++-- kernel/fs/ext4/move_extent.c | 19 +- kernel/fs/ext4/namei.c | 25 +- kernel/fs/ext4/super.c | 196 ++++++--- kernel/fs/ext4/sysfs.c | 4 + kernel/fs/ext4/truncate.h | 2 + kernel/fs/ext4/xattr.c | 66 +-- kernel/fs/f2fs/acl.c | 6 +- kernel/fs/f2fs/crypto_policy.c | 3 + kernel/fs/f2fs/debug.c | 1 + kernel/fs/fhandle.c | 2 +- kernel/fs/fs-writeback.c | 37 +- kernel/fs/fuse/cuse.c | 4 +- kernel/fs/fuse/dev.c | 3 +- kernel/fs/fuse/dir.c | 59 ++- kernel/fs/fuse/file.c | 70 +++- kernel/fs/fuse/fuse_i.h | 9 + kernel/fs/fuse/inode.c | 2 +- kernel/fs/gfs2/acl.c | 12 +- kernel/fs/hfsplus/posix_acl.c | 4 +- kernel/fs/hostfs/hostfs_kern.c | 7 +- kernel/fs/hpfs/super.c | 42 +- kernel/fs/inode.c | 6 +- kernel/fs/isofs/inode.c | 8 +- kernel/fs/isofs/rock.c | 13 +- kernel/fs/jbd2/commit.c | 2 +- kernel/fs/jbd2/journal.c | 17 +- kernel/fs/jbd2/transaction.c | 3 +- kernel/fs/jffs2/acl.c | 9 +- kernel/fs/jfs/acl.c | 6 +- kernel/fs/kernfs/file.c | 28 +- kernel/fs/locks.c | 2 +- kernel/fs/namei.c | 30 +- kernel/fs/namespace.c | 75 +++- kernel/fs/nfs/callback.c | 3 +- kernel/fs/nfs/callback_xdr.c | 6 +- kernel/fs/nfs/delegation.c | 16 +- kernel/fs/nfs/dir.c | 23 +- kernel/fs/nfs/file.c | 2 +- kernel/fs/nfs/filelayout/filelayout.c | 3 +- kernel/fs/nfs/filelayout/filelayoutdev.c | 3 +- kernel/fs/nfs/flexfilelayout/flexfilelayout.c | 3 +- kernel/fs/nfs/inode.c | 2 +- kernel/fs/nfs/nfs42proc.c | 1 + kernel/fs/nfs/nfs4file.c | 4 +- kernel/fs/nfs/nfs4proc.c | 20 +- kernel/fs/nfs/nfs4state.c | 3 + kernel/fs/nfs/pnfs.c | 12 +- kernel/fs/nfs/write.c | 5 +- kernel/fs/nfsd/nfs2acl.c | 20 +- kernel/fs/nfsd/nfs3acl.c | 16 +- kernel/fs/nfsd/nfs4acl.c | 16 +- kernel/fs/nfsd/nfs4callback.c | 18 +- kernel/fs/nfsd/nfs4layouts.c | 5 +- kernel/fs/nfsd/nfs4proc.c | 1 + kernel/fs/nfsd/nfs4state.c | 172 ++++---- kernel/fs/nfsd/nfs4xdr.c | 13 +- kernel/fs/nfsd/state.h | 6 +- kernel/fs/nilfs2/the_nilfs.c | 2 +- kernel/fs/notify/fanotify/fanotify.c | 13 +- kernel/fs/notify/fanotify/fanotify_user.c | 36 +- kernel/fs/notify/group.c | 19 + kernel/fs/notify/notification.c | 23 +- kernel/fs/ocfs2/acl.c | 97 ++++- kernel/fs/ocfs2/acl.h | 5 + kernel/fs/ocfs2/dlm/dlmconvert.c | 26 +- kernel/fs/ocfs2/dlm/dlmrecovery.c | 1 - kernel/fs/ocfs2/dlmglue.c | 10 + kernel/fs/ocfs2/file.c | 38 +- kernel/fs/ocfs2/namei.c | 23 +- kernel/fs/ocfs2/refcounttree.c | 17 +- kernel/fs/ocfs2/stackglue.c | 6 + kernel/fs/ocfs2/stackglue.h | 3 + kernel/fs/ocfs2/xattr.c | 14 +- kernel/fs/ocfs2/xattr.h | 4 +- kernel/fs/open.c | 18 +- kernel/fs/overlayfs/copy_up.c | 16 +- kernel/fs/overlayfs/dir.c | 118 +++--- kernel/fs/overlayfs/inode.c | 25 +- kernel/fs/overlayfs/overlayfs.h | 2 + kernel/fs/overlayfs/super.c | 55 ++- kernel/fs/pipe.c | 47 ++- kernel/fs/pnode.c | 32 +- kernel/fs/posix_acl.c | 81 +++- kernel/fs/proc/base.c | 10 +- kernel/fs/proc/proc_sysctl.c | 3 +- kernel/fs/proc/root.c | 7 + kernel/fs/proc/task_mmu.c | 99 +++-- kernel/fs/proc/task_nommu.c | 49 +-- kernel/fs/proc_namespace.c | 2 + kernel/fs/pstore/inode.c | 1 - kernel/fs/pstore/ram.c | 17 +- kernel/fs/pstore/ram_core.c | 49 +-- kernel/fs/quota/dquot.c | 3 +- kernel/fs/reiserfs/file.c | 6 +- kernel/fs/reiserfs/ibalance.c | 3 +- kernel/fs/reiserfs/namei.c | 18 +- kernel/fs/reiserfs/super.c | 12 +- kernel/fs/reiserfs/xattr.c | 54 --- kernel/fs/reiserfs/xattr.h | 9 +- kernel/fs/reiserfs/xattr_acl.c | 8 +- kernel/fs/reiserfs/xattr_security.c | 14 +- kernel/fs/reiserfs/xattr_trusted.c | 14 +- kernel/fs/reiserfs/xattr_user.c | 14 +- kernel/fs/seq_file.c | 4 +- kernel/fs/splice.c | 3 + kernel/fs/super.c | 6 +- kernel/fs/sysfs/file.c | 8 +- kernel/fs/ubifs/dir.c | 16 +- kernel/fs/ubifs/file.c | 24 ++ kernel/fs/ubifs/tnc.c | 25 +- kernel/fs/ubifs/tnc_commit.c | 2 +- kernel/fs/ubifs/xattr.c | 2 + kernel/fs/utimes.c | 16 +- kernel/fs/xfs/libxfs/xfs_alloc.c | 2 + kernel/fs/xfs/libxfs/xfs_alloc_btree.c | 1 + kernel/fs/xfs/libxfs/xfs_attr_leaf.c | 1 + kernel/fs/xfs/libxfs/xfs_attr_remote.c | 1 + kernel/fs/xfs/libxfs/xfs_bmap_btree.c | 1 + kernel/fs/xfs/libxfs/xfs_da_btree.c | 1 + kernel/fs/xfs/libxfs/xfs_dir2_block.c | 1 + kernel/fs/xfs/libxfs/xfs_dir2_data.c | 2 + kernel/fs/xfs/libxfs/xfs_dir2_leaf.c | 2 + kernel/fs/xfs/libxfs/xfs_dir2_node.c | 1 + kernel/fs/xfs/libxfs/xfs_dquot_buf.c | 40 +- kernel/fs/xfs/libxfs/xfs_ialloc.c | 1 + kernel/fs/xfs/libxfs/xfs_ialloc_btree.c | 1 + kernel/fs/xfs/libxfs/xfs_inode_buf.c | 4 + kernel/fs/xfs/libxfs/xfs_quota_defs.h | 2 +- kernel/fs/xfs/libxfs/xfs_sb.c | 5 +- kernel/fs/xfs/libxfs/xfs_shared.h | 1 + kernel/fs/xfs/libxfs/xfs_symlink_remote.c | 1 + kernel/fs/xfs/xfs_acl.c | 13 +- kernel/fs/xfs/xfs_attr_list.c | 19 +- kernel/fs/xfs/xfs_buf.c | 2 +- kernel/fs/xfs/xfs_buf.h | 1 + kernel/fs/xfs/xfs_error.c | 4 +- kernel/fs/xfs/xfs_fsops.c | 4 +- kernel/fs/xfs/xfs_inode.c | 26 +- kernel/fs/xfs/xfs_log_recover.c | 10 +- kernel/fs/xfs/xfs_super.c | 10 + kernel/include/asm-generic/bitops/lock.h | 14 +- kernel/include/asm-generic/futex.h | 8 +- kernel/include/asm-generic/preempt.h | 4 +- kernel/include/asm-generic/qspinlock.h | 32 +- kernel/include/asm-generic/siginfo.h | 15 - kernel/include/asm-generic/uaccess.h | 20 +- kernel/include/asm-generic/vmlinux.lds.h | 4 + kernel/include/crypto/ghash.h | 23 + kernel/include/drm/drmP.h | 3 +- kernel/include/drm/drm_cache.h | 2 + kernel/include/drm/i915_pciids.h | 6 +- kernel/include/drm/ttm/ttm_bo_api.h | 14 + kernel/include/linux/acpi.h | 2 +- kernel/include/linux/backing-dev-defs.h | 1 + kernel/include/linux/backing-dev.h | 1 + kernel/include/linux/bcma/bcma.h | 1 + kernel/include/linux/bio.h | 3 + kernel/include/linux/blkdev.h | 2 +- kernel/include/linux/bpf.h | 7 +- kernel/include/linux/can/core.h | 7 +- kernel/include/linux/can/dev.h | 25 +- kernel/include/linux/capability.h | 9 +- kernel/include/linux/cgroup-defs.h | 6 +- kernel/include/linux/clk-provider.h | 1 + kernel/include/linux/compiler-gcc.h | 6 +- kernel/include/linux/cpu.h | 15 +- kernel/include/linux/cpumask.h | 8 +- kernel/include/linux/cpuset.h | 6 - kernel/include/linux/cred.h | 5 +- kernel/include/linux/dcache.h | 34 ++ kernel/include/linux/devfreq-event.h | 5 - kernel/include/linux/device-mapper.h | 2 + kernel/include/linux/devpts_fs.h | 34 +- kernel/include/linux/filter.h | 10 +- kernel/include/linux/fs.h | 42 +- kernel/include/linux/fsnotify_backend.h | 6 +- kernel/include/linux/hash.h | 20 +- kernel/include/linux/i8042.h | 6 - kernel/include/linux/if_bridge.h | 4 - kernel/include/linux/init_task.h | 3 +- kernel/include/linux/intel-iommu.h | 1 + kernel/include/linux/irq.h | 10 + kernel/include/linux/irqchip/arm-gic-v3.h | 4 +- kernel/include/linux/jump_label.h | 16 +- kernel/include/linux/jump_label_ratelimit.h | 5 + kernel/include/linux/kernel.h | 53 ++- kernel/include/linux/lightnvm.h | 126 +++--- kernel/include/linux/locallock.h | 6 + kernel/include/linux/memcontrol.h | 8 + kernel/include/linux/memory_hotplug.h | 3 +- kernel/include/linux/mfd/88pm80x.h | 4 +- kernel/include/linux/mfd/cros_ec.h | 15 + kernel/include/linux/mfd/samsung/s2mps11.h | 2 + kernel/include/linux/mfd/ti_am335x_tscadc.h | 8 +- kernel/include/linux/mlx5/device.h | 11 + kernel/include/linux/mlx5/driver.h | 6 +- kernel/include/linux/mlx5/qp.h | 5 +- kernel/include/linux/mm.h | 9 +- kernel/include/linux/mm_types.h | 1 + kernel/include/linux/mmc/dw_mmc.h | 2 +- kernel/include/linux/mroute.h | 2 +- kernel/include/linux/mroute6.h | 2 +- kernel/include/linux/msi.h | 8 +- kernel/include/linux/net.h | 11 +- kernel/include/linux/netdevice.h | 103 ++++- kernel/include/linux/netfilter/x_tables.h | 12 +- kernel/include/linux/nfs4.h | 3 +- kernel/include/linux/nvmem-consumer.h | 2 +- kernel/include/linux/pagemap.h | 38 +- kernel/include/linux/pci.h | 32 +- kernel/include/linux/pci_ids.h | 7 + kernel/include/linux/percpu-refcount.h | 4 +- kernel/include/linux/perf_event.h | 1 + kernel/include/linux/pipe_fs_i.h | 4 + kernel/include/linux/platform_data/asoc-s3c.h | 4 + kernel/include/linux/posix_acl.h | 1 + kernel/include/linux/ptrace.h | 1 - kernel/include/linux/pwm.h | 5 + kernel/include/linux/sched.h | 10 +- kernel/include/linux/sem.h | 1 + kernel/include/linux/serio.h | 24 +- kernel/include/linux/signal.h | 15 + kernel/include/linux/skbuff.h | 68 +++ kernel/include/linux/smc91x.h | 10 + kernel/include/linux/sock_diag.h | 6 + kernel/include/linux/spinlock.h | 6 + kernel/include/linux/spinlock_rt.h | 2 + kernel/include/linux/string.h | 6 +- kernel/include/linux/sunrpc/clnt.h | 3 +- kernel/include/linux/swap.h | 2 + kernel/include/linux/{work-simple.h => swork.h} | 0 kernel/include/linux/sysctl.h | 2 + kernel/include/linux/tcp.h | 7 +- kernel/include/linux/thermal.h | 2 + kernel/include/linux/time.h | 26 ++ kernel/include/linux/tty.h | 4 +- kernel/include/linux/uio.h | 6 +- kernel/include/linux/usb.h | 2 +- kernel/include/linux/usb/ehci_def.h | 4 +- kernel/include/linux/usb_usual.h | 2 + kernel/include/media/videobuf2-core.h | 1 + kernel/include/net/af_unix.h | 2 +- kernel/include/net/bonding.h | 1 + kernel/include/net/cfg80211.h | 11 + kernel/include/net/cipso_ipv4.h | 4 + kernel/include/net/codel.h | 4 + kernel/include/net/ip.h | 4 +- kernel/include/net/ip6_tunnel.h | 1 + kernel/include/net/ip_tunnels.h | 17 + kernel/include/net/ip_vs.h | 17 + kernel/include/net/sch_generic.h | 29 +- kernel/include/net/sock.h | 10 + kernel/include/net/switchdev.h | 2 +- kernel/include/net/tcp.h | 3 + kernel/include/rdma/ib.h | 16 + kernel/include/rdma/ib_addr.h | 6 +- kernel/include/scsi/scsi_device.h | 1 + kernel/include/target/target_core_backend.h | 2 +- kernel/include/target/target_core_base.h | 2 + kernel/include/target/target_core_fabric.h | 1 - kernel/include/trace/events/sunrpc.h | 47 ++- kernel/include/uapi/linux/can.h | 1 + kernel/include/uapi/linux/hyperv.h | 1 + kernel/include/uapi/linux/if.h | 28 ++ kernel/include/uapi/linux/libc-compat.h | 44 ++ kernel/include/uapi/linux/rtnetlink.h | 2 +- kernel/include/uapi/linux/usb/ch9.h | 1 + kernel/include/uapi/linux/v4l2-dv-timings.h | 30 +- kernel/include/uapi/linux/videodev2.h | 3 + kernel/include/uapi/scsi/cxlflash_ioctl.h | 10 + kernel/include/video/imx-ipu-v3.h | 2 + kernel/include/xen/page.h | 4 +- kernel/ipc/msg.c | 2 +- kernel/ipc/sem.c | 142 ++++--- kernel/kernel/audit_watch.c | 8 +- kernel/kernel/auditsc.c | 332 ++++++++------- kernel/kernel/bpf/helpers.c | 2 +- kernel/kernel/bpf/inode.c | 10 +- kernel/kernel/bpf/syscall.c | 24 +- kernel/kernel/bpf/verifier.c | 66 +-- kernel/kernel/capability.c | 82 +++- kernel/kernel/cgroup.c | 38 +- kernel/kernel/configs/tiny.config | 8 + kernel/kernel/cpu.c | 12 +- kernel/kernel/cpuset.c | 36 +- kernel/kernel/cred.c | 2 + kernel/kernel/debug/debug_core.c | 4 +- kernel/kernel/events/core.c | 139 +++--- kernel/kernel/events/ring_buffer.c | 10 +- kernel/kernel/events/uprobes.c | 5 +- kernel/kernel/exit.c | 29 +- kernel/kernel/fork.c | 51 ++- kernel/kernel/futex.c | 29 +- kernel/kernel/irq/generic-chip.c | 21 + kernel/kernel/irq/msi.c | 20 +- kernel/kernel/jump_label.c | 43 +- kernel/kernel/kexec_file.c | 3 + kernel/kernel/locking/mcs_spinlock.h | 8 +- kernel/kernel/locking/mutex.c | 9 +- kernel/kernel/locking/qspinlock.c | 60 +++ kernel/kernel/locking/rtmutex.c | 121 +++++- kernel/kernel/locking/rtmutex_common.h | 5 +- kernel/kernel/memremap.c | 4 + kernel/kernel/module.c | 13 +- kernel/kernel/panic.c | 45 +- kernel/kernel/power/hibernate.c | 5 +- kernel/kernel/power/snapshot.c | 10 +- kernel/kernel/power/suspend_test.c | 4 +- kernel/kernel/printk/braille.c | 4 +- kernel/kernel/printk/printk.c | 10 + kernel/kernel/ptrace.c | 28 +- kernel/kernel/rcu/tree_plugin.h | 1 + kernel/kernel/sched/Makefile | 2 +- kernel/kernel/sched/core.c | 74 ++-- kernel/kernel/sched/cputime.c | 99 ++--- kernel/kernel/sched/fair.c | 78 +++- kernel/kernel/sched/loadavg.c | 11 +- kernel/kernel/sched/sched.h | 13 + kernel/kernel/sched/{work-simple.c => swork.c} | 2 +- kernel/kernel/sysctl.c | 60 ++- kernel/kernel/sysctl_binary.c | 2 +- kernel/kernel/time/clocksource.c | 52 ++- kernel/kernel/time/hrtimer.c | 7 +- kernel/kernel/time/ntp.c | 20 +- kernel/kernel/time/posix-cpu-timers.c | 1 + kernel/kernel/time/tick-broadcast.c | 3 + kernel/kernel/time/timekeeping.c | 35 +- kernel/kernel/time/timekeeping_debug.c | 9 +- kernel/kernel/time/timer.c | 4 +- kernel/kernel/trace/Makefile | 4 + kernel/kernel/trace/ring_buffer.c | 35 +- kernel/kernel/trace/trace.c | 66 +-- kernel/kernel/trace/trace_events.c | 17 +- kernel/kernel/trace/trace_functions_graph.c | 17 +- kernel/kernel/trace/trace_irqsoff.c | 8 +- kernel/kernel/trace/trace_printk.c | 10 +- kernel/kernel/watchdog.c | 12 +- kernel/kernel/workqueue.c | 42 ++ kernel/lib/asn1_decoder.c | 16 +- kernel/lib/assoc_array.c | 4 +- kernel/lib/dma-debug.c | 2 +- kernel/lib/genalloc.c | 3 +- kernel/lib/iov_iter.c | 24 +- kernel/lib/kstrtox.c | 64 +++ kernel/lib/lz4/lz4defs.h | 21 +- kernel/lib/mpi/mpi-pow.c | 7 +- kernel/lib/mpi/mpicoder.c | 60 +-- kernel/lib/string.c | 29 -- kernel/lib/test-string_helpers.c | 67 ++- kernel/localversion-rt | 2 +- kernel/mm/backing-dev.c | 19 + kernel/mm/cma.c | 7 +- kernel/mm/compaction.c | 71 ++-- kernel/mm/filemap.c | 106 ++--- kernel/mm/gup.c | 14 +- kernel/mm/huge_memory.c | 6 +- kernel/mm/hugetlb.c | 57 ++- kernel/mm/init-mm.c | 2 + kernel/mm/internal.h | 3 +- kernel/mm/kasan/kasan.h | 3 + kernel/mm/ksm.c | 3 +- kernel/mm/list_lru.c | 2 + kernel/mm/memcontrol.c | 239 ++++++++--- kernel/mm/memory.c | 40 ++ kernel/mm/memory_hotplug.c | 28 +- kernel/mm/mempolicy.c | 2 +- kernel/mm/migrate.c | 10 +- kernel/mm/page-writeback.c | 27 +- kernel/mm/page_alloc.c | 73 ++-- kernel/mm/page_isolation.c | 8 +- kernel/mm/percpu.c | 73 ++-- kernel/mm/shmem.c | 8 +- kernel/mm/slab_common.c | 4 +- kernel/mm/slub.c | 22 +- kernel/mm/swap.c | 37 +- kernel/mm/swapfile.c | 2 + kernel/mm/util.c | 27 +- kernel/mm/vmscan.c | 37 +- kernel/mm/workingset.c | 10 +- kernel/mm/zsmalloc.c | 13 +- kernel/mm/zswap.c | 38 +- kernel/net/8021q/vlan.c | 2 +- kernel/net/ax25/af_ax25.c | 3 +- kernel/net/ax25/ax25_ds_timer.c | 5 +- kernel/net/ax25/ax25_ip.c | 15 + kernel/net/ax25/ax25_std_timer.c | 5 +- kernel/net/ax25/ax25_subr.c | 3 +- kernel/net/batman-adv/distributed-arp-table.c | 17 +- kernel/net/batman-adv/originator.c | 6 - kernel/net/batman-adv/routing.c | 9 + kernel/net/batman-adv/send.c | 6 + kernel/net/batman-adv/soft-interface.c | 8 +- kernel/net/batman-adv/translation-table.c | 4 +- kernel/net/batman-adv/types.h | 3 - kernel/net/bluetooth/l2cap_sock.c | 2 +- kernel/net/bluetooth/mgmt.c | 4 + kernel/net/bridge/br_fdb.c | 2 + kernel/net/bridge/br_ioctl.c | 5 +- kernel/net/bridge/br_multicast.c | 41 +- kernel/net/bridge/br_netlink.c | 33 +- kernel/net/bridge/br_private.h | 23 +- kernel/net/bridge/br_stp.c | 13 +- kernel/net/caif/cfpkt_skbuff.c | 2 +- kernel/net/can/af_can.c | 12 +- kernel/net/can/af_can.h | 3 +- kernel/net/can/bcm.c | 59 ++- kernel/net/can/gw.c | 2 +- kernel/net/can/raw.c | 7 +- kernel/net/ceph/messenger.c | 13 + kernel/net/ceph/osdmap.c | 156 +++++-- kernel/net/core/dev.c | 139 +++--- kernel/net/core/drop_monitor.c | 39 +- kernel/net/core/filter.c | 61 +-- kernel/net/core/flow_dissector.c | 56 ++- kernel/net/core/neighbour.c | 6 +- kernel/net/core/net_namespace.c | 2 + kernel/net/core/pktgen.c | 38 +- kernel/net/core/rtnetlink.c | 24 +- kernel/net/core/skbuff.c | 15 +- kernel/net/core/sock.c | 5 +- kernel/net/dccp/ipv4.c | 28 +- kernel/net/dccp/ipv6.c | 16 +- kernel/net/dccp/proto.c | 4 + kernel/net/decnet/dn_route.c | 9 +- kernel/net/dsa/slave.c | 2 + kernel/net/ethernet/eth.c | 3 +- kernel/net/ipv4/af_inet.c | 26 +- kernel/net/ipv4/cipso_ipv4.c | 4 + kernel/net/ipv4/devinet.c | 4 + kernel/net/ipv4/esp4.c | 54 ++- kernel/net/ipv4/fib_frontend.c | 28 +- kernel/net/ipv4/fib_semantics.c | 28 +- kernel/net/ipv4/fib_trie.c | 29 +- kernel/net/ipv4/fou.c | 17 +- kernel/net/ipv4/gre_offload.c | 7 +- kernel/net/ipv4/icmp.c | 8 + kernel/net/ipv4/igmp.c | 10 +- kernel/net/ipv4/ip_gre.c | 19 +- kernel/net/ipv4/ip_output.c | 8 +- kernel/net/ipv4/ip_sockglue.c | 19 +- kernel/net/ipv4/ip_tunnel.c | 23 +- kernel/net/ipv4/ip_tunnel_core.c | 3 +- kernel/net/ipv4/ip_vti.c | 31 ++ kernel/net/ipv4/ipmr.c | 7 +- kernel/net/ipv4/netfilter/arp_tables.c | 326 +++++--------- kernel/net/ipv4/netfilter/ip_tables.c | 356 +++++----------- kernel/net/ipv4/netfilter/nf_nat_masquerade_ipv4.c | 12 +- kernel/net/ipv4/ping.c | 6 + kernel/net/ipv4/route.c | 21 +- kernel/net/ipv4/sysctl_net_ipv4.c | 8 +- kernel/net/ipv4/tcp.c | 8 +- kernel/net/ipv4/tcp_dctcp.c | 13 +- kernel/net/ipv4/tcp_fastopen.c | 3 +- kernel/net/ipv4/tcp_input.c | 57 +-- kernel/net/ipv4/tcp_ipv4.c | 36 +- kernel/net/ipv4/tcp_metrics.c | 2 +- kernel/net/ipv4/tcp_minisocks.c | 3 +- kernel/net/ipv4/tcp_output.c | 30 +- kernel/net/ipv4/tcp_yeah.c | 2 +- kernel/net/ipv4/udp.c | 22 +- kernel/net/ipv4/udp_offload.c | 10 +- kernel/net/ipv4/udp_tunnel.c | 2 + kernel/net/ipv6/addrconf.c | 8 +- kernel/net/ipv6/esp6.c | 2 +- kernel/net/ipv6/exthdrs_core.c | 6 +- kernel/net/ipv6/ip6_fib.c | 1 + kernel/net/ipv6/ip6_gre.c | 44 +- kernel/net/ipv6/ip6_offload.c | 18 +- kernel/net/ipv6/ip6_output.c | 19 +- kernel/net/ipv6/ip6_tunnel.c | 53 ++- kernel/net/ipv6/ip6mr.c | 6 +- kernel/net/ipv6/mcast.c | 3 +- kernel/net/ipv6/netfilter/ip6_tables.c | 349 +++++---------- kernel/net/ipv6/output_core.c | 2 + kernel/net/ipv6/ping.c | 9 +- kernel/net/ipv6/raw.c | 6 +- kernel/net/ipv6/reassembly.c | 6 +- kernel/net/ipv6/route.c | 9 +- kernel/net/ipv6/sit.c | 10 +- kernel/net/ipv6/tcp_ipv6.c | 42 +- kernel/net/ipv6/udp.c | 21 +- kernel/net/irda/af_irda.c | 7 +- kernel/net/irda/iriap.c | 8 +- kernel/net/l2tp/l2tp_core.c | 2 +- kernel/net/l2tp/l2tp_core.h | 1 + kernel/net/l2tp/l2tp_ip.c | 40 +- kernel/net/l2tp/l2tp_ip6.c | 15 +- kernel/net/llc/af_llc.c | 1 + kernel/net/mac80211/cfg.c | 2 +- kernel/net/mac80211/ibss.c | 22 +- kernel/net/mac80211/iface.c | 7 +- kernel/net/mac80211/mesh.c | 13 +- kernel/net/mac80211/mlme.c | 21 +- kernel/net/mac80211/rx.c | 29 +- kernel/net/mac80211/sta_info.c | 37 +- kernel/net/mac80211/sta_info.h | 2 +- kernel/net/mac80211/tx.c | 9 +- kernel/net/mpls/af_mpls.c | 3 + kernel/net/netfilter/ipvs/ip_vs_core.c | 37 +- kernel/net/netfilter/ipvs/ip_vs_pe_sip.c | 6 +- kernel/net/netfilter/ipvs/ip_vs_sync.c | 6 +- kernel/net/netfilter/nf_conntrack_core.c | 4 +- kernel/net/netfilter/nf_log.c | 6 +- kernel/net/netfilter/nft_dynset.c | 6 +- kernel/net/netfilter/x_tables.c | 251 ++++++++++- kernel/net/netlabel/netlabel_kapi.c | 12 +- kernel/net/netlink/af_netlink.c | 39 +- kernel/net/netlink/af_netlink.h | 2 + kernel/net/openvswitch/actions.c | 12 +- kernel/net/openvswitch/conntrack.c | 3 +- kernel/net/openvswitch/vport-netdev.c | 2 +- kernel/net/openvswitch/vport-vxlan.c | 2 + kernel/net/openvswitch/vport.h | 7 - kernel/net/packet/af_packet.c | 75 ++-- kernel/net/rds/recv.c | 2 + kernel/net/rds/tcp.c | 5 +- kernel/net/sched/act_csum.c | 8 +- kernel/net/sched/act_mirred.c | 2 +- kernel/net/sched/act_nat.c | 18 +- kernel/net/sched/act_pedit.c | 24 +- kernel/net/sched/act_vlan.c | 9 + kernel/net/sched/cls_api.c | 7 +- kernel/net/sched/cls_basic.c | 4 - kernel/net/sched/cls_bpf.c | 4 - kernel/net/sched/cls_cgroup.c | 7 +- kernel/net/sched/cls_flow.c | 1 - kernel/net/sched/cls_flower.c | 31 +- kernel/net/sched/cls_rsvp.h | 3 +- kernel/net/sched/cls_tcindex.c | 1 - kernel/net/sched/sch_api.c | 8 +- kernel/net/sched/sch_cbq.c | 12 +- kernel/net/sched/sch_choke.c | 6 +- kernel/net/sched/sch_codel.c | 10 +- kernel/net/sched/sch_drr.c | 9 +- kernel/net/sched/sch_dsmark.c | 11 +- kernel/net/sched/sch_fifo.c | 4 + kernel/net/sched/sch_fq.c | 4 +- kernel/net/sched/sch_fq_codel.c | 17 +- kernel/net/sched/sch_generic.c | 5 +- kernel/net/sched/sch_hfsc.c | 9 +- kernel/net/sched/sch_hhf.c | 10 +- kernel/net/sched/sch_htb.c | 24 +- kernel/net/sched/sch_multiq.c | 16 +- kernel/net/sched/sch_netem.c | 82 +++- kernel/net/sched/sch_pie.c | 5 +- kernel/net/sched/sch_prio.c | 15 +- kernel/net/sched/sch_qfq.c | 9 +- kernel/net/sched/sch_red.c | 10 +- kernel/net/sched/sch_sfb.c | 10 +- kernel/net/sched/sch_sfq.c | 16 +- kernel/net/sched/sch_tbf.c | 15 +- kernel/net/sctp/ipv6.c | 2 + kernel/net/sctp/sm_statefuns.c | 12 +- kernel/net/sctp/socket.c | 15 +- kernel/net/socket.c | 40 +- kernel/net/sunrpc/auth_gss/auth_gss.c | 8 +- kernel/net/sunrpc/auth_gss/gss_rpc_xdr.c | 2 +- kernel/net/sunrpc/auth_gss/svcauth_gss.c | 6 +- kernel/net/sunrpc/cache.c | 6 +- kernel/net/sunrpc/clnt.c | 17 +- kernel/net/sunrpc/sunrpc_syms.c | 1 + kernel/net/sunrpc/svc.c | 8 +- kernel/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 2 - kernel/net/sunrpc/xprtsock.c | 15 +- kernel/net/switchdev/switchdev.c | 6 +- kernel/net/sysctl_net.c | 2 +- kernel/net/tipc/link.c | 2 + kernel/net/tipc/name_distr.c | 1 + kernel/net/tipc/netlink_compat.c | 5 +- kernel/net/tipc/socket.c | 39 +- kernel/net/tipc/subscr.c | 3 +- kernel/net/tipc/udp_media.c | 5 - kernel/net/unix/af_unix.c | 141 +++---- kernel/net/vmw_vsock/af_vsock.c | 21 +- kernel/net/wireless/core.h | 2 + kernel/net/wireless/mlme.c | 12 + kernel/net/wireless/nl80211.c | 20 +- kernel/net/wireless/scan.c | 69 +++ kernel/net/wireless/sme.c | 14 + kernel/net/x25/x25_facilities.c | 1 + kernel/net/xfrm/xfrm_input.c | 3 + kernel/samples/bpf/trace_output_kern.c | 1 - kernel/scripts/Makefile.extrawarn | 1 + .../coccinelle/iterators/use_after_iter.cocci | 2 +- kernel/scripts/gcc-x86_64-has-stack-protector.sh | 2 +- kernel/scripts/kconfig/Makefile | 4 +- kernel/scripts/kconfig/confdata.c | 12 +- kernel/scripts/kconfig/nconf.gui.c | 15 +- kernel/scripts/mod/file2alias.c | 2 +- kernel/scripts/package/mkspec | 8 +- kernel/scripts/recordmcount.c | 9 +- kernel/security/apparmor/apparmorfs.c | 1 + kernel/security/apparmor/domain.c | 6 +- kernel/security/integrity/ima/ima_api.c | 2 +- kernel/security/integrity/ima/ima_appraise.c | 4 +- kernel/security/keys/key.c | 2 +- kernel/security/keys/proc.c | 2 +- kernel/security/selinux/hooks.c | 2 +- kernel/sound/core/control.c | 2 + kernel/sound/core/info.c | 9 +- kernel/sound/core/pcm.c | 14 +- kernel/sound/core/pcm_lib.c | 4 +- kernel/sound/core/rawmidi.c | 4 +- kernel/sound/core/seq/seq_memory.c | 9 +- kernel/sound/core/seq/seq_queue.c | 33 +- kernel/sound/core/timer.c | 29 +- kernel/sound/drivers/dummy.c | 1 + kernel/sound/firewire/fireworks/fireworks.h | 1 - kernel/sound/firewire/fireworks/fireworks_hwdep.c | 71 +++- kernel/sound/firewire/fireworks/fireworks_proc.c | 4 +- .../firewire/fireworks/fireworks_transaction.c | 5 +- kernel/sound/firewire/tascam/tascam-hwdep.c | 33 +- kernel/sound/firewire/tascam/tascam-stream.c | 2 +- kernel/sound/hda/array.c | 4 +- kernel/sound/pci/ali5451/ali5451.c | 2 + kernel/sound/pci/au88x0/au88x0_core.c | 5 +- kernel/sound/pci/echoaudio/echoaudio.c | 4 +- kernel/sound/pci/hda/dell_wmi_helper.c | 2 +- kernel/sound/pci/hda/hda_auto_parser.c | 4 +- kernel/sound/pci/hda/hda_controller.c | 7 +- kernel/sound/pci/hda/hda_controller.h | 4 +- kernel/sound/pci/hda/hda_generic.c | 8 +- kernel/sound/pci/hda/hda_intel.c | 80 +++- kernel/sound/pci/hda/hda_sysfs.c | 8 - kernel/sound/pci/hda/hda_tegra.c | 5 +- kernel/sound/pci/hda/patch_ca0132.c | 1 + kernel/sound/pci/hda/patch_cirrus.c | 22 +- kernel/sound/pci/hda/patch_conexant.c | 34 +- kernel/sound/pci/hda/patch_hdmi.c | 9 +- kernel/sound/pci/hda/patch_realtek.c | 279 +++++++++++- kernel/sound/pci/hda/thinkpad_helper.c | 2 +- kernel/sound/pci/intel8x0.c | 1 + kernel/sound/pci/pcxhr/pcxhr_core.c | 1 + kernel/sound/soc/atmel/atmel_ssc_dai.c | 5 +- kernel/sound/soc/codecs/ak4642.c | 3 + kernel/sound/soc/codecs/cs4270.c | 8 +- kernel/sound/soc/codecs/rt5640.c | 2 +- kernel/sound/soc/codecs/rt5640.h | 36 +- kernel/sound/soc/codecs/ssm4567.c | 5 + .../sound/soc/intel/atom/sst-mfld-platform-pcm.c | 6 + kernel/sound/soc/intel/atom/sst/sst_pvt.c | 14 +- kernel/sound/soc/intel/skylake/skl.c | 4 +- kernel/sound/soc/omap/omap-mcpdm.c | 5 +- kernel/sound/soc/samsung/ac97.c | 26 +- kernel/sound/soc/samsung/dma.h | 2 +- kernel/sound/soc/samsung/dmaengine.c | 4 +- kernel/sound/soc/samsung/i2s.c | 31 +- kernel/sound/soc/samsung/pcm.c | 20 +- kernel/sound/soc/samsung/s3c-i2s-v2.c | 2 +- kernel/sound/soc/samsung/s3c-i2s-v2.h | 2 +- kernel/sound/soc/samsung/s3c2412-i2s.c | 4 +- kernel/sound/soc/samsung/s3c24xx-i2s.c | 4 +- kernel/sound/soc/samsung/spdif.c | 10 +- kernel/sound/soc/soc-dapm.c | 13 +- kernel/sound/soc/soc-topology.c | 1 + kernel/sound/soc/sunxi/sun4i-codec.c | 7 +- kernel/sound/usb/card.c | 1 - kernel/sound/usb/clock.c | 2 + kernel/sound/usb/endpoint.c | 20 +- kernel/sound/usb/endpoint.h | 2 +- kernel/sound/usb/hiface/pcm.c | 2 + kernel/sound/usb/line6/driver.c | 2 +- kernel/sound/usb/line6/pcm.c | 3 +- kernel/sound/usb/line6/pod.c | 12 +- kernel/sound/usb/mixer.c | 3 +- kernel/sound/usb/mixer_maps.c | 14 + kernel/sound/usb/mixer_quirks.c | 26 +- kernel/sound/usb/pcm.c | 12 +- kernel/sound/usb/quirks-table.h | 17 + kernel/sound/usb/quirks.c | 36 +- kernel/sound/usb/stream.c | 6 +- kernel/tools/hv/Makefile | 2 + kernel/tools/hv/hv_fcopy_daemon.c | 20 +- kernel/tools/lib/traceevent/parse-filter.c | 4 +- kernel/tools/perf/Documentation/perf-stat.txt | 8 + kernel/tools/perf/Makefile.perf | 2 +- kernel/tools/perf/arch/x86/util/intel-pt.c | 6 +- kernel/tools/perf/tests/bpf.c | 14 +- kernel/tools/perf/tests/llvm.c | 8 +- kernel/tools/perf/ui/browsers/hists.c | 11 +- kernel/tools/perf/util/event.c | 2 +- kernel/tools/perf/util/evlist.c | 4 +- .../perf/util/intel-pt-decoder/intel-pt-decoder.c | 38 ++ kernel/tools/perf/util/intel-pt.c | 17 +- kernel/tools/perf/util/parse-events.c | 6 +- kernel/tools/perf/util/pmu.c | 15 +- kernel/tools/perf/util/setup.py | 4 + kernel/tools/perf/util/stat.c | 10 + kernel/tools/perf/util/symbol-elf.c | 3 +- kernel/tools/perf/util/symbol.c | 5 +- kernel/tools/perf/util/trace-event-scripting.c | 6 +- kernel/tools/testing/nvdimm/test/nfit.c | 2 + kernel/tools/testing/selftests/Makefile | 2 +- .../tools/testing/selftests/net/run_netsocktests | 2 +- .../powerpc/pmu/ebb/pmc56_overflow_test.c | 2 +- kernel/tools/vm/slabinfo.c | 3 +- kernel/virt/kvm/arm/arch_timer.c | 49 ++- kernel/virt/kvm/irqchip.c | 2 +- kernel/virt/kvm/kvm_main.c | 25 +- kernel/virt/lib/irqbypass.c | 4 +- 2347 files changed, 30013 insertions(+), 15192 deletions(-) create mode 100644 kernel/arch/sparc/kernel/urtt_fill.S delete mode 100644 kernel/arch/sparc/lib/user_fixup.c create mode 100644 kernel/include/crypto/ghash.h rename kernel/include/linux/{work-simple.h => swork.h} (100%) rename kernel/kernel/sched/{work-simple.c => swork.c} (99%) diff --git a/kernel/Documentation/ABI/testing/sysfs-bus-iio-proximity-as3935 b/kernel/Documentation/ABI/testing/sysfs-bus-iio-proximity-as3935 index 6708c5e26..33e96f740 100644 --- a/kernel/Documentation/ABI/testing/sysfs-bus-iio-proximity-as3935 +++ b/kernel/Documentation/ABI/testing/sysfs-bus-iio-proximity-as3935 @@ -1,4 +1,4 @@ -What /sys/bus/iio/devices/iio:deviceX/in_proximity_raw +What /sys/bus/iio/devices/iio:deviceX/in_proximity_input Date: March 2014 KernelVersion: 3.15 Contact: Matt Ranostay diff --git a/kernel/Documentation/devicetree/bindings/ata/ahci-platform.txt b/kernel/Documentation/devicetree/bindings/ata/ahci-platform.txt index c2340eeeb..c000832a7 100644 --- a/kernel/Documentation/devicetree/bindings/ata/ahci-platform.txt +++ b/kernel/Documentation/devicetree/bindings/ata/ahci-platform.txt @@ -30,6 +30,10 @@ Optional properties: - target-supply : regulator for SATA target power - phys : reference to the SATA PHY node - phy-names : must be "sata-phy" +- ports-implemented : Mask that indicates which ports that the HBA supports + are available for software to use. Useful if PORTS_IMPL + is not programmed by the BIOS, which is true with + some embedded SOC's. Required properties when using sub-nodes: - #address-cells : number of cells to encode an address diff --git a/kernel/Documentation/devicetree/bindings/clock/imx31-clock.txt b/kernel/Documentation/devicetree/bindings/clock/imx31-clock.txt index 19df842c6..8163d565f 100644 --- a/kernel/Documentation/devicetree/bindings/clock/imx31-clock.txt +++ b/kernel/Documentation/devicetree/bindings/clock/imx31-clock.txt @@ -77,7 +77,7 @@ Examples: clks: ccm@53f80000{ compatible = "fsl,imx31-ccm"; reg = <0x53f80000 0x4000>; - interrupts = <0 31 0x04 0 53 0x04>; + interrupts = <31>, <53>; #clock-cells = <1>; }; diff --git a/kernel/Documentation/devicetree/bindings/clock/imx35-clock.txt b/kernel/Documentation/devicetree/bindings/clock/imx35-clock.txt index a70356452..f49783213 100644 --- a/kernel/Documentation/devicetree/bindings/clock/imx35-clock.txt +++ b/kernel/Documentation/devicetree/bindings/clock/imx35-clock.txt @@ -94,6 +94,7 @@ clocks and IDs. csi_sel 79 iim_gate 80 gpu2d_gate 81 + ckli_gate 82 Examples: diff --git a/kernel/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.txt b/kernel/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.txt index a9a5fe19f..ec9d65682 100644 --- a/kernel/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.txt +++ b/kernel/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.txt @@ -12,6 +12,11 @@ Required properties: - vref-supply: The regulator supply ADC reference voltage. - #io-channel-cells: Should be 1, see ../iio-bindings.txt +Optional properties: +- resets: Must contain an entry for each entry in reset-names if need support + this option. See ../reset/reset.txt for details. +- reset-names: Must include the name "saradc-apb". + Example: saradc: saradc@2006c000 { compatible = "rockchip,saradc"; @@ -19,6 +24,8 @@ Example: interrupts = ; clocks = <&cru SCLK_SARADC>, <&cru PCLK_SARADC>; clock-names = "saradc", "apb_pclk"; + resets = <&cru SRST_SARADC>; + reset-names = "saradc-apb"; #io-channel-cells = <1>; vref-supply = <&vcc18>; }; diff --git a/kernel/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt b/kernel/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt index 08a4a32c8..0326154c7 100644 --- a/kernel/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt +++ b/kernel/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt @@ -134,12 +134,12 @@ mfio80 ddr_debug, mips_trace_data, mips_debug mfio81 dreq0, mips_trace_data, eth_debug mfio82 dreq1, mips_trace_data, eth_debug mfio83 mips_pll_lock, mips_trace_data, usb_debug -mfio84 sys_pll_lock, mips_trace_data, usb_debug -mfio85 wifi_pll_lock, mips_trace_data, sdhost_debug -mfio86 bt_pll_lock, mips_trace_data, sdhost_debug -mfio87 rpu_v_pll_lock, dreq2, socif_debug -mfio88 rpu_l_pll_lock, dreq3, socif_debug -mfio89 audio_pll_lock, dreq4, dreq5 +mfio84 audio_pll_lock, mips_trace_data, usb_debug +mfio85 rpu_v_pll_lock, mips_trace_data, sdhost_debug +mfio86 rpu_l_pll_lock, mips_trace_data, sdhost_debug +mfio87 sys_pll_lock, dreq2, socif_debug +mfio88 wifi_pll_lock, dreq3, socif_debug +mfio89 bt_pll_lock, dreq4, dreq5 tck trstn tdi diff --git a/kernel/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt b/kernel/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt index d00bfd862..e0381c287 100644 --- a/kernel/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt +++ b/kernel/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.txt @@ -81,9 +81,9 @@ pm8916: l14, l15, l16, l17, l18 pm8941: - s1, s2, s3, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, - l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2, lvs3, - mvs1, mvs2 + s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, + l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2, lvs3, + 5vs1, 5vs2 The content of each sub-node is defined by the standard binding for regulators - see regulator.txt - with additional custom properties described below: diff --git a/kernel/Documentation/filesystems/proc.txt b/kernel/Documentation/filesystems/proc.txt index 402ab99e4..6716413c1 100644 --- a/kernel/Documentation/filesystems/proc.txt +++ b/kernel/Documentation/filesystems/proc.txt @@ -346,7 +346,7 @@ address perms offset dev inode pathname a7cb1000-a7cb2000 ---p 00000000 00:00 0 a7cb2000-a7eb2000 rw-p 00000000 00:00 0 a7eb2000-a7eb3000 ---p 00000000 00:00 0 -a7eb3000-a7ed5000 rw-p 00000000 00:00 0 [stack:1001] +a7eb3000-a7ed5000 rw-p 00000000 00:00 0 a7ed5000-a8008000 r-xp 00000000 03:00 4222 /lib/libc.so.6 a8008000-a800a000 r--p 00133000 03:00 4222 /lib/libc.so.6 a800a000-a800b000 rw-p 00135000 03:00 4222 /lib/libc.so.6 @@ -378,7 +378,6 @@ is not associated with a file: [heap] = the heap of the program [stack] = the stack of the main process - [stack:1001] = the stack of the thread with tid 1001 [vdso] = the "virtual dynamic shared object", the kernel system call handler @@ -386,10 +385,8 @@ is not associated with a file: The /proc/PID/task/TID/maps is a view of the virtual memory from the viewpoint of the individual tasks of a process. In this file you will see a mapping marked -as [stack] if that task sees it as a stack. This is a key difference from the -content of /proc/PID/maps, where you will see all mappings that are being used -as stack by all of those tasks. Hence, for the example above, the task-level -map, i.e. /proc/PID/task/TID/maps for thread 1001 will look like this: +as [stack] if that task sees it as a stack. Hence, for the example above, the +task-level map, i.e. /proc/PID/task/TID/maps for thread 1001 will look like this: 08048000-08049000 r-xp 00000000 03:00 8312 /opt/test 08049000-0804a000 rw-p 00001000 03:00 8312 /opt/test diff --git a/kernel/Documentation/kernel-parameters.txt b/kernel/Documentation/kernel-parameters.txt index bcaf28e3b..5489dea35 100644 --- a/kernel/Documentation/kernel-parameters.txt +++ b/kernel/Documentation/kernel-parameters.txt @@ -1371,7 +1371,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted. i8042.nopnp [HW] Don't use ACPIPnP / PnPBIOS to discover KBD/AUX controllers i8042.notimeout [HW] Ignore timeout condition signalled by controller - i8042.reset [HW] Reset the controller during init and cleanup + i8042.reset [HW] Reset the controller during init, cleanup and + suspend-to-ram transitions, only during s2r + transitions, or never reset + Format: { 1 | Y | y | 0 | N | n } + 1, Y, y: always reset controller + 0, N, n: don't ever reset controller + Default: only on s2r transitions on x86; most other + architectures force reset to be always executed i8042.unlock [HW] Unlock (ignore) the keylock i8042.kbdreset [HW] Reset device connected to KBD port @@ -3937,6 +3944,8 @@ bytes respectively. Such letter suffixes can also be entirely omitted. sector if the number is odd); i = IGNORE_DEVICE (don't bind to this device); + j = NO_REPORT_LUNS (don't use report luns + command, uas only); l = NOT_LOCKABLE (don't try to lock and unlock ejectable media); m = MAX_SECTORS_64 (don't transfer more diff --git a/kernel/Documentation/mic/mpssd/mpssd.c b/kernel/Documentation/mic/mpssd/mpssd.c index aaeafa18d..c99a75968 100644 --- a/kernel/Documentation/mic/mpssd/mpssd.c +++ b/kernel/Documentation/mic/mpssd/mpssd.c @@ -1538,9 +1538,9 @@ set_cmdline(struct mic_info *mic) len = snprintf(buffer, PATH_MAX, "clocksource=tsc highres=off nohz=off "); - len += snprintf(buffer + len, PATH_MAX, + len += snprintf(buffer + len, PATH_MAX - len, "cpufreq_on;corec6_off;pc3_off;pc6_off "); - len += snprintf(buffer + len, PATH_MAX, + len += snprintf(buffer + len, PATH_MAX - len, "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0", mic->id + 1); diff --git a/kernel/Documentation/module-signing.txt b/kernel/Documentation/module-signing.txt index a78bf1ffa..39b7f612c 100644 --- a/kernel/Documentation/module-signing.txt +++ b/kernel/Documentation/module-signing.txt @@ -271,3 +271,9 @@ Since the private key is used to sign modules, viruses and malware could use the private key to sign modules and compromise the operating system. The private key must be either destroyed or moved to a secure location and not kept in the root node of the kernel source tree. + +If you use the same private key to sign modules for multiple kernel +configurations, you must ensure that the module version information is +sufficient to prevent loading a module into a different kernel. Either +set CONFIG_MODVERSIONS=y or ensure that each configuration has a different +kernel release string by changing EXTRAVERSION or CONFIG_LOCALVERSION. diff --git a/kernel/Documentation/pinctrl.txt b/kernel/Documentation/pinctrl.txt index 4976389e4..dd15a699e 100644 --- a/kernel/Documentation/pinctrl.txt +++ b/kernel/Documentation/pinctrl.txt @@ -831,7 +831,7 @@ separate memory range only intended for GPIO driving, and the register range dealing with pin config and pin multiplexing get placed into a different memory range and a separate section of the data sheet. -A flag "strict" in struct pinctrl_desc is available to check and deny +A flag "strict" in struct pinmux_ops is available to check and deny simultaneous access to the same pin from GPIO and pin multiplexing consumers on hardware of this type. The pinctrl driver should set this flag accordingly. diff --git a/kernel/Documentation/scsi/scsi_eh.txt b/kernel/Documentation/scsi/scsi_eh.txt index 8638f61c8..37eca0079 100644 --- a/kernel/Documentation/scsi/scsi_eh.txt +++ b/kernel/Documentation/scsi/scsi_eh.txt @@ -263,19 +263,23 @@ scmd->allowed. 3. scmd recovered ACTION: scsi_eh_finish_cmd() is invoked to EH-finish scmd - - shost->host_failed-- - clear scmd->eh_eflags - scsi_setup_cmd_retry() - move from local eh_work_q to local eh_done_q LOCKING: none + CONCURRENCY: at most one thread per separate eh_work_q to + keep queue manipulation lockless 4. EH completes ACTION: scsi_eh_flush_done_q() retries scmds or notifies upper - layer of failure. + layer of failure. May be called concurrently but must have + a no more than one thread per separate eh_work_q to + manipulate the queue locklessly - scmd is removed from eh_done_q and scmd->eh_entry is cleared - if retry is necessary, scmd is requeued using scsi_queue_insert() - otherwise, scsi_finish_command() is invoked for scmd + - zero shost->host_failed LOCKING: queue or finish function performs appropriate locking diff --git a/kernel/Documentation/serial/tty.txt b/kernel/Documentation/serial/tty.txt index bc3842dc3..e2dea3dc4 100644 --- a/kernel/Documentation/serial/tty.txt +++ b/kernel/Documentation/serial/tty.txt @@ -213,9 +213,6 @@ TTY_IO_ERROR If set, causes all subsequent userspace read/write TTY_OTHER_CLOSED Device is a pty and the other side has closed. -TTY_OTHER_DONE Device is a pty and the other side has closed and - all pending input processing has been completed. - TTY_NO_WRITE_SPLIT Prevent driver from splitting up writes into smaller chunks. diff --git a/kernel/Documentation/sysctl/fs.txt b/kernel/Documentation/sysctl/fs.txt index 88152f214..302b5ed61 100644 --- a/kernel/Documentation/sysctl/fs.txt +++ b/kernel/Documentation/sysctl/fs.txt @@ -32,6 +32,8 @@ Currently, these files are in /proc/sys/fs: - nr_open - overflowuid - overflowgid +- pipe-user-pages-hard +- pipe-user-pages-soft - protected_hardlinks - protected_symlinks - suid_dumpable @@ -159,6 +161,27 @@ The default is 65534. ============================================================== +pipe-user-pages-hard: + +Maximum total number of pages a non-privileged user may allocate for pipes. +Once this limit is reached, no new pipes may be allocated until usage goes +below the limit again. When set to 0, no limit is applied, which is the default +setting. + +============================================================== + +pipe-user-pages-soft: + +Maximum total number of pages a non-privileged user may allocate for pipes +before the pipe size gets limited to a single page. Once this limit is reached, +new pipes will be limited to a single page in size for this user in order to +limit total memory usage, and trying to increase them using fcntl() will be +denied until usage goes below the limit again. The default value allows to +allocate up to 1024 pipes at their default size. When set to 0, no limit is +applied. + +============================================================== + protected_hardlinks: A long-standing class of security issues is the hardlink-based diff --git a/kernel/Documentation/virtual/kvm/api.txt b/kernel/Documentation/virtual/kvm/api.txt index 092ee9fba..df8ab4fc2 100644 --- a/kernel/Documentation/virtual/kvm/api.txt +++ b/kernel/Documentation/virtual/kvm/api.txt @@ -1991,6 +1991,7 @@ registers, find a list below: PPC | KVM_REG_PPC_TM_VSCR | 32 PPC | KVM_REG_PPC_TM_DSCR | 64 PPC | KVM_REG_PPC_TM_TAR | 64 + PPC | KVM_REG_PPC_TM_XER | 64 | | MIPS | KVM_REG_MIPS_R0 | 64 ... diff --git a/kernel/Documentation/x86/pat.txt b/kernel/Documentation/x86/pat.txt index 54944c71b..2a4ee6302 100644 --- a/kernel/Documentation/x86/pat.txt +++ b/kernel/Documentation/x86/pat.txt @@ -196,3 +196,35 @@ Another, more verbose way of getting PAT related debug messages is with "debugpat" boot parameter. With this parameter, various debug messages are printed to dmesg log. +PAT Initialization +------------------ + +The following table describes how PAT is initialized under various +configurations. The PAT MSR must be updated by Linux in order to support WC +and WT attributes. Otherwise, the PAT MSR has the value programmed in it +by the firmware. Note, Xen enables WC attribute in the PAT MSR for guests. + + MTRR PAT Call Sequence PAT State PAT MSR + ========================================================= + E E MTRR -> PAT init Enabled OS + E D MTRR -> PAT init Disabled - + D E MTRR -> PAT disable Disabled BIOS + D D MTRR -> PAT disable Disabled - + - np/E PAT -> PAT disable Disabled BIOS + - np/D PAT -> PAT disable Disabled - + E !P/E MTRR -> PAT init Disabled BIOS + D !P/E MTRR -> PAT disable Disabled BIOS + !M !P/E MTRR stub -> PAT disable Disabled BIOS + + Legend + ------------------------------------------------ + E Feature enabled in CPU + D Feature disabled/unsupported in CPU + np "nopat" boot option specified + !P CONFIG_X86_PAT option unset + !M CONFIG_MTRR option unset + Enabled PAT state set to enabled + Disabled PAT state set to disabled + OS PAT initializes PAT MSR with OS setting + BIOS PAT keeps PAT MSR with BIOS setting + diff --git a/kernel/MAINTAINERS b/kernel/MAINTAINERS index d826f1b9e..ab65bbecb 100644 --- a/kernel/MAINTAINERS +++ b/kernel/MAINTAINERS @@ -230,13 +230,13 @@ F: kernel/sys_ni.c ABIT UGURU 1,2 HARDWARE MONITOR DRIVER M: Hans de Goede -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/abituguru.c ABIT UGURU 3 HARDWARE MONITOR DRIVER M: Alistair John Strachan -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/abituguru3.c @@ -373,14 +373,14 @@ S: Maintained ADM1025 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/adm1025 F: drivers/hwmon/adm1025.c ADM1029 HARDWARE MONITOR DRIVER M: Corentin Labbe -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/adm1029.c @@ -425,7 +425,7 @@ F: drivers/video/backlight/adp8860_bl.c ADS1015 HARDWARE MONITOR DRIVER M: Dirk Eibach -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/ads1015 F: drivers/hwmon/ads1015.c @@ -438,7 +438,7 @@ F: drivers/macintosh/therm_adt746x.c ADT7475 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/adt7475 F: drivers/hwmon/adt7475.c @@ -615,7 +615,7 @@ F: include/linux/ccp.h AMD FAM15H PROCESSOR POWER MONITORING DRIVER M: Andreas Herrmann -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/fam15h_power F: drivers/hwmon/fam15h_power.c @@ -779,7 +779,7 @@ F: drivers/input/mouse/bcm5974.c APPLE SMC DRIVER M: Henrik Rydberg -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Odd fixes F: drivers/hwmon/applesmc.c @@ -1777,7 +1777,7 @@ F: include/media/as3645a.h ASC7621 HARDWARE MONITOR DRIVER M: George Joseph -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/asc7621 F: drivers/hwmon/asc7621.c @@ -1864,7 +1864,7 @@ F: drivers/net/wireless/ath/carl9170/ ATK0110 HWMON DRIVER M: Luca Tettamanti -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/asus_atk0110.c @@ -2984,7 +2984,7 @@ F: mm/swap_cgroup.c CORETEMP HARDWARE MONITORING DRIVER M: Fenghua Yu -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/coretemp F: drivers/hwmon/coretemp.c @@ -3549,7 +3549,7 @@ T: git git://git.infradead.org/users/vkoul/slave-dma.git DME1737 HARDWARE MONITOR DRIVER M: Juerg Haefliger -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/dme1737 F: drivers/hwmon/dme1737.c @@ -4097,8 +4097,8 @@ F: Documentation/efi-stub.txt F: arch/ia64/kernel/efi.c F: arch/x86/boot/compressed/eboot.[ch] F: arch/x86/include/asm/efi.h -F: arch/x86/platform/efi/* -F: drivers/firmware/efi/* +F: arch/x86/platform/efi/ +F: drivers/firmware/efi/ F: include/linux/efi*.h EFI VARIABLE FILESYSTEM @@ -4262,7 +4262,7 @@ F: include/video/exynos_mipi* F71805F HARDWARE MONITORING DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/f71805f F: drivers/hwmon/f71805f.c @@ -4341,7 +4341,7 @@ F: fs/* FINTEK F75375S HARDWARE MONITOR AND FAN CONTROLLER DRIVER M: Riku Voipio -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/f75375s.c F: include/linux/f75375s.h @@ -4883,8 +4883,8 @@ F: drivers/media/usb/hackrf/ HARDWARE MONITORING M: Jean Delvare M: Guenter Roeck -L: lm-sensors@lm-sensors.org -W: http://www.lm-sensors.org/ +L: linux-hwmon@vger.kernel.org +W: http://hwmon.wiki.kernel.org/ T: quilt http://jdelvare.nerim.net/devel/linux/jdelvare-hwmon/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git S: Maintained @@ -5393,7 +5393,7 @@ F: drivers/usb/atm/ueagle-atm.c INA209 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/ina209 F: Documentation/devicetree/bindings/i2c/ina209.txt @@ -5401,7 +5401,7 @@ F: drivers/hwmon/ina209.c INA2XX HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/ina2xx F: drivers/hwmon/ina2xx.c @@ -5884,7 +5884,7 @@ F: drivers/isdn/hardware/eicon/ IT87 HARDWARE MONITORING DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/it87 F: drivers/hwmon/it87.c @@ -5920,7 +5920,7 @@ F: drivers/media/dvb-frontends/ix2505v* JC42.4 TEMPERATURE SENSOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/jc42.c F: Documentation/hwmon/jc42 @@ -5970,14 +5970,14 @@ F: drivers/tty/serial/jsm/ K10TEMP HARDWARE MONITORING DRIVER M: Clemens Ladisch -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/k10temp F: drivers/hwmon/k10temp.c K8TEMP HARDWARE MONITORING DRIVER M: Rudolf Marek -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/k8temp F: drivers/hwmon/k8temp.c @@ -6485,27 +6485,27 @@ F: net/llc/ LM73 HARDWARE MONITOR DRIVER M: Guillaume Ligneul -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/lm73.c LM78 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/lm78 F: drivers/hwmon/lm78.c LM83 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/lm83 F: drivers/hwmon/lm83.c LM90 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/lm90 F: Documentation/devicetree/bindings/hwmon/lm90.txt @@ -6513,7 +6513,7 @@ F: drivers/hwmon/lm90.c LM95234 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/lm95234 F: drivers/hwmon/lm95234.c @@ -6580,7 +6580,7 @@ F: drivers/scsi/sym53c8xx_2/ LTC4261 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/ltc4261 F: drivers/hwmon/ltc4261.c @@ -6749,28 +6749,28 @@ F: include/uapi/linux/matroxfb.h MAX16065 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/max16065 F: drivers/hwmon/max16065.c MAX20751 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/max20751 F: drivers/hwmon/max20751.c MAX6650 HARDWARE MONITOR AND FAN CONTROLLER DRIVER M: "Hans J. Koch" -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/max6650 F: drivers/hwmon/max6650.c MAX6697 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/max6697 F: Documentation/devicetree/bindings/i2c/max6697.txt @@ -7303,7 +7303,7 @@ F: drivers/scsi/NCR_D700.* NCT6775 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/nct6775 F: drivers/hwmon/nct6775.c @@ -8064,7 +8064,7 @@ F: drivers/video/logo/logo_parisc* PC87360 HARDWARE MONITORING DRIVER M: Jim Cromie -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/pc87360 F: drivers/hwmon/pc87360.c @@ -8076,7 +8076,7 @@ F: drivers/char/pc8736x_gpio.c PC87427 HARDWARE MONITORING DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/pc87427 F: drivers/hwmon/pc87427.c @@ -8415,8 +8415,8 @@ F: drivers/rtc/rtc-puv3.c PMBUS HARDWARE MONITORING DRIVERS M: Guenter Roeck -L: lm-sensors@lm-sensors.org -W: http://www.lm-sensors.org/ +L: linux-hwmon@vger.kernel.org +W: http://hwmon.wiki.kernel.org/ W: http://www.roeck-us.net/linux/drivers/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git S: Maintained @@ -8610,7 +8610,7 @@ F: drivers/media/usb/pwc/* PWM FAN DRIVER M: Kamil Debski -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Supported F: Documentation/devicetree/bindings/hwmon/pwm-fan.txt F: Documentation/hwmon/pwm-fan @@ -9882,28 +9882,28 @@ F: Documentation/devicetree/bindings/media/i2c/nokia,smia.txt SMM665 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/smm665 F: drivers/hwmon/smm665.c SMSC EMC2103 HARDWARE MONITOR DRIVER M: Steve Glendinning -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/emc2103 F: drivers/hwmon/emc2103.c SMSC SCH5627 HARDWARE MONITOR DRIVER M: Hans de Goede -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Supported F: Documentation/hwmon/sch5627 F: drivers/hwmon/sch5627.c SMSC47B397 HARDWARE MONITOR DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/smsc47b397 F: drivers/hwmon/smsc47b397.c @@ -10830,7 +10830,7 @@ F: include/linux/mmc/sh_mobile_sdhi.h TMP401 HARDWARE MONITOR DRIVER M: Guenter Roeck -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/tmp401 F: drivers/hwmon/tmp401.c @@ -11564,14 +11564,14 @@ F: Documentation/networking/vrf.txt VT1211 HARDWARE MONITOR DRIVER M: Juerg Haefliger -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/vt1211 F: drivers/hwmon/vt1211.c VT8231 HARDWARE MONITOR DRIVER M: Roger Lucas -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/vt8231.c @@ -11590,21 +11590,21 @@ F: drivers/w1/ W83791D HARDWARE MONITORING DRIVER M: Marc Hulsman -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/w83791d F: drivers/hwmon/w83791d.c W83793 HARDWARE MONITORING DRIVER M: Rudolf Marek -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/w83793 F: drivers/hwmon/w83793.c W83795 HARDWARE MONITORING DRIVER M: Jean Delvare -L: lm-sensors@lm-sensors.org +L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/w83795.c diff --git a/kernel/Makefile b/kernel/Makefile index d95c4a4c3..a0ecd131c 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -1,6 +1,6 @@ VERSION = 4 PATCHLEVEL = 4 -SUBLEVEL = 6 +SUBLEVEL = 50 EXTRAVERSION = NAME = Blurry Fish Butt @@ -128,6 +128,10 @@ _all: # Cancel implicit rules on top Makefile $(CURDIR)/Makefile Makefile: ; +ifneq ($(words $(subst :, ,$(CURDIR))), 1) + $(error main directory cannot contain spaces nor colons) +endif + ifneq ($(KBUILD_OUTPUT),) # Invoke a second make in the output directory, passing relevant variables # check that the output directory actually exists @@ -364,7 +368,7 @@ AFLAGS_MODULE = LDFLAGS_MODULE = CFLAGS_KERNEL = AFLAGS_KERNEL = -CFLAGS_GCOV = -fprofile-arcs -ftest-coverage +CFLAGS_GCOV = -fprofile-arcs -ftest-coverage -fno-tree-loop-im # Use USERINCLUDE when you must reference the UAPI directories only. @@ -391,11 +395,12 @@ KBUILD_CFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -fno-common \ -Werror-implicit-function-declaration \ -Wno-format-security \ - -std=gnu89 + -std=gnu89 $(call cc-option,-fno-PIE) + KBUILD_AFLAGS_KERNEL := KBUILD_CFLAGS_KERNEL := -KBUILD_AFLAGS := -D__ASSEMBLY__ +KBUILD_AFLAGS := -D__ASSEMBLY__ $(call cc-option,-fno-PIE) KBUILD_AFLAGS_MODULE := -DMODULE KBUILD_CFLAGS_MODULE := -DMODULE KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds @@ -495,6 +500,12 @@ ifeq ($(KBUILD_EXTMOD),) endif endif endif +# install and module_install need also be processed one by one +ifneq ($(filter install,$(MAKECMDGOALS)),) + ifneq ($(filter modules_install,$(MAKECMDGOALS)),) + mixed-targets := 1 + endif +endif ifeq ($(mixed-targets),1) # =========================================================================== @@ -606,11 +617,17 @@ ARCH_CFLAGS := include arch/$(SRCARCH)/Makefile KBUILD_CFLAGS += $(call cc-option,-fno-delete-null-pointer-checks,) +KBUILD_CFLAGS += $(call cc-disable-warning,maybe-uninitialized,) +KBUILD_CFLAGS += $(call cc-disable-warning,frame-address,) ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -KBUILD_CFLAGS += -Os $(call cc-disable-warning,maybe-uninitialized,) +KBUILD_CFLAGS += -Os else +ifdef CONFIG_PROFILE_ALL_BRANCHES KBUILD_CFLAGS += -O2 +else +KBUILD_CFLAGS += -O2 +endif endif # Tell gcc to never replace conditional load with a non-conditional one @@ -682,9 +699,10 @@ KBUILD_CFLAGS += $(call cc-option, -mno-global-merge,) KBUILD_CFLAGS += $(call cc-option, -fcatch-undefined-behavior) else -# This warning generated too much noise in a regular build. -# Use make W=1 to enable this warning (see scripts/Makefile.build) +# These warnings generated too much noise in a regular build. +# Use make W=1 to enable them (see scripts/Makefile.build) KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable) +KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable) endif ifdef CONFIG_FRAME_POINTER @@ -1262,7 +1280,7 @@ help: @echo ' firmware_install- Install all firmware to INSTALL_FW_PATH' @echo ' (default: $$(INSTALL_MOD_PATH)/lib/firmware)' @echo ' dir/ - Build all files in dir and below' - @echo ' dir/file.[oisS] - Build specified target only' + @echo ' dir/file.[ois] - Build specified target only' @echo ' dir/file.lst - Build specified mixed source/assembly target only' @echo ' (requires a recent binutils and recent build (System.map))' @echo ' dir/file.ko - Build module including final link' @@ -1502,11 +1520,11 @@ image_name: # Clear a bunch of variables before executing the submake tools/: FORCE $(Q)mkdir -p $(objtree)/tools - $(Q)$(MAKE) LDFLAGS= MAKEFLAGS="$(filter --j% -j,$(MAKEFLAGS))" O=$(O) subdir=tools -C $(src)/tools/ + $(Q)$(MAKE) LDFLAGS= MAKEFLAGS="$(filter --j% -j,$(MAKEFLAGS))" O=$(shell cd $(objtree) && /bin/pwd) subdir=tools -C $(src)/tools/ tools/%: FORCE $(Q)mkdir -p $(objtree)/tools - $(Q)$(MAKE) LDFLAGS= MAKEFLAGS="$(filter --j% -j,$(MAKEFLAGS))" O=$(O) subdir=tools -C $(src)/tools/ $* + $(Q)$(MAKE) LDFLAGS= MAKEFLAGS="$(filter --j% -j,$(MAKEFLAGS))" O=$(shell cd $(objtree) && /bin/pwd) subdir=tools -C $(src)/tools/ $* # Single targets # --------------------------------------------------------------------------- diff --git a/kernel/arch/alpha/include/asm/uaccess.h b/kernel/arch/alpha/include/asm/uaccess.h index 9b0d40093..c0ddbbf73 100644 --- a/kernel/arch/alpha/include/asm/uaccess.h +++ b/kernel/arch/alpha/include/asm/uaccess.h @@ -371,14 +371,6 @@ __copy_tofrom_user_nocheck(void *to, const void *from, long len) return __cu_len; } -extern inline long -__copy_tofrom_user(void *to, const void *from, long len, const void __user *validate) -{ - if (__access_ok((unsigned long)validate, len, get_fs())) - len = __copy_tofrom_user_nocheck(to, from, len); - return len; -} - #define __copy_to_user(to, from, n) \ ({ \ __chk_user_ptr(to); \ @@ -393,17 +385,22 @@ __copy_tofrom_user(void *to, const void *from, long len, const void __user *vali #define __copy_to_user_inatomic __copy_to_user #define __copy_from_user_inatomic __copy_from_user - extern inline long copy_to_user(void __user *to, const void *from, long n) { - return __copy_tofrom_user((__force void *)to, from, n, to); + if (likely(__access_ok((unsigned long)to, n, get_fs()))) + n = __copy_tofrom_user_nocheck((__force void *)to, from, n); + return n; } extern inline long copy_from_user(void *to, const void __user *from, long n) { - return __copy_tofrom_user(to, (__force void *)from, n, from); + if (likely(__access_ok((unsigned long)from, n, get_fs()))) + n = __copy_tofrom_user_nocheck(to, (__force void *)from, n); + else + memset(to, 0, n); + return n; } extern void __do_clear_user(void); diff --git a/kernel/arch/arc/Kconfig b/kernel/arch/arc/Kconfig index 6312f6079..2d785f5a3 100644 --- a/kernel/arch/arc/Kconfig +++ b/kernel/arch/arc/Kconfig @@ -387,7 +387,7 @@ config ARC_HAS_LLSC config ARC_STAR_9000923308 bool "Workaround for llock/scond livelock" - default y + default n depends on ISA_ARCV2 && SMP && ARC_HAS_LLSC config ARC_HAS_SWAPE diff --git a/kernel/arch/arc/Makefile b/kernel/arch/arc/Makefile index aeb190210..c05ea2b54 100644 --- a/kernel/arch/arc/Makefile +++ b/kernel/arch/arc/Makefile @@ -18,6 +18,20 @@ cflags-y += -fno-common -pipe -fno-builtin -D__linux__ cflags-$(CONFIG_ISA_ARCOMPACT) += -mA7 cflags-$(CONFIG_ISA_ARCV2) += -mcpu=archs +is_700 = $(shell $(CC) -dM -E - < /dev/null | grep -q "ARC700" && echo 1 || echo 0) + +ifdef CONFIG_ISA_ARCOMPACT +ifeq ($(is_700), 0) + $(error Toolchain not configured for ARCompact builds) +endif +endif + +ifdef CONFIG_ISA_ARCV2 +ifeq ($(is_700), 1) + $(error Toolchain not configured for ARCv2 builds) +endif +endif + ifdef CONFIG_ARC_CURR_IN_REG # For a global register defintion, make sure it gets passed to every file # We had a customer reported bug where some code built in kernel was NOT using @@ -48,8 +62,6 @@ endif endif -cflags-$(CONFIG_ARC_DW2_UNWIND) += -fasynchronous-unwind-tables - # By default gcc 4.8 generates dwarf4 which kernel unwinder can't grok ifeq ($(atleast_gcc48),y) cflags-$(CONFIG_ARC_DW2_UNWIND) += -gdwarf-2 diff --git a/kernel/arch/arc/include/asm/arcregs.h b/kernel/arch/arc/include/asm/arcregs.h index 7fac7d85e..2c30a016c 100644 --- a/kernel/arch/arc/include/asm/arcregs.h +++ b/kernel/arch/arc/include/asm/arcregs.h @@ -374,12 +374,6 @@ static inline int is_isa_arcompact(void) return IS_ENABLED(CONFIG_ISA_ARCOMPACT); } -#if defined(CONFIG_ISA_ARCOMPACT) && !defined(_CPU_DEFAULT_A7) -#error "Toolchain not configured for ARCompact builds" -#elif defined(CONFIG_ISA_ARCV2) && !defined(_CPU_DEFAULT_HS) -#error "Toolchain not configured for ARCv2 builds" -#endif - #endif /* __ASEMBLY__ */ #endif /* _ASM_ARC_ARCREGS_H */ diff --git a/kernel/arch/arc/include/asm/bitops.h b/kernel/arch/arc/include/asm/bitops.h index 57c1f3384..0352fb8d2 100644 --- a/kernel/arch/arc/include/asm/bitops.h +++ b/kernel/arch/arc/include/asm/bitops.h @@ -35,21 +35,6 @@ static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ \ m += nr >> 5; \ \ - /* \ - * ARC ISA micro-optimization: \ - * \ - * Instructions dealing with bitpos only consider lower 5 bits \ - * e.g (x << 33) is handled like (x << 1) by ASL instruction \ - * (mem pointer still needs adjustment to point to next word) \ - * \ - * Hence the masking to clamp @nr arg can be elided in general. \ - * \ - * However if @nr is a constant (above assumed in a register), \ - * and greater than 31, gcc can optimize away (x << 33) to 0, \ - * as overflow, given the 32-bit ISA. Thus masking needs to be \ - * done for const @nr, but no code is generated due to gcc \ - * const prop. \ - */ \ nr &= 0x1f; \ \ __asm__ __volatile__( \ diff --git a/kernel/arch/arc/include/asm/cacheflush.h b/kernel/arch/arc/include/asm/cacheflush.h index fbe3587c4..56aeb5efe 100644 --- a/kernel/arch/arc/include/asm/cacheflush.h +++ b/kernel/arch/arc/include/asm/cacheflush.h @@ -85,6 +85,10 @@ void flush_anon_page(struct vm_area_struct *vma, */ #define PG_dc_clean PG_arch_1 +#define CACHE_COLORS_NUM 4 +#define CACHE_COLORS_MSK (CACHE_COLORS_NUM - 1) +#define CACHE_COLOR(addr) (((unsigned long)(addr) >> (PAGE_SHIFT)) & CACHE_COLORS_MSK) + /* * Simple wrapper over config option * Bootup code ensures that hardware matches kernel configuration @@ -94,8 +98,6 @@ static inline int cache_is_vipt_aliasing(void) return IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); } -#define CACHE_COLOR(addr) (((unsigned long)(addr) >> (PAGE_SHIFT)) & 1) - /* * checks if two addresses (after page aligning) index into same cache set */ diff --git a/kernel/arch/arc/include/asm/delay.h b/kernel/arch/arc/include/asm/delay.h index 08e7e2a16..d5da2115d 100644 --- a/kernel/arch/arc/include/asm/delay.h +++ b/kernel/arch/arc/include/asm/delay.h @@ -22,10 +22,13 @@ static inline void __delay(unsigned long loops) { __asm__ __volatile__( - " lp 1f \n" - " nop \n" - "1: \n" - : "+l"(loops)); + " mov lp_count, %0 \n" + " lp 1f \n" + " nop \n" + "1: \n" + : + : "r"(loops) + : "lp_count"); } extern void __bad_udelay(void); diff --git a/kernel/arch/arc/include/asm/entry.h b/kernel/arch/arc/include/asm/entry.h index ad7860c5c..51597f344 100644 --- a/kernel/arch/arc/include/asm/entry.h +++ b/kernel/arch/arc/include/asm/entry.h @@ -142,7 +142,7 @@ #ifdef CONFIG_ARC_CURR_IN_REG ; Retrieve orig r25 and save it with rest of callee_regs - ld.as r12, [r12, PT_user_r25] + ld r12, [r12, PT_user_r25] PUSH r12 #else PUSH r25 @@ -198,7 +198,7 @@ ; SP is back to start of pt_regs #ifdef CONFIG_ARC_CURR_IN_REG - st.as r12, [sp, PT_user_r25] + st r12, [sp, PT_user_r25] #endif .endm diff --git a/kernel/arch/arc/include/asm/io.h b/kernel/arch/arc/include/asm/io.h index 694ece8a0..cb69299a4 100644 --- a/kernel/arch/arc/include/asm/io.h +++ b/kernel/arch/arc/include/asm/io.h @@ -13,6 +13,15 @@ #include #include +#ifdef CONFIG_ISA_ARCV2 +#include +#define __iormb() rmb() +#define __iowmb() wmb() +#else +#define __iormb() do { } while (0) +#define __iowmb() do { } while (0) +#endif + extern void __iomem *ioremap(unsigned long physaddr, unsigned long size); extern void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, unsigned long flags); @@ -22,6 +31,15 @@ extern void iounmap(const void __iomem *addr); #define ioremap_wc(phy, sz) ioremap(phy, sz) #define ioremap_wt(phy, sz) ioremap(phy, sz) +/* + * io{read,write}{16,32}be() macros + */ +#define ioread16be(p) ({ u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(); __v; }) +#define ioread32be(p) ({ u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(); __v; }) + +#define iowrite16be(v,p) ({ __iowmb(); __raw_writew((__force u16)cpu_to_be16(v), p); }) +#define iowrite32be(v,p) ({ __iowmb(); __raw_writel((__force u32)cpu_to_be32(v), p); }) + /* Change struct page to physical address */ #define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) @@ -99,15 +117,6 @@ static inline void __raw_writel(u32 w, volatile void __iomem *addr) } -#ifdef CONFIG_ISA_ARCV2 -#include -#define __iormb() rmb() -#define __iowmb() wmb() -#else -#define __iormb() do { } while (0) -#define __iowmb() do { } while (0) -#endif - /* * MMIO can also get buffered/optimized in micro-arch, so barriers needed * Based on ARM model for the typical use case @@ -129,15 +138,23 @@ static inline void __raw_writel(u32 w, volatile void __iomem *addr) #define writel(v,c) ({ __iowmb(); writel_relaxed(v,c); }) /* - * Relaxed API for drivers which can handle any ordering themselves + * Relaxed API for drivers which can handle barrier ordering themselves + * + * Also these are defined to perform little endian accesses. + * To provide the typical device register semantics of fixed endian, + * swap the byte order for Big Endian + * + * http://lkml.kernel.org/r/201603100845.30602.arnd@arndb.de */ #define readb_relaxed(c) __raw_readb(c) -#define readw_relaxed(c) __raw_readw(c) -#define readl_relaxed(c) __raw_readl(c) +#define readw_relaxed(c) ({ u16 __r = le16_to_cpu((__force __le16) \ + __raw_readw(c)); __r; }) +#define readl_relaxed(c) ({ u32 __r = le32_to_cpu((__force __le32) \ + __raw_readl(c)); __r; }) #define writeb_relaxed(v,c) __raw_writeb(v,c) -#define writew_relaxed(v,c) __raw_writew(v,c) -#define writel_relaxed(v,c) __raw_writel(v,c) +#define writew_relaxed(v,c) __raw_writew((__force u16) cpu_to_le16(v),c) +#define writel_relaxed(v,c) __raw_writel((__force u32) cpu_to_le32(v),c) #include diff --git a/kernel/arch/arc/include/asm/irqflags-compact.h b/kernel/arch/arc/include/asm/irqflags-compact.h index c1d36458b..4c6eed80c 100644 --- a/kernel/arch/arc/include/asm/irqflags-compact.h +++ b/kernel/arch/arc/include/asm/irqflags-compact.h @@ -188,10 +188,10 @@ static inline int arch_irqs_disabled(void) .endm .macro IRQ_ENABLE scratch + TRACE_ASM_IRQ_ENABLE lr \scratch, [status32] or \scratch, \scratch, (STATUS_E1_MASK | STATUS_E2_MASK) flag \scratch - TRACE_ASM_IRQ_ENABLE .endm #endif /* __ASSEMBLY__ */ diff --git a/kernel/arch/arc/include/asm/pgtable.h b/kernel/arch/arc/include/asm/pgtable.h index 57af2f05a..e5fec320f 100644 --- a/kernel/arch/arc/include/asm/pgtable.h +++ b/kernel/arch/arc/include/asm/pgtable.h @@ -110,7 +110,7 @@ #define ___DEF (_PAGE_PRESENT | _PAGE_CACHEABLE) /* Set of bits not changed in pte_modify */ -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY) +#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_SPECIAL) /* More Abbrevaited helpers */ #define PAGE_U_NONE __pgprot(___DEF) @@ -277,8 +277,7 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) #define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) -#define pfn_pte(pfn, prot) (__pte(((pte_t)(pfn) << PAGE_SHIFT) | \ - pgprot_val(prot))) +#define pfn_pte(pfn, prot) (__pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))) #define __pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) /* diff --git a/kernel/arch/arc/include/asm/uaccess.h b/kernel/arch/arc/include/asm/uaccess.h index d1da6032b..d4d8df706 100644 --- a/kernel/arch/arc/include/asm/uaccess.h +++ b/kernel/arch/arc/include/asm/uaccess.h @@ -83,7 +83,10 @@ "2: ;nop\n" \ " .section .fixup, \"ax\"\n" \ " .align 4\n" \ - "3: mov %0, %3\n" \ + "3: # return -EFAULT\n" \ + " mov %0, %3\n" \ + " # zero out dst ptr\n" \ + " mov %1, 0\n" \ " j 2b\n" \ " .previous\n" \ " .section __ex_table, \"a\"\n" \ @@ -101,7 +104,11 @@ "2: ;nop\n" \ " .section .fixup, \"ax\"\n" \ " .align 4\n" \ - "3: mov %0, %3\n" \ + "3: # return -EFAULT\n" \ + " mov %0, %3\n" \ + " # zero out dst ptr\n" \ + " mov %1, 0\n" \ + " mov %R1, 0\n" \ " j 2b\n" \ " .previous\n" \ " .section __ex_table, \"a\"\n" \ diff --git a/kernel/arch/arc/kernel/setup.c b/kernel/arch/arc/kernel/setup.c index e1b87444e..05131805a 100644 --- a/kernel/arch/arc/kernel/setup.c +++ b/kernel/arch/arc/kernel/setup.c @@ -332,10 +332,6 @@ static void arc_chk_core_config(void) pr_warn("CONFIG_ARC_FPU_SAVE_RESTORE needed for working apps\n"); else if (!cpu->extn.fpu_dp && fpu_enabled) panic("FPU non-existent, disable CONFIG_ARC_FPU_SAVE_RESTORE\n"); - - if (is_isa_arcv2() && IS_ENABLED(CONFIG_SMP) && cpu->isa.atomic && - !IS_ENABLED(CONFIG_ARC_STAR_9000923308)) - panic("llock/scond livelock workaround missing\n"); } /* diff --git a/kernel/arch/arc/kernel/signal.c b/kernel/arch/arc/kernel/signal.c index 004b7f0bc..257b8699e 100644 --- a/kernel/arch/arc/kernel/signal.c +++ b/kernel/arch/arc/kernel/signal.c @@ -107,13 +107,13 @@ static int restore_usr_regs(struct pt_regs *regs, struct rt_sigframe __user *sf) struct user_regs_struct uregs; err = __copy_from_user(&set, &sf->uc.uc_sigmask, sizeof(set)); - if (!err) - set_current_blocked(&set); - err |= __copy_from_user(&uregs.scratch, &(sf->uc.uc_mcontext.regs.scratch), sizeof(sf->uc.uc_mcontext.regs.scratch)); + if (err) + return err; + set_current_blocked(&set); regs->bta = uregs.scratch.bta; regs->lp_start = uregs.scratch.lp_start; regs->lp_end = uregs.scratch.lp_end; @@ -138,7 +138,7 @@ static int restore_usr_regs(struct pt_regs *regs, struct rt_sigframe __user *sf) regs->r0 = uregs.scratch.r0; regs->sp = uregs.scratch.sp; - return err; + return 0; } static inline int is_do_ss_needed(unsigned int magic) diff --git a/kernel/arch/arc/kernel/stacktrace.c b/kernel/arch/arc/kernel/stacktrace.c index 001de4ce7..11b50959f 100644 --- a/kernel/arch/arc/kernel/stacktrace.c +++ b/kernel/arch/arc/kernel/stacktrace.c @@ -142,7 +142,7 @@ arc_unwind_core(struct task_struct *tsk, struct pt_regs *regs, * prelogue is setup (callee regs saved and then fp set and not other * way around */ - pr_warn("CONFIG_ARC_DW2_UNWIND needs to be enabled\n"); + pr_warn_once("CONFIG_ARC_DW2_UNWIND needs to be enabled\n"); return 0; #endif diff --git a/kernel/arch/arc/kernel/time.c b/kernel/arch/arc/kernel/time.c index dfad287f1..dbedc576e 100644 --- a/kernel/arch/arc/kernel/time.c +++ b/kernel/arch/arc/kernel/time.c @@ -130,14 +130,17 @@ static cycle_t arc_counter_read(struct clocksource *cs) cycle_t full; } stamp; - - __asm__ __volatile( - "1: \n" - " lr %0, [AUX_RTC_LOW] \n" - " lr %1, [AUX_RTC_HIGH] \n" - " lr %2, [AUX_RTC_CTRL] \n" - " bbit0.nt %2, 31, 1b \n" - : "=r" (stamp.low), "=r" (stamp.high), "=r" (status)); + /* + * hardware has an internal state machine which tracks readout of + * low/high and updates the CTRL.status if + * - interrupt/exception taken between the two reads + * - high increments after low has been read + */ + do { + stamp.low = read_aux_reg(AUX_RTC_LOW); + stamp.high = read_aux_reg(AUX_RTC_HIGH); + status = read_aux_reg(AUX_RTC_CTRL); + } while (!(status & _BITUL(31))); return stamp.full; } diff --git a/kernel/arch/arc/kernel/unaligned.c b/kernel/arch/arc/kernel/unaligned.c index abd961f3e..5f69c3bd5 100644 --- a/kernel/arch/arc/kernel/unaligned.c +++ b/kernel/arch/arc/kernel/unaligned.c @@ -241,8 +241,9 @@ int misaligned_fixup(unsigned long address, struct pt_regs *regs, if (state.fault) goto fault; + /* clear any remanants of delay slot */ if (delay_mode(regs)) { - regs->ret = regs->bta; + regs->ret = regs->bta & ~1U; regs->status32 &= ~STATUS_DE_MASK; } else { regs->ret += state.instr_len; diff --git a/kernel/arch/arc/mm/cache.c b/kernel/arch/arc/mm/cache.c index ff7ff6cbb..d81b6d7e1 100644 --- a/kernel/arch/arc/mm/cache.c +++ b/kernel/arch/arc/mm/cache.c @@ -914,6 +914,15 @@ void arc_cache_init(void) printk(arc_cache_mumbojumbo(0, str, sizeof(str))); + /* + * Only master CPU needs to execute rest of function: + * - Assume SMP so all cores will have same cache config so + * any geomtry checks will be same for all + * - IOC setup / dma callbacks only need to be setup once + */ + if (cpu) + return; + if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; @@ -951,11 +960,16 @@ void arc_cache_init(void) /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ if (is_isa_arcompact()) { int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); - - if (dc->alias && !handled) - panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); - else if (!dc->alias && handled) + int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); + + if (dc->alias) { + if (!handled) + panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); + if (CACHE_COLORS_NUM != num_colors) + panic("CACHE_COLORS_NUM not optimized for config\n"); + } else if (!dc->alias && handled) { panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); + } } } diff --git a/kernel/arch/arm/boot/compressed/head.S b/kernel/arch/arm/boot/compressed/head.S index 06e983f59..856913705 100644 --- a/kernel/arch/arm/boot/compressed/head.S +++ b/kernel/arch/arm/boot/compressed/head.S @@ -776,7 +776,7 @@ __armv7_mmu_cache_on: orrne r0, r0, #1 @ MMU enabled movne r1, #0xfffffffd @ domain 0 = client bic r6, r6, #1 << 31 @ 32-bit translation system - bic r6, r6, #3 << 0 @ use only ttbr0 + bic r6, r6, #(7 << 0) | (1 << 4) @ use only ttbr0 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer mcrne p15, 0, r1, c3, c0, 0 @ load domain access control mcrne p15, 0, r6, c2, c0, 2 @ load ttb control diff --git a/kernel/arch/arm/boot/dts/am43x-epos-evm.dts b/kernel/arch/arm/boot/dts/am43x-epos-evm.dts index 47954ed99..00707aac7 100644 --- a/kernel/arch/arm/boot/dts/am43x-epos-evm.dts +++ b/kernel/arch/arm/boot/dts/am43x-epos-evm.dts @@ -792,3 +792,8 @@ tx-num-evt = <32>; rx-num-evt = <32>; }; + +&synctimer_32kclk { + assigned-clocks = <&mux_synctimer32k_ck>; + assigned-clock-parents = <&clkdiv32k_ick>; +}; diff --git a/kernel/arch/arm/boot/dts/armada-375.dtsi b/kernel/arch/arm/boot/dts/armada-375.dtsi index 7ccce7529..cc952cf8e 100644 --- a/kernel/arch/arm/boot/dts/armada-375.dtsi +++ b/kernel/arch/arm/boot/dts/armada-375.dtsi @@ -529,7 +529,7 @@ }; sata@a0000 { - compatible = "marvell,orion-sata"; + compatible = "marvell,armada-370-sata"; reg = <0xa0000 0x5000>; interrupts = ; clocks = <&gateclk 14>, <&gateclk 20>; diff --git a/kernel/arch/arm/boot/dts/armada-385-linksys.dtsi b/kernel/arch/arm/boot/dts/armada-385-linksys.dtsi index 3710755c6..22f7a13e2 100644 --- a/kernel/arch/arm/boot/dts/armada-385-linksys.dtsi +++ b/kernel/arch/arm/boot/dts/armada-385-linksys.dtsi @@ -58,8 +58,8 @@ soc { ranges = ; + MBUS_ID(0x09, 0x19) 0 0xf1100000 0x10000 + MBUS_ID(0x09, 0x15) 0 0xf1110000 0x10000>; internal-regs { @@ -117,7 +117,7 @@ }; /* USB part of the eSATA/USB 2.0 port */ - usb@50000 { + usb@58000 { status = "okay"; }; @@ -245,7 +245,7 @@ button@2 { label = "Factory Reset Button"; linux,code = ; - gpios = <&gpio1 15 GPIO_ACTIVE_LOW>; + gpios = <&gpio0 29 GPIO_ACTIVE_LOW>; }; }; @@ -260,7 +260,7 @@ }; sata { - gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + gpios = <&gpio1 22 GPIO_ACTIVE_LOW>; default-state = "off"; }; }; @@ -313,7 +313,7 @@ &pinctrl { keys_pin: keys-pin { - marvell,pins = "mpp24", "mpp47"; + marvell,pins = "mpp24", "mpp29"; marvell,function = "gpio"; }; diff --git a/kernel/arch/arm/boot/dts/armada-390.dtsi b/kernel/arch/arm/boot/dts/armada-390.dtsi index 094e39c66..6cd18d8aa 100644 --- a/kernel/arch/arm/boot/dts/armada-390.dtsi +++ b/kernel/arch/arm/boot/dts/armada-390.dtsi @@ -47,6 +47,8 @@ #include "armada-39x.dtsi" / { + compatible = "marvell,armada390"; + soc { internal-regs { pinctrl@18000 { @@ -54,4 +56,5 @@ reg = <0x18000 0x20>; }; }; + }; }; diff --git a/kernel/arch/arm/boot/dts/armada-xp-linksys-mamba.dts b/kernel/arch/arm/boot/dts/armada-xp-linksys-mamba.dts index b89e6cf12..7a461541c 100644 --- a/kernel/arch/arm/boot/dts/armada-xp-linksys-mamba.dts +++ b/kernel/arch/arm/boot/dts/armada-xp-linksys-mamba.dts @@ -304,13 +304,13 @@ button@1 { label = "WPS"; linux,code = ; - gpios = <&gpio1 0 GPIO_ACTIVE_HIGH>; + gpios = <&gpio1 0 GPIO_ACTIVE_LOW>; }; button@2 { label = "Factory Reset Button"; linux,code = ; - gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>; + gpios = <&gpio1 1 GPIO_ACTIVE_LOW>; }; }; diff --git a/kernel/arch/arm/boot/dts/at91-sama5d3_xplained.dts b/kernel/arch/arm/boot/dts/at91-sama5d3_xplained.dts index ff888d21c..f3e2b96c0 100644 --- a/kernel/arch/arm/boot/dts/at91-sama5d3_xplained.dts +++ b/kernel/arch/arm/boot/dts/at91-sama5d3_xplained.dts @@ -303,6 +303,7 @@ regulator-name = "mmc0-card-supply"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + regulator-always-on; }; gpio_keys { diff --git a/kernel/arch/arm/boot/dts/at91-sama5d4_xplained.dts b/kernel/arch/arm/boot/dts/at91-sama5d4_xplained.dts index 569026e8f..da84e65b5 100644 --- a/kernel/arch/arm/boot/dts/at91-sama5d4_xplained.dts +++ b/kernel/arch/arm/boot/dts/at91-sama5d4_xplained.dts @@ -268,5 +268,6 @@ regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; vin-supply = <&vcc_3v3_reg>; + regulator-always-on; }; }; diff --git a/kernel/arch/arm/boot/dts/at91sam9x5.dtsi b/kernel/arch/arm/boot/dts/at91sam9x5.dtsi index 0827d594b..cd0cd5fd0 100644 --- a/kernel/arch/arm/boot/dts/at91sam9x5.dtsi +++ b/kernel/arch/arm/boot/dts/at91sam9x5.dtsi @@ -106,7 +106,7 @@ pmc: pmc@fffffc00 { compatible = "atmel,at91sam9x5-pmc", "syscon"; - reg = <0xfffffc00 0x100>; + reg = <0xfffffc00 0x200>; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; interrupt-controller; #address-cells = <1>; diff --git a/kernel/arch/arm/boot/dts/da850-evm.dts b/kernel/arch/arm/boot/dts/da850-evm.dts index 4f935ad9f..6881757b0 100644 --- a/kernel/arch/arm/boot/dts/da850-evm.dts +++ b/kernel/arch/arm/boot/dts/da850-evm.dts @@ -85,6 +85,7 @@ #size-cells = <1>; compatible = "m25p64"; spi-max-frequency = <30000000>; + m25p,fast-read; reg = <0>; partition@0 { label = "U-Boot-SPL"; diff --git a/kernel/arch/arm/boot/dts/exynos4210-trats.dts b/kernel/arch/arm/boot/dts/exynos4210-trats.dts index a50be640f..59411e447 100644 --- a/kernel/arch/arm/boot/dts/exynos4210-trats.dts +++ b/kernel/arch/arm/boot/dts/exynos4210-trats.dts @@ -298,6 +298,8 @@ compatible = "maxim,max8997-pmic"; reg = <0x66>; + interrupt-parent = <&gpx0>; + interrupts = <7 0>; max8997,pmic-buck1-uses-gpio-dvs; max8997,pmic-buck2-uses-gpio-dvs; diff --git a/kernel/arch/arm/boot/dts/imx31.dtsi b/kernel/arch/arm/boot/dts/imx31.dtsi index 5fdb22263..cbe5fd5ed 100644 --- a/kernel/arch/arm/boot/dts/imx31.dtsi +++ b/kernel/arch/arm/boot/dts/imx31.dtsi @@ -30,11 +30,11 @@ }; }; - avic: avic-interrupt-controller@60000000 { + avic: interrupt-controller@68000000 { compatible = "fsl,imx31-avic", "fsl,avic"; interrupt-controller; #interrupt-cells = <1>; - reg = <0x60000000 0x100000>; + reg = <0x68000000 0x100000>; }; soc { @@ -110,13 +110,6 @@ interrupts = <19>; clocks = <&clks 25>; }; - - clks: ccm@53f80000{ - compatible = "fsl,imx31-ccm"; - reg = <0x53f80000 0x4000>; - interrupts = <0 31 0x04 0 53 0x04>; - #clock-cells = <1>; - }; }; aips@53f00000 { /* AIPS2 */ @@ -126,6 +119,13 @@ reg = <0x53f00000 0x100000>; ranges; + clks: ccm@53f80000{ + compatible = "fsl,imx31-ccm"; + reg = <0x53f80000 0x4000>; + interrupts = <31>, <53>; + #clock-cells = <1>; + }; + gpt: timer@53f90000 { compatible = "fsl,imx31-gpt"; reg = <0x53f90000 0x4000>; diff --git a/kernel/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/kernel/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi index a35d54fd9..ddfdb75a6 100644 --- a/kernel/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi +++ b/kernel/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi @@ -319,8 +319,6 @@ compatible = "fsl,imx6q-nitrogen6_max-sgtl5000", "fsl,imx-audio-sgtl5000"; model = "imx6q-nitrogen6_max-sgtl5000"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_sgtl5000>; ssi-controller = <&ssi1>; audio-codec = <&codec>; audio-routing = @@ -401,6 +399,8 @@ codec: sgtl5000@0a { compatible = "fsl,sgtl5000"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sgtl5000>; reg = <0x0a>; clocks = <&clks 201>; VDDA-supply = <®_2p5v>; diff --git a/kernel/arch/arm/boot/dts/imx6qdl.dtsi b/kernel/arch/arm/boot/dts/imx6qdl.dtsi index 2b6cc8bf3..e6af41c4b 100644 --- a/kernel/arch/arm/boot/dts/imx6qdl.dtsi +++ b/kernel/arch/arm/boot/dts/imx6qdl.dtsi @@ -221,7 +221,7 @@ clocks = <&clks IMX6QDL_CLK_SPDIF_GCLK>, <&clks IMX6QDL_CLK_OSC>, <&clks IMX6QDL_CLK_SPDIF>, <&clks IMX6QDL_CLK_ASRC>, <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_ESAI_EXTAL>, - <&clks IMX6QDL_CLK_IPG>, <&clks IMX6QDL_CLK_MLB>, + <&clks IMX6QDL_CLK_IPG>, <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_DUMMY>, <&clks IMX6QDL_CLK_SPBA>; clock-names = "core", "rxtx0", "rxtx1", "rxtx2", diff --git a/kernel/arch/arm/boot/dts/kirkwood-ib62x0.dts b/kernel/arch/arm/boot/dts/kirkwood-ib62x0.dts index bfa5edde1..2c1e7f092 100644 --- a/kernel/arch/arm/boot/dts/kirkwood-ib62x0.dts +++ b/kernel/arch/arm/boot/dts/kirkwood-ib62x0.dts @@ -113,7 +113,7 @@ partition@e0000 { label = "u-boot environment"; - reg = <0xe0000 0x100000>; + reg = <0xe0000 0x20000>; }; partition@100000 { diff --git a/kernel/arch/arm/boot/dts/omap3-overo-base.dtsi b/kernel/arch/arm/boot/dts/omap3-overo-base.dtsi index a29ad16cc..64c5af30c 100644 --- a/kernel/arch/arm/boot/dts/omap3-overo-base.dtsi +++ b/kernel/arch/arm/boot/dts/omap3-overo-base.dtsi @@ -223,7 +223,9 @@ }; &gpmc { - ranges = <0 0 0x00000000 0x20000000>; + ranges = <0 0 0x30000000 0x1000000>, /* CS0 */ + <4 0 0x2b000000 0x1000000>, /* CS4 */ + <5 0 0x2c000000 0x1000000>; /* CS5 */ nand@0,0 { linux,mtd-name= "micron,mt29c4g96maz"; diff --git a/kernel/arch/arm/boot/dts/omap3-overo-chestnut43-common.dtsi b/kernel/arch/arm/boot/dts/omap3-overo-chestnut43-common.dtsi index 17b82f826..640477882 100644 --- a/kernel/arch/arm/boot/dts/omap3-overo-chestnut43-common.dtsi +++ b/kernel/arch/arm/boot/dts/omap3-overo-chestnut43-common.dtsi @@ -55,8 +55,6 @@ #include "omap-gpmc-smsc9221.dtsi" &gpmc { - ranges = <5 0 0x2c000000 0x1000000>; /* CS5 */ - ethernet@gpmc { reg = <5 0 0xff>; interrupt-parent = <&gpio6>; diff --git a/kernel/arch/arm/boot/dts/omap3-overo-tobi-common.dtsi b/kernel/arch/arm/boot/dts/omap3-overo-tobi-common.dtsi index 9e24b6a1d..1b304e2f1 100644 --- a/kernel/arch/arm/boot/dts/omap3-overo-tobi-common.dtsi +++ b/kernel/arch/arm/boot/dts/omap3-overo-tobi-common.dtsi @@ -27,8 +27,6 @@ #include "omap-gpmc-smsc9221.dtsi" &gpmc { - ranges = <5 0 0x2c000000 0x1000000>; /* CS5 */ - ethernet@gpmc { reg = <5 0 0xff>; interrupt-parent = <&gpio6>; diff --git a/kernel/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi b/kernel/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi index 334109e14..82e98ee30 100644 --- a/kernel/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi +++ b/kernel/arch/arm/boot/dts/omap3-overo-tobiduo-common.dtsi @@ -15,9 +15,6 @@ #include "omap-gpmc-smsc9221.dtsi" &gpmc { - ranges = <4 0 0x2b000000 0x1000000>, /* CS4 */ - <5 0 0x2c000000 0x1000000>; /* CS5 */ - smsc1: ethernet@gpmc { reg = <5 0 0xff>; interrupt-parent = <&gpio6>; diff --git a/kernel/arch/arm/boot/dts/pxa3xx.dtsi b/kernel/arch/arm/boot/dts/pxa3xx.dtsi index cf6998a08..564341af7 100644 --- a/kernel/arch/arm/boot/dts/pxa3xx.dtsi +++ b/kernel/arch/arm/boot/dts/pxa3xx.dtsi @@ -30,7 +30,7 @@ reg = <0x43100000 90>; interrupts = <45>; clocks = <&clks CLK_NAND>; - dmas = <&pdma 97>; + dmas = <&pdma 97 3>; dma-names = "data"; #address-cells = <1>; #size-cells = <1>; diff --git a/kernel/arch/arm/boot/dts/qcom-apq8064.dtsi b/kernel/arch/arm/boot/dts/qcom-apq8064.dtsi index a4c1762b5..e00d50ef6 100644 --- a/kernel/arch/arm/boot/dts/qcom-apq8064.dtsi +++ b/kernel/arch/arm/boot/dts/qcom-apq8064.dtsi @@ -5,6 +5,7 @@ #include #include #include +#include #include / { model = "Qualcomm APQ8064"; @@ -354,22 +355,50 @@ compatible = "qcom,pm8921-gpio"; reg = <0x150>; - interrupts = <192 1>, <193 1>, <194 1>, - <195 1>, <196 1>, <197 1>, - <198 1>, <199 1>, <200 1>, - <201 1>, <202 1>, <203 1>, - <204 1>, <205 1>, <206 1>, - <207 1>, <208 1>, <209 1>, - <210 1>, <211 1>, <212 1>, - <213 1>, <214 1>, <215 1>, - <216 1>, <217 1>, <218 1>, - <219 1>, <220 1>, <221 1>, - <222 1>, <223 1>, <224 1>, - <225 1>, <226 1>, <227 1>, - <228 1>, <229 1>, <230 1>, - <231 1>, <232 1>, <233 1>, - <234 1>, <235 1>; - + interrupts = <192 IRQ_TYPE_NONE>, + <193 IRQ_TYPE_NONE>, + <194 IRQ_TYPE_NONE>, + <195 IRQ_TYPE_NONE>, + <196 IRQ_TYPE_NONE>, + <197 IRQ_TYPE_NONE>, + <198 IRQ_TYPE_NONE>, + <199 IRQ_TYPE_NONE>, + <200 IRQ_TYPE_NONE>, + <201 IRQ_TYPE_NONE>, + <202 IRQ_TYPE_NONE>, + <203 IRQ_TYPE_NONE>, + <204 IRQ_TYPE_NONE>, + <205 IRQ_TYPE_NONE>, + <206 IRQ_TYPE_NONE>, + <207 IRQ_TYPE_NONE>, + <208 IRQ_TYPE_NONE>, + <209 IRQ_TYPE_NONE>, + <210 IRQ_TYPE_NONE>, + <211 IRQ_TYPE_NONE>, + <212 IRQ_TYPE_NONE>, + <213 IRQ_TYPE_NONE>, + <214 IRQ_TYPE_NONE>, + <215 IRQ_TYPE_NONE>, + <216 IRQ_TYPE_NONE>, + <217 IRQ_TYPE_NONE>, + <218 IRQ_TYPE_NONE>, + <219 IRQ_TYPE_NONE>, + <220 IRQ_TYPE_NONE>, + <221 IRQ_TYPE_NONE>, + <222 IRQ_TYPE_NONE>, + <223 IRQ_TYPE_NONE>, + <224 IRQ_TYPE_NONE>, + <225 IRQ_TYPE_NONE>, + <226 IRQ_TYPE_NONE>, + <227 IRQ_TYPE_NONE>, + <228 IRQ_TYPE_NONE>, + <229 IRQ_TYPE_NONE>, + <230 IRQ_TYPE_NONE>, + <231 IRQ_TYPE_NONE>, + <232 IRQ_TYPE_NONE>, + <233 IRQ_TYPE_NONE>, + <234 IRQ_TYPE_NONE>, + <235 IRQ_TYPE_NONE>; gpio-controller; #gpio-cells = <2>; @@ -381,9 +410,18 @@ gpio-controller; #gpio-cells = <2>; interrupts = - <128 1>, <129 1>, <130 1>, <131 1>, - <132 1>, <133 1>, <134 1>, <135 1>, - <136 1>, <137 1>, <138 1>, <139 1>; + <128 IRQ_TYPE_NONE>, + <129 IRQ_TYPE_NONE>, + <130 IRQ_TYPE_NONE>, + <131 IRQ_TYPE_NONE>, + <132 IRQ_TYPE_NONE>, + <133 IRQ_TYPE_NONE>, + <134 IRQ_TYPE_NONE>, + <135 IRQ_TYPE_NONE>, + <136 IRQ_TYPE_NONE>, + <137 IRQ_TYPE_NONE>, + <138 IRQ_TYPE_NONE>, + <139 IRQ_TYPE_NONE>; }; rtc@11d { diff --git a/kernel/arch/arm/boot/dts/r8a7794.dtsi b/kernel/arch/arm/boot/dts/r8a7794.dtsi index a9977d6ee..169653586 100644 --- a/kernel/arch/arm/boot/dts/r8a7794.dtsi +++ b/kernel/arch/arm/boot/dts/r8a7794.dtsi @@ -1023,7 +1023,7 @@ mstp7_clks: mstp7_clks@e615014c { compatible = "renesas,r8a7794-mstp-clocks", "renesas,cpg-mstp-clocks"; reg = <0 0xe615014c 0 4>, <0 0xe61501c4 0 4>; - clocks = <&mp_clk>, <&mp_clk>, + clocks = <&mp_clk>, <&hp_clk>, <&zs_clk>, <&p_clk>, <&p_clk>, <&zs_clk>, <&zs_clk>, <&p_clk>, <&p_clk>, <&p_clk>, <&p_clk>; #clock-cells = <1>; diff --git a/kernel/arch/arm/boot/dts/sama5d2-pinfunc.h b/kernel/arch/arm/boot/dts/sama5d2-pinfunc.h index b0c912fea..8a394f336 100644 --- a/kernel/arch/arm/boot/dts/sama5d2-pinfunc.h +++ b/kernel/arch/arm/boot/dts/sama5d2-pinfunc.h @@ -837,8 +837,8 @@ #define PIN_PD23__ISC_FIELD PINMUX_PIN(PIN_PD23, 6, 4) #define PIN_PD24 120 #define PIN_PD24__GPIO PINMUX_PIN(PIN_PD24, 0, 0) -#define PIN_PD24__UTXD2 PINMUX_PIN(PIN_PD23, 1, 2) -#define PIN_PD24__FLEXCOM4_IO3 PINMUX_PIN(PIN_PD23, 3, 3) +#define PIN_PD24__UTXD2 PINMUX_PIN(PIN_PD24, 1, 2) +#define PIN_PD24__FLEXCOM4_IO3 PINMUX_PIN(PIN_PD24, 3, 3) #define PIN_PD25 121 #define PIN_PD25__GPIO PINMUX_PIN(PIN_PD25, 0, 0) #define PIN_PD25__SPI1_SPCK PINMUX_PIN(PIN_PD25, 1, 3) diff --git a/kernel/arch/arm/boot/dts/stih407-family.dtsi b/kernel/arch/arm/boot/dts/stih407-family.dtsi index 81f81214c..bbf95375c 100644 --- a/kernel/arch/arm/boot/dts/stih407-family.dtsi +++ b/kernel/arch/arm/boot/dts/stih407-family.dtsi @@ -497,8 +497,9 @@ interrupt-names = "mmcirq"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_mmc0>; - clock-names = "mmc"; - clocks = <&clk_s_c0_flexgen CLK_MMC_0>; + clock-names = "mmc", "icn"; + clocks = <&clk_s_c0_flexgen CLK_MMC_0>, + <&clk_s_c0_flexgen CLK_RX_ICN_HVA>; bus-width = <8>; non-removable; }; @@ -512,8 +513,9 @@ interrupt-names = "mmcirq"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_sd1>; - clock-names = "mmc"; - clocks = <&clk_s_c0_flexgen CLK_MMC_1>; + clock-names = "mmc", "icn"; + clocks = <&clk_s_c0_flexgen CLK_MMC_1>, + <&clk_s_c0_flexgen CLK_RX_ICN_HVA>; resets = <&softreset STIH407_MMC1_SOFTRESET>; bus-width = <4>; }; diff --git a/kernel/arch/arm/boot/dts/stih410.dtsi b/kernel/arch/arm/boot/dts/stih410.dtsi index 18ed1ad10..40318869c 100644 --- a/kernel/arch/arm/boot/dts/stih410.dtsi +++ b/kernel/arch/arm/boot/dts/stih410.dtsi @@ -41,7 +41,8 @@ compatible = "st,st-ohci-300x"; reg = <0x9a03c00 0x100>; interrupts = ; - clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>; + clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>, + <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>; resets = <&powerdown STIH407_USB2_PORT0_POWERDOWN>, <&softreset STIH407_USB2_PORT0_SOFTRESET>; reset-names = "power", "softreset"; @@ -57,7 +58,8 @@ interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; - clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>; + clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>, + <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>; resets = <&powerdown STIH407_USB2_PORT0_POWERDOWN>, <&softreset STIH407_USB2_PORT0_SOFTRESET>; reset-names = "power", "softreset"; @@ -71,7 +73,8 @@ compatible = "st,st-ohci-300x"; reg = <0x9a83c00 0x100>; interrupts = ; - clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>; + clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>, + <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>; resets = <&powerdown STIH407_USB2_PORT1_POWERDOWN>, <&softreset STIH407_USB2_PORT1_SOFTRESET>; reset-names = "power", "softreset"; @@ -87,7 +90,8 @@ interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; - clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>; + clocks = <&clk_s_c0_flexgen CLK_TX_ICN_DISP_0>, + <&clk_s_c0_flexgen CLK_RX_ICN_DISP_0>; resets = <&powerdown STIH407_USB2_PORT1_POWERDOWN>, <&softreset STIH407_USB2_PORT1_SOFTRESET>; reset-names = "power", "softreset"; diff --git a/kernel/arch/arm/boot/dts/sun4i-a10-a1000.dts b/kernel/arch/arm/boot/dts/sun4i-a10-a1000.dts index 97570cb7f..1d23527d4 100644 --- a/kernel/arch/arm/boot/dts/sun4i-a10-a1000.dts +++ b/kernel/arch/arm/boot/dts/sun4i-a10-a1000.dts @@ -84,6 +84,7 @@ regulator-name = "emac-3v3"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + startup-delay-us = <20000>; enable-active-high; gpio = <&pio 7 15 GPIO_ACTIVE_HIGH>; }; diff --git a/kernel/arch/arm/boot/dts/sun4i-a10-hackberry.dts b/kernel/arch/arm/boot/dts/sun4i-a10-hackberry.dts index 2b17c5199..6de83a618 100644 --- a/kernel/arch/arm/boot/dts/sun4i-a10-hackberry.dts +++ b/kernel/arch/arm/boot/dts/sun4i-a10-hackberry.dts @@ -66,6 +66,7 @@ regulator-name = "emac-3v3"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + startup-delay-us = <20000>; enable-active-high; gpio = <&pio 7 19 GPIO_ACTIVE_HIGH>; }; diff --git a/kernel/arch/arm/boot/dts/sun4i-a10-jesurun-q5.dts b/kernel/arch/arm/boot/dts/sun4i-a10-jesurun-q5.dts index 7afc7a64e..e28f080b1 100644 --- a/kernel/arch/arm/boot/dts/sun4i-a10-jesurun-q5.dts +++ b/kernel/arch/arm/boot/dts/sun4i-a10-jesurun-q5.dts @@ -80,6 +80,7 @@ regulator-name = "emac-3v3"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + startup-delay-us = <20000>; enable-active-high; gpio = <&pio 7 19 GPIO_ACTIVE_HIGH>; /* PH19 */ }; diff --git a/kernel/arch/arm/boot/dts/sun5i-a10s-wobo-i5.dts b/kernel/arch/arm/boot/dts/sun5i-a10s-wobo-i5.dts index 9fea918f9..39731a78f 100644 --- a/kernel/arch/arm/boot/dts/sun5i-a10s-wobo-i5.dts +++ b/kernel/arch/arm/boot/dts/sun5i-a10s-wobo-i5.dts @@ -79,6 +79,7 @@ regulator-name = "emac-3v3"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + startup-delay-us = <20000>; enable-active-high; gpio = <&pio 0 2 GPIO_ACTIVE_HIGH>; }; diff --git a/kernel/arch/arm/boot/dts/sun5i-a13.dtsi b/kernel/arch/arm/boot/dts/sun5i-a13.dtsi index d910d3a6c..84bdba480 100644 --- a/kernel/arch/arm/boot/dts/sun5i-a13.dtsi +++ b/kernel/arch/arm/boot/dts/sun5i-a13.dtsi @@ -83,7 +83,7 @@ trips { cpu_alert0: cpu_alert0 { /* milliCelsius */ - temperature = <850000>; + temperature = <85000>; hysteresis = <2000>; type = "passive"; }; diff --git a/kernel/arch/arm/boot/dts/sun5i-r8-chip.dts b/kernel/arch/arm/boot/dts/sun5i-r8-chip.dts index 530ab28e9..d21f50ba3 100644 --- a/kernel/arch/arm/boot/dts/sun5i-r8-chip.dts +++ b/kernel/arch/arm/boot/dts/sun5i-r8-chip.dts @@ -52,7 +52,7 @@ / { model = "NextThing C.H.I.P."; - compatible = "nextthing,chip", "allwinner,sun5i-r8"; + compatible = "nextthing,chip", "allwinner,sun5i-r8", "allwinner,sun5i-a13"; aliases { i2c0 = &i2c0; diff --git a/kernel/arch/arm/common/sa1111.c b/kernel/arch/arm/common/sa1111.c index 3d224941b..a3a9ad4dc 100644 --- a/kernel/arch/arm/common/sa1111.c +++ b/kernel/arch/arm/common/sa1111.c @@ -869,9 +869,9 @@ struct sa1111_save_data { #ifdef CONFIG_PM -static int sa1111_suspend(struct platform_device *dev, pm_message_t state) +static int sa1111_suspend_noirq(struct device *dev) { - struct sa1111 *sachip = platform_get_drvdata(dev); + struct sa1111 *sachip = dev_get_drvdata(dev); struct sa1111_save_data *save; unsigned long flags; unsigned int val; @@ -934,9 +934,9 @@ static int sa1111_suspend(struct platform_device *dev, pm_message_t state) * restored by their respective drivers, and must be called * via LDM after this function. */ -static int sa1111_resume(struct platform_device *dev) +static int sa1111_resume_noirq(struct device *dev) { - struct sa1111 *sachip = platform_get_drvdata(dev); + struct sa1111 *sachip = dev_get_drvdata(dev); struct sa1111_save_data *save; unsigned long flags, id; void __iomem *base; @@ -952,7 +952,7 @@ static int sa1111_resume(struct platform_device *dev) id = sa1111_readl(sachip->base + SA1111_SKID); if ((id & SKID_ID_MASK) != SKID_SA1111_ID) { __sa1111_remove(sachip); - platform_set_drvdata(dev, NULL); + dev_set_drvdata(dev, NULL); kfree(save); return 0; } @@ -1003,8 +1003,8 @@ static int sa1111_resume(struct platform_device *dev) } #else -#define sa1111_suspend NULL -#define sa1111_resume NULL +#define sa1111_suspend_noirq NULL +#define sa1111_resume_noirq NULL #endif static int sa1111_probe(struct platform_device *pdev) @@ -1038,6 +1038,11 @@ static int sa1111_remove(struct platform_device *pdev) return 0; } +static struct dev_pm_ops sa1111_pm_ops = { + .suspend_noirq = sa1111_suspend_noirq, + .resume_noirq = sa1111_resume_noirq, +}; + /* * Not sure if this should be on the system bus or not yet. * We really want some way to register a system device at @@ -1050,10 +1055,9 @@ static int sa1111_remove(struct platform_device *pdev) static struct platform_driver sa1111_device_driver = { .probe = sa1111_probe, .remove = sa1111_remove, - .suspend = sa1111_suspend, - .resume = sa1111_resume, .driver = { .name = "sa1111", + .pm = &sa1111_pm_ops, }, }; diff --git a/kernel/arch/arm/crypto/aes-ce-glue.c b/kernel/arch/arm/crypto/aes-ce-glue.c index b445a5d56..679c589c4 100644 --- a/kernel/arch/arm/crypto/aes-ce-glue.c +++ b/kernel/arch/arm/crypto/aes-ce-glue.c @@ -87,8 +87,13 @@ static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, u32 *rki = ctx->key_enc + (i * kwords); u32 *rko = rki + kwords; +#ifndef CONFIG_CPU_BIG_ENDIAN rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); rko[0] = rko[0] ^ rki[0] ^ rcon[i]; +#else + rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8); + rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24); +#endif rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; @@ -279,7 +284,7 @@ static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } - if (nbytes) { + if (walk.nbytes % AES_BLOCK_SIZE) { u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; u8 __aligned(8) tail[AES_BLOCK_SIZE]; diff --git a/kernel/arch/arm/crypto/ghash-ce-glue.c b/kernel/arch/arm/crypto/ghash-ce-glue.c index 03a39fe29..9d9ba9acd 100644 --- a/kernel/arch/arm/crypto/ghash-ce-glue.c +++ b/kernel/arch/arm/crypto/ghash-ce-glue.c @@ -226,6 +226,27 @@ static int ghash_async_digest(struct ahash_request *req) } } +static int ghash_async_import(struct ahash_request *req, const void *in) +{ + struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + + desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); + desc->flags = req->base.flags; + + return crypto_shash_import(desc, in); +} + +static int ghash_async_export(struct ahash_request *req, void *out) +{ + struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + + return crypto_shash_export(desc, out); +} + static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { @@ -274,7 +295,10 @@ static struct ahash_alg ghash_async_alg = { .final = ghash_async_final, .setkey = ghash_async_setkey, .digest = ghash_async_digest, + .import = ghash_async_import, + .export = ghash_async_export, .halg.digestsize = GHASH_DIGEST_SIZE, + .halg.statesize = sizeof(struct ghash_desc_ctx), .halg.base = { .cra_name = "ghash", .cra_driver_name = "ghash-ce", diff --git a/kernel/arch/arm/include/asm/cputype.h b/kernel/arch/arm/include/asm/cputype.h index 85e374f87..e9d04f475 100644 --- a/kernel/arch/arm/include/asm/cputype.h +++ b/kernel/arch/arm/include/asm/cputype.h @@ -81,6 +81,9 @@ #define ARM_CPU_XSCALE_ARCH_V2 0x4000 #define ARM_CPU_XSCALE_ARCH_V3 0x6000 +/* Qualcomm implemented cores */ +#define ARM_CPU_PART_SCORPION 0x510002d0 + extern unsigned int processor_id; #ifdef CONFIG_CPU_CP15 diff --git a/kernel/arch/arm/include/asm/dma-mapping.h b/kernel/arch/arm/include/asm/dma-mapping.h index ccb3aa646..b91a2d17a 100644 --- a/kernel/arch/arm/include/asm/dma-mapping.h +++ b/kernel/arch/arm/include/asm/dma-mapping.h @@ -119,7 +119,7 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) /* The ARM override for dma_max_pfn() */ static inline unsigned long dma_max_pfn(struct device *dev) { - return PHYS_PFN_OFFSET + dma_to_pfn(dev, *dev->dma_mask); + return dma_to_pfn(dev, *dev->dma_mask); } #define dma_max_pfn(dev) dma_max_pfn(dev) diff --git a/kernel/arch/arm/include/asm/floppy.h b/kernel/arch/arm/include/asm/floppy.h index f4882553f..85a34cc83 100644 --- a/kernel/arch/arm/include/asm/floppy.h +++ b/kernel/arch/arm/include/asm/floppy.h @@ -17,7 +17,7 @@ #define fd_outb(val,port) \ do { \ - if ((port) == FD_DOR) \ + if ((port) == (u32)FD_DOR) \ fd_setdor((val)); \ else \ outb((val),(port)); \ diff --git a/kernel/arch/arm/include/asm/pgtable-2level.h b/kernel/arch/arm/include/asm/pgtable-2level.h index aeddd28b3..92fd2c8a9 100644 --- a/kernel/arch/arm/include/asm/pgtable-2level.h +++ b/kernel/arch/arm/include/asm/pgtable-2level.h @@ -193,6 +193,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define pmd_large(pmd) (pmd_val(pmd) & 2) #define pmd_bad(pmd) (pmd_val(pmd) & 2) +#define pmd_present(pmd) (pmd_val(pmd)) #define copy_pmd(pmdpd,pmdps) \ do { \ diff --git a/kernel/arch/arm/include/asm/pgtable-3level.h b/kernel/arch/arm/include/asm/pgtable-3level.h index a745a2a53..fd929b5de 100644 --- a/kernel/arch/arm/include/asm/pgtable-3level.h +++ b/kernel/arch/arm/include/asm/pgtable-3level.h @@ -212,6 +212,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) : !!(pmd_val(pmd) & (val))) #define pmd_isclear(pmd, val) (!(pmd_val(pmd) & (val))) +#define pmd_present(pmd) (pmd_isset((pmd), L_PMD_SECT_VALID)) #define pmd_young(pmd) (pmd_isset((pmd), PMD_SECT_AF)) #define pte_special(pte) (pte_isset((pte), L_PTE_SPECIAL)) static inline pte_t pte_mkspecial(pte_t pte) @@ -257,10 +258,10 @@ PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF); #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) -/* represent a notpresent pmd by zero, this is used by pmdp_invalidate */ +/* represent a notpresent pmd by faulting entry, this is used by pmdp_invalidate */ static inline pmd_t pmd_mknotpresent(pmd_t pmd) { - return __pmd(0); + return __pmd(pmd_val(pmd) & ~L_PMD_SECT_VALID); } static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) diff --git a/kernel/arch/arm/include/asm/pgtable.h b/kernel/arch/arm/include/asm/pgtable.h index 348caabb7..d62204060 100644 --- a/kernel/arch/arm/include/asm/pgtable.h +++ b/kernel/arch/arm/include/asm/pgtable.h @@ -182,7 +182,6 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; #define pgd_offset_k(addr) pgd_offset(&init_mm, addr) #define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_present(pmd) (pmd_val(pmd)) static inline pte_t *pmd_page_vaddr(pmd_t pmd) { diff --git a/kernel/arch/arm/kernel/devtree.c b/kernel/arch/arm/kernel/devtree.c index 65addcbf5..b3b950fc8 100644 --- a/kernel/arch/arm/kernel/devtree.c +++ b/kernel/arch/arm/kernel/devtree.c @@ -87,6 +87,8 @@ void __init arm_dt_init_cpu_maps(void) return; for_each_child_of_node(cpus, cpu) { + const __be32 *cell; + int prop_bytes; u32 hwid; if (of_node_cmp(cpu->type, "cpu")) @@ -98,7 +100,8 @@ void __init arm_dt_init_cpu_maps(void) * properties is considered invalid to build the * cpu_logical_map. */ - if (of_property_read_u32(cpu, "reg", &hwid)) { + cell = of_get_property(cpu, "reg", &prop_bytes); + if (!cell || prop_bytes < sizeof(*cell)) { pr_debug(" * %s missing reg property\n", cpu->full_name); of_node_put(cpu); @@ -106,10 +109,15 @@ void __init arm_dt_init_cpu_maps(void) } /* - * 8 MSBs must be set to 0 in the DT since the reg property + * Bits n:24 must be set to 0 in the DT since the reg property * defines the MPIDR[23:0]. */ - if (hwid & ~MPIDR_HWID_BITMASK) { + do { + hwid = be32_to_cpu(*cell++); + prop_bytes -= sizeof(*cell); + } while (!hwid && prop_bytes > 0); + + if (prop_bytes || (hwid & ~MPIDR_HWID_BITMASK)) { of_node_put(cpu); return; } diff --git a/kernel/arch/arm/kernel/entry-armv.S b/kernel/arch/arm/kernel/entry-armv.S index d66b1aef2..d044cea59 100644 --- a/kernel/arch/arm/kernel/entry-armv.S +++ b/kernel/arch/arm/kernel/entry-armv.S @@ -244,7 +244,11 @@ svc_preempt: bne 1b tst r0, #_TIF_NEED_RESCHED_LAZY reteq r8 @ go again - b 1b + ldr r0, [tsk, #TI_PREEMPT_LAZY] @ get preempt lazy count + teq r0, #0 @ if preempt lazy count != 0 + beq 1b + ret r8 @ go again + #endif __und_fault: diff --git a/kernel/arch/arm/kernel/hw_breakpoint.c b/kernel/arch/arm/kernel/hw_breakpoint.c index 6284779d6..abcbea1ae 100644 --- a/kernel/arch/arm/kernel/hw_breakpoint.c +++ b/kernel/arch/arm/kernel/hw_breakpoint.c @@ -1066,6 +1066,22 @@ static int __init arch_hw_breakpoint_init(void) return 0; } + /* + * Scorpion CPUs (at least those in APQ8060) seem to set DBGPRSR.SPD + * whenever a WFI is issued, even if the core is not powered down, in + * violation of the architecture. When DBGPRSR.SPD is set, accesses to + * breakpoint and watchpoint registers are treated as undefined, so + * this results in boot time and runtime failures when these are + * accessed and we unexpectedly take a trap. + * + * It's not clear if/how this can be worked around, so we blacklist + * Scorpion CPUs to avoid these issues. + */ + if (read_cpuid_part() == ARM_CPU_PART_SCORPION) { + pr_info("Scorpion CPU detected. Hardware breakpoints and watchpoints disabled\n"); + return 0; + } + has_ossr = core_has_os_save_restore(); /* Determine how many BRPs/WRPs are available. */ diff --git a/kernel/arch/arm/kernel/patch.c b/kernel/arch/arm/kernel/patch.c index 69bda1a57..1f665acaa 100644 --- a/kernel/arch/arm/kernel/patch.c +++ b/kernel/arch/arm/kernel/patch.c @@ -15,7 +15,7 @@ struct patch { unsigned int insn; }; -static DEFINE_SPINLOCK(patch_lock); +static DEFINE_RAW_SPINLOCK(patch_lock); static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) __acquires(&patch_lock) @@ -32,7 +32,7 @@ static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags) return addr; if (flags) - spin_lock_irqsave(&patch_lock, *flags); + raw_spin_lock_irqsave(&patch_lock, *flags); else __acquire(&patch_lock); @@ -47,7 +47,7 @@ static void __kprobes patch_unmap(int fixmap, unsigned long *flags) clear_fixmap(fixmap); if (flags) - spin_unlock_irqrestore(&patch_lock, *flags); + raw_spin_unlock_irqrestore(&patch_lock, *flags); else __release(&patch_lock); } diff --git a/kernel/arch/arm/kernel/ptrace.c b/kernel/arch/arm/kernel/ptrace.c index ef9119f74..d54c53b7a 100644 --- a/kernel/arch/arm/kernel/ptrace.c +++ b/kernel/arch/arm/kernel/ptrace.c @@ -600,7 +600,7 @@ static int gpr_set(struct task_struct *target, const void *kbuf, const void __user *ubuf) { int ret; - struct pt_regs newregs; + struct pt_regs newregs = *task_pt_regs(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, @@ -733,8 +733,8 @@ static int vfp_set(struct task_struct *target, if (ret) return ret; - vfp_flush_hwstate(thread); thread->vfpstate.hard = new_vfp; + vfp_flush_hwstate(thread); return 0; } diff --git a/kernel/arch/arm/kernel/smp_tlb.c b/kernel/arch/arm/kernel/smp_tlb.c index 2e72be4f6..7cb079e74 100644 --- a/kernel/arch/arm/kernel/smp_tlb.c +++ b/kernel/arch/arm/kernel/smp_tlb.c @@ -9,6 +9,7 @@ */ #include #include +#include #include #include @@ -40,8 +41,11 @@ static inline void ipi_flush_tlb_mm(void *arg) static inline void ipi_flush_tlb_page(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; + unsigned int __ua_flags = uaccess_save_and_enable(); local_flush_tlb_page(ta->ta_vma, ta->ta_start); + + uaccess_restore(__ua_flags); } static inline void ipi_flush_tlb_kernel_page(void *arg) @@ -54,8 +58,11 @@ static inline void ipi_flush_tlb_kernel_page(void *arg) static inline void ipi_flush_tlb_range(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; + unsigned int __ua_flags = uaccess_save_and_enable(); local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); + + uaccess_restore(__ua_flags); } static inline void ipi_flush_tlb_kernel_range(void *arg) diff --git a/kernel/arch/arm/kernel/sys_oabi-compat.c b/kernel/arch/arm/kernel/sys_oabi-compat.c index 087acb569..5f221acd2 100644 --- a/kernel/arch/arm/kernel/sys_oabi-compat.c +++ b/kernel/arch/arm/kernel/sys_oabi-compat.c @@ -279,8 +279,12 @@ asmlinkage long sys_oabi_epoll_wait(int epfd, mm_segment_t fs; long ret, err, i; - if (maxevents <= 0 || maxevents > (INT_MAX/sizeof(struct epoll_event))) + if (maxevents <= 0 || + maxevents > (INT_MAX/sizeof(*kbuf)) || + maxevents > (INT_MAX/sizeof(*events))) return -EINVAL; + if (!access_ok(VERIFY_WRITE, events, sizeof(*events) * maxevents)) + return -EFAULT; kbuf = kmalloc(sizeof(*kbuf) * maxevents, GFP_KERNEL); if (!kbuf) return -ENOMEM; @@ -317,6 +321,8 @@ asmlinkage long sys_oabi_semtimedop(int semid, if (nsops < 1 || nsops > SEMOPM) return -EINVAL; + if (!access_ok(VERIFY_READ, tsops, sizeof(*tsops) * nsops)) + return -EFAULT; sops = kmalloc(sizeof(*sops) * nsops, GFP_KERNEL); if (!sops) return -ENOMEM; diff --git a/kernel/arch/arm/kvm/arm.c b/kernel/arch/arm/kvm/arm.c index bb43bc396..36a3e5149 100644 --- a/kernel/arch/arm/kvm/arm.c +++ b/kernel/arch/arm/kvm/arm.c @@ -155,8 +155,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm) { int i; - kvm_free_stage2_pgd(kvm); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { if (kvm->vcpus[i]) { kvm_arch_vcpu_free(kvm->vcpus[i]); diff --git a/kernel/arch/arm/kvm/mmu.c b/kernel/arch/arm/kvm/mmu.c index 61d96a645..11b6595c2 100644 --- a/kernel/arch/arm/kvm/mmu.c +++ b/kernel/arch/arm/kvm/mmu.c @@ -886,11 +886,14 @@ static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd)); old_pmd = *pmd; - kvm_set_pmd(pmd, *new_pmd); - if (pmd_present(old_pmd)) + if (pmd_present(old_pmd)) { + pmd_clear(pmd); kvm_tlb_flush_vmid_ipa(kvm, addr); - else + } else { get_page(virt_to_page(pmd)); + } + + kvm_set_pmd(pmd, *new_pmd); return 0; } @@ -939,12 +942,14 @@ static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, /* Create 2nd stage page table mapping - Level 3 */ old_pte = *pte; - kvm_set_pte(pte, *new_pte); - if (pte_present(old_pte)) + if (pte_present(old_pte)) { + kvm_set_pte(pte, __pte(0)); kvm_tlb_flush_vmid_ipa(kvm, addr); - else + } else { get_page(virt_to_page(pte)); + } + kvm_set_pte(pte, *new_pte); return 0; } @@ -1847,6 +1852,7 @@ void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) void kvm_arch_flush_shadow_all(struct kvm *kvm) { + kvm_free_stage2_pgd(kvm); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, diff --git a/kernel/arch/arm/mach-cns3xxx/pcie.c b/kernel/arch/arm/mach-cns3xxx/pcie.c index 47905a50e..318394ed5 100644 --- a/kernel/arch/arm/mach-cns3xxx/pcie.c +++ b/kernel/arch/arm/mach-cns3xxx/pcie.c @@ -220,13 +220,13 @@ static void cns3xxx_write_config(struct cns3xxx_pcie *cnspci, u32 mask = (0x1ull << (size * 8)) - 1; int shift = (where % 4) * 8; - v = readl_relaxed(base + (where & 0xffc)); + v = readl_relaxed(base); v &= ~(mask << shift); v |= (val & mask) << shift; - writel_relaxed(v, base + (where & 0xffc)); - readl_relaxed(base + (where & 0xffc)); + writel_relaxed(v, base); + readl_relaxed(base); } static void __init cns3xxx_pcie_hw_init(struct cns3xxx_pcie *cnspci) diff --git a/kernel/arch/arm/mach-davinci/da850.c b/kernel/arch/arm/mach-davinci/da850.c index 676997895..f5f81a107 100644 --- a/kernel/arch/arm/mach-davinci/da850.c +++ b/kernel/arch/arm/mach-davinci/da850.c @@ -298,6 +298,16 @@ static struct clk emac_clk = { .gpsc = 1, }; +/* + * In order to avoid adding the emac_clk to the clock lookup table twice (and + * screwing up the linked list in the process) create a separate clock for + * mdio inheriting the rate from emac_clk. + */ +static struct clk mdio_clk = { + .name = "mdio", + .parent = &emac_clk, +}; + static struct clk mcasp_clk = { .name = "mcasp", .parent = &pll0_sysclk2, @@ -462,7 +472,7 @@ static struct clk_lookup da850_clks[] = { CLK(NULL, "arm", &arm_clk), CLK(NULL, "rmii", &rmii_clk), CLK("davinci_emac.1", NULL, &emac_clk), - CLK("davinci_mdio.0", "fck", &emac_clk), + CLK("davinci_mdio.0", "fck", &mdio_clk), CLK("davinci-mcasp.0", NULL, &mcasp_clk), CLK("da8xx_lcdc.0", "fck", &lcdc_clk), CLK("da830-mmc.0", NULL, &mmcsd0_clk), diff --git a/kernel/arch/arm/mach-exynos/Kconfig b/kernel/arch/arm/mach-exynos/Kconfig index 3a10f1a83..bfd8bb371 100644 --- a/kernel/arch/arm/mach-exynos/Kconfig +++ b/kernel/arch/arm/mach-exynos/Kconfig @@ -26,6 +26,7 @@ menuconfig ARCH_EXYNOS select S5P_DEV_MFC select SRAM select THERMAL + select THERMAL_OF select MFD_SYSCON help Support for SAMSUNG EXYNOS SoCs (EXYNOS4/5) diff --git a/kernel/arch/arm/mach-exynos/pm_domains.c b/kernel/arch/arm/mach-exynos/pm_domains.c index 7c21760f5..875a2bab6 100644 --- a/kernel/arch/arm/mach-exynos/pm_domains.c +++ b/kernel/arch/arm/mach-exynos/pm_domains.c @@ -92,7 +92,7 @@ static int exynos_pd_power(struct generic_pm_domain *domain, bool power_on) if (IS_ERR(pd->clk[i])) break; - if (IS_ERR(pd->clk[i])) + if (IS_ERR(pd->pclk[i])) continue; /* Skip on first power up */ if (clk_set_parent(pd->clk[i], pd->pclk[i])) pr_err("%s: error setting parent to clock%d\n", diff --git a/kernel/arch/arm/mach-imx/mach-imx6ul.c b/kernel/arch/arm/mach-imx/mach-imx6ul.c index acaf7056e..e08d02667 100644 --- a/kernel/arch/arm/mach-imx/mach-imx6ul.c +++ b/kernel/arch/arm/mach-imx/mach-imx6ul.c @@ -46,7 +46,7 @@ static int ksz8081_phy_fixup(struct phy_device *dev) static void __init imx6ul_enet_phy_init(void) { if (IS_BUILTIN(CONFIG_PHYLIB)) - phy_register_fixup_for_uid(PHY_ID_KSZ8081, 0xffffffff, + phy_register_fixup_for_uid(PHY_ID_KSZ8081, MICREL_PHY_ID_MASK, ksz8081_phy_fixup); } diff --git a/kernel/arch/arm/mach-imx/pm-imx6.c b/kernel/arch/arm/mach-imx/pm-imx6.c index 4470376af..a19d20f23 100644 --- a/kernel/arch/arm/mach-imx/pm-imx6.c +++ b/kernel/arch/arm/mach-imx/pm-imx6.c @@ -295,7 +295,7 @@ int imx6_set_lpm(enum mxc_cpu_pwr_mode mode) val &= ~BM_CLPCR_SBYOS; if (cpu_is_imx6sl()) val |= BM_CLPCR_BYPASS_PMIC_READY; - if (cpu_is_imx6sl() || cpu_is_imx6sx()) + if (cpu_is_imx6sl() || cpu_is_imx6sx() || cpu_is_imx6ul()) val |= BM_CLPCR_BYP_MMDC_CH0_LPM_HS; else val |= BM_CLPCR_BYP_MMDC_CH1_LPM_HS; @@ -310,7 +310,7 @@ int imx6_set_lpm(enum mxc_cpu_pwr_mode mode) val |= 0x3 << BP_CLPCR_STBY_COUNT; val |= BM_CLPCR_VSTBY; val |= BM_CLPCR_SBYOS; - if (cpu_is_imx6sl()) + if (cpu_is_imx6sl() || cpu_is_imx6sx()) val |= BM_CLPCR_BYPASS_PMIC_READY; if (cpu_is_imx6sl() || cpu_is_imx6sx() || cpu_is_imx6ul()) val |= BM_CLPCR_BYP_MMDC_CH0_LPM_HS; diff --git a/kernel/arch/arm/mach-mvebu/coherency.c b/kernel/arch/arm/mach-mvebu/coherency.c index 55348ee5a..feed36b32 100644 --- a/kernel/arch/arm/mach-mvebu/coherency.c +++ b/kernel/arch/arm/mach-mvebu/coherency.c @@ -162,22 +162,16 @@ exit: } /* - * This ioremap hook is used on Armada 375/38x to ensure that PCIe - * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This - * is needed as a workaround for a deadlock issue between the PCIe - * interface and the cache controller. + * This ioremap hook is used on Armada 375/38x to ensure that all MMIO + * areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is + * needed for the HW I/O coherency mechanism to work properly without + * deadlock. */ static void __iomem * -armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size, - unsigned int mtype, void *caller) +armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size, + unsigned int mtype, void *caller) { - struct resource pcie_mem; - - mvebu_mbus_get_pcie_mem_aperture(&pcie_mem); - - if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end) - mtype = MT_UNCACHED; - + mtype = MT_UNCACHED; return __arm_ioremap_caller(phys_addr, size, mtype, caller); } @@ -186,7 +180,7 @@ static void __init armada_375_380_coherency_init(struct device_node *np) struct device_node *cache_dn; coherency_cpu_base = of_iomap(np, 0); - arch_ioremap_caller = armada_pcie_wa_ioremap_caller; + arch_ioremap_caller = armada_wa_ioremap_caller; /* * We should switch the PL310 to I/O coherency mode only if diff --git a/kernel/arch/arm/mach-omap2/cpuidle34xx.c b/kernel/arch/arm/mach-omap2/cpuidle34xx.c index aa7b379e2..2a3db0bd9 100644 --- a/kernel/arch/arm/mach-omap2/cpuidle34xx.c +++ b/kernel/arch/arm/mach-omap2/cpuidle34xx.c @@ -34,6 +34,7 @@ #include "pm.h" #include "control.h" #include "common.h" +#include "soc.h" /* Mach specific information to be recorded in the C-state driver_data */ struct omap3_idle_statedata { @@ -315,6 +316,69 @@ static struct cpuidle_driver omap3_idle_driver = { .safe_state_index = 0, }; +/* + * Numbers based on measurements made in October 2009 for PM optimized kernel + * with CPU freq enabled on device Nokia N900. Assumes OPP2 (main idle OPP, + * and worst case latencies). + */ +static struct cpuidle_driver omap3430_idle_driver = { + .name = "omap3430_idle", + .owner = THIS_MODULE, + .states = { + { + .enter = omap3_enter_idle_bm, + .exit_latency = 110 + 162, + .target_residency = 5, + .name = "C1", + .desc = "MPU ON + CORE ON", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 106 + 180, + .target_residency = 309, + .name = "C2", + .desc = "MPU ON + CORE ON", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 107 + 410, + .target_residency = 46057, + .name = "C3", + .desc = "MPU RET + CORE ON", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 121 + 3374, + .target_residency = 46057, + .name = "C4", + .desc = "MPU OFF + CORE ON", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 855 + 1146, + .target_residency = 46057, + .name = "C5", + .desc = "MPU RET + CORE RET", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 7580 + 4134, + .target_residency = 484329, + .name = "C6", + .desc = "MPU OFF + CORE RET", + }, + { + .enter = omap3_enter_idle_bm, + .exit_latency = 7505 + 15274, + .target_residency = 484329, + .name = "C7", + .desc = "MPU OFF + CORE OFF", + }, + }, + .state_count = ARRAY_SIZE(omap3_idle_data), + .safe_state_index = 0, +}; + /* Public functions */ /** @@ -333,5 +397,8 @@ int __init omap3_idle_init(void) if (!mpu_pd || !core_pd || !per_pd || !cam_pd) return -ENODEV; - return cpuidle_register(&omap3_idle_driver, NULL); + if (cpu_is_omap3430()) + return cpuidle_register(&omap3430_idle_driver, NULL); + else + return cpuidle_register(&omap3_idle_driver, NULL); } diff --git a/kernel/arch/arm/mach-omap2/io.c b/kernel/arch/arm/mach-omap2/io.c index 3eaeaca5d..3a911d8de 100644 --- a/kernel/arch/arm/mach-omap2/io.c +++ b/kernel/arch/arm/mach-omap2/io.c @@ -368,6 +368,7 @@ void __init omap5_map_io(void) void __init dra7xx_map_io(void) { iotable_init(dra7xx_io_desc, ARRAY_SIZE(dra7xx_io_desc)); + omap_barriers_init(); } #endif /* diff --git a/kernel/arch/arm/mach-omap2/omap-mpuss-lowpower.c b/kernel/arch/arm/mach-omap2/omap-mpuss-lowpower.c index 65024af16..d3c14da7d 100644 --- a/kernel/arch/arm/mach-omap2/omap-mpuss-lowpower.c +++ b/kernel/arch/arm/mach-omap2/omap-mpuss-lowpower.c @@ -243,10 +243,9 @@ int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state) save_state = 1; break; case PWRDM_POWER_RET: - if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE)) { + if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE)) save_state = 0; - break; - } + break; default: /* * CPUx CSWR is invalid hardware state. Also CPUx OSWR diff --git a/kernel/arch/arm/mach-omap2/omap_hwmod.c b/kernel/arch/arm/mach-omap2/omap_hwmod.c index 8e0bd5939..147c90e70 100644 --- a/kernel/arch/arm/mach-omap2/omap_hwmod.c +++ b/kernel/arch/arm/mach-omap2/omap_hwmod.c @@ -1416,9 +1416,7 @@ static void _enable_sysc(struct omap_hwmod *oh) (sf & SYSC_HAS_CLOCKACTIVITY)) _set_clockactivity(oh, oh->class->sysc->clockact, &v); - /* If the cached value is the same as the new value, skip the write */ - if (oh->_sysc_cache != v) - _write_sysconfig(v, oh); + _write_sysconfig(v, oh); /* * Set the autoidle bit only after setting the smartidle bit @@ -1481,7 +1479,9 @@ static void _idle_sysc(struct omap_hwmod *oh) _set_master_standbymode(oh, idlemode, &v); } - _write_sysconfig(v, oh); + /* If the cached value is the same as the new value, skip the write */ + if (oh->_sysc_cache != v) + _write_sysconfig(v, oh); } /** diff --git a/kernel/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c b/kernel/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c index 907a452b7..b31ad596b 100644 --- a/kernel/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c +++ b/kernel/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c @@ -1474,6 +1474,7 @@ static void omap_hwmod_am43xx_rst(void) { RSTCTRL(am33xx_pruss_hwmod, AM43XX_RM_PER_RSTCTRL_OFFSET); RSTCTRL(am33xx_gfx_hwmod, AM43XX_RM_GFX_RSTCTRL_OFFSET); + RSTST(am33xx_pruss_hwmod, AM43XX_RM_PER_RSTST_OFFSET); RSTST(am33xx_gfx_hwmod, AM43XX_RM_GFX_RSTST_OFFSET); } diff --git a/kernel/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c b/kernel/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c index aff78d519..131f89675 100644 --- a/kernel/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c +++ b/kernel/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c @@ -723,8 +723,20 @@ static struct omap_hwmod omap3xxx_dss_dispc_hwmod = { * display serial interface controller */ +static struct omap_hwmod_class_sysconfig omap3xxx_dsi_sysc = { + .rev_offs = 0x0000, + .sysc_offs = 0x0010, + .syss_offs = 0x0014, + .sysc_flags = (SYSC_HAS_AUTOIDLE | SYSC_HAS_CLOCKACTIVITY | + SYSC_HAS_ENAWAKEUP | SYSC_HAS_SIDLEMODE | + SYSC_HAS_SOFTRESET | SYSS_HAS_RESET_STATUS), + .idlemodes = (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART), + .sysc_fields = &omap_hwmod_sysc_type1, +}; + static struct omap_hwmod_class omap3xxx_dsi_hwmod_class = { .name = "dsi", + .sysc = &omap3xxx_dsi_sysc, }; static struct omap_hwmod_irq_info omap3xxx_dsi1_irqs[] = { diff --git a/kernel/arch/arm/mach-omap2/prcm43xx.h b/kernel/arch/arm/mach-omap2/prcm43xx.h index 7c34c44eb..babb5db5a 100644 --- a/kernel/arch/arm/mach-omap2/prcm43xx.h +++ b/kernel/arch/arm/mach-omap2/prcm43xx.h @@ -39,6 +39,7 @@ /* RM RSTST offsets */ #define AM43XX_RM_GFX_RSTST_OFFSET 0x0014 +#define AM43XX_RM_PER_RSTST_OFFSET 0x0014 #define AM43XX_RM_WKUP_RSTST_OFFSET 0x0014 /* CM instances */ diff --git a/kernel/arch/arm/mach-prima2/Kconfig b/kernel/arch/arm/mach-prima2/Kconfig index 9ab893240..56e55fd37 100644 --- a/kernel/arch/arm/mach-prima2/Kconfig +++ b/kernel/arch/arm/mach-prima2/Kconfig @@ -1,6 +1,7 @@ menuconfig ARCH_SIRF bool "CSR SiRF" if ARCH_MULTI_V7 select ARCH_HAS_RESET_CONTROLLER + select RESET_CONTROLLER select ARCH_REQUIRE_GPIOLIB select GENERIC_IRQ_CHIP select NO_IOPORT_MAP diff --git a/kernel/arch/arm/mach-pxa/idp.c b/kernel/arch/arm/mach-pxa/idp.c index f6d02e4cb..5c87dff5d 100644 --- a/kernel/arch/arm/mach-pxa/idp.c +++ b/kernel/arch/arm/mach-pxa/idp.c @@ -83,7 +83,8 @@ static struct resource smc91x_resources[] = { }; static struct smc91x_platdata smc91x_platdata = { - .flags = SMC91X_USE_32BIT | SMC91X_USE_DMA | SMC91X_NOWAIT, + .flags = SMC91X_USE_8BIT | SMC91X_USE_16BIT | SMC91X_USE_32BIT | + SMC91X_USE_DMA | SMC91X_NOWAIT, }; static struct platform_device smc91x_device = { diff --git a/kernel/arch/arm/mach-pxa/pxa_cplds_irqs.c b/kernel/arch/arm/mach-pxa/pxa_cplds_irqs.c index 2385052b0..e362f865f 100644 --- a/kernel/arch/arm/mach-pxa/pxa_cplds_irqs.c +++ b/kernel/arch/arm/mach-pxa/pxa_cplds_irqs.c @@ -41,30 +41,35 @@ static irqreturn_t cplds_irq_handler(int in_irq, void *d) unsigned long pending; unsigned int bit; - pending = readl(fpga->base + FPGA_IRQ_SET_CLR) & fpga->irq_mask; - for_each_set_bit(bit, &pending, CPLDS_NB_IRQ) - generic_handle_irq(irq_find_mapping(fpga->irqdomain, bit)); + do { + pending = readl(fpga->base + FPGA_IRQ_SET_CLR) & fpga->irq_mask; + for_each_set_bit(bit, &pending, CPLDS_NB_IRQ) { + generic_handle_irq(irq_find_mapping(fpga->irqdomain, + bit)); + } + } while (pending); return IRQ_HANDLED; } -static void cplds_irq_mask_ack(struct irq_data *d) +static void cplds_irq_mask(struct irq_data *d) { struct cplds *fpga = irq_data_get_irq_chip_data(d); unsigned int cplds_irq = irqd_to_hwirq(d); - unsigned int set, bit = BIT(cplds_irq); + unsigned int bit = BIT(cplds_irq); fpga->irq_mask &= ~bit; writel(fpga->irq_mask, fpga->base + FPGA_IRQ_MASK_EN); - set = readl(fpga->base + FPGA_IRQ_SET_CLR); - writel(set & ~bit, fpga->base + FPGA_IRQ_SET_CLR); } static void cplds_irq_unmask(struct irq_data *d) { struct cplds *fpga = irq_data_get_irq_chip_data(d); unsigned int cplds_irq = irqd_to_hwirq(d); - unsigned int bit = BIT(cplds_irq); + unsigned int set, bit = BIT(cplds_irq); + + set = readl(fpga->base + FPGA_IRQ_SET_CLR); + writel(set & ~bit, fpga->base + FPGA_IRQ_SET_CLR); fpga->irq_mask |= bit; writel(fpga->irq_mask, fpga->base + FPGA_IRQ_MASK_EN); @@ -72,7 +77,8 @@ static void cplds_irq_unmask(struct irq_data *d) static struct irq_chip cplds_irq_chip = { .name = "pxa_cplds", - .irq_mask_ack = cplds_irq_mask_ack, + .irq_ack = cplds_irq_mask, + .irq_mask = cplds_irq_mask, .irq_unmask = cplds_irq_unmask, .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE, }; diff --git a/kernel/arch/arm/mach-pxa/xcep.c b/kernel/arch/arm/mach-pxa/xcep.c index 13b1d4586..900131271 100644 --- a/kernel/arch/arm/mach-pxa/xcep.c +++ b/kernel/arch/arm/mach-pxa/xcep.c @@ -120,7 +120,8 @@ static struct resource smc91x_resources[] = { }; static struct smc91x_platdata xcep_smc91x_info = { - .flags = SMC91X_USE_32BIT | SMC91X_NOWAIT | SMC91X_USE_DMA, + .flags = SMC91X_USE_8BIT | SMC91X_USE_16BIT | SMC91X_USE_32BIT | + SMC91X_NOWAIT | SMC91X_USE_DMA, }; static struct platform_device smc91x_device = { diff --git a/kernel/arch/arm/mach-realview/core.c b/kernel/arch/arm/mach-realview/core.c index 44575edc4..cf0a7c235 100644 --- a/kernel/arch/arm/mach-realview/core.c +++ b/kernel/arch/arm/mach-realview/core.c @@ -95,7 +95,8 @@ static struct smsc911x_platform_config smsc911x_config = { }; static struct smc91x_platdata smc91x_platdata = { - .flags = SMC91X_USE_32BIT | SMC91X_NOWAIT, + .flags = SMC91X_USE_8BIT | SMC91X_USE_16BIT | SMC91X_USE_32BIT | + SMC91X_NOWAIT, }; static struct platform_device realview_eth_device = { diff --git a/kernel/arch/arm/mach-s3c64xx/dev-audio.c b/kernel/arch/arm/mach-s3c64xx/dev-audio.c index ff780a8d8..9a42736ef 100644 --- a/kernel/arch/arm/mach-s3c64xx/dev-audio.c +++ b/kernel/arch/arm/mach-s3c64xx/dev-audio.c @@ -54,12 +54,12 @@ static int s3c64xx_i2s_cfg_gpio(struct platform_device *pdev) static struct resource s3c64xx_iis0_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_IIS0, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_I2S0_OUT), - [2] = DEFINE_RES_DMA(DMACH_I2S0_IN), }; -static struct s3c_audio_pdata i2sv3_pdata = { +static struct s3c_audio_pdata i2s0_pdata = { .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .dma_playback = DMACH_I2S0_OUT, + .dma_capture = DMACH_I2S0_IN, }; struct platform_device s3c64xx_device_iis0 = { @@ -68,15 +68,19 @@ struct platform_device s3c64xx_device_iis0 = { .num_resources = ARRAY_SIZE(s3c64xx_iis0_resource), .resource = s3c64xx_iis0_resource, .dev = { - .platform_data = &i2sv3_pdata, + .platform_data = &i2s0_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis0); static struct resource s3c64xx_iis1_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_IIS1, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_I2S1_OUT), - [2] = DEFINE_RES_DMA(DMACH_I2S1_IN), +}; + +static struct s3c_audio_pdata i2s1_pdata = { + .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .dma_playback = DMACH_I2S1_OUT, + .dma_capture = DMACH_I2S1_IN, }; struct platform_device s3c64xx_device_iis1 = { @@ -85,19 +89,19 @@ struct platform_device s3c64xx_device_iis1 = { .num_resources = ARRAY_SIZE(s3c64xx_iis1_resource), .resource = s3c64xx_iis1_resource, .dev = { - .platform_data = &i2sv3_pdata, + .platform_data = &i2s1_pdata, }, }; EXPORT_SYMBOL(s3c64xx_device_iis1); static struct resource s3c64xx_iisv4_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_IISV4, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_HSI_I2SV40_TX), - [2] = DEFINE_RES_DMA(DMACH_HSI_I2SV40_RX), }; static struct s3c_audio_pdata i2sv4_pdata = { .cfg_gpio = s3c64xx_i2s_cfg_gpio, + .dma_playback = DMACH_HSI_I2SV40_TX, + .dma_capture = DMACH_HSI_I2SV40_RX, .type = { .i2s = { .quirks = QUIRK_PRI_6CHAN, @@ -142,12 +146,12 @@ static int s3c64xx_pcm_cfg_gpio(struct platform_device *pdev) static struct resource s3c64xx_pcm0_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_PCM0, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_PCM0_TX), - [2] = DEFINE_RES_DMA(DMACH_PCM0_RX), }; static struct s3c_audio_pdata s3c_pcm0_pdata = { .cfg_gpio = s3c64xx_pcm_cfg_gpio, + .dma_capture = DMACH_PCM0_RX, + .dma_playback = DMACH_PCM0_TX, }; struct platform_device s3c64xx_device_pcm0 = { @@ -163,12 +167,12 @@ EXPORT_SYMBOL(s3c64xx_device_pcm0); static struct resource s3c64xx_pcm1_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_PCM1, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_PCM1_TX), - [2] = DEFINE_RES_DMA(DMACH_PCM1_RX), }; static struct s3c_audio_pdata s3c_pcm1_pdata = { .cfg_gpio = s3c64xx_pcm_cfg_gpio, + .dma_playback = DMACH_PCM1_TX, + .dma_capture = DMACH_PCM1_RX, }; struct platform_device s3c64xx_device_pcm1 = { @@ -196,13 +200,14 @@ static int s3c64xx_ac97_cfg_gpe(struct platform_device *pdev) static struct resource s3c64xx_ac97_resource[] = { [0] = DEFINE_RES_MEM(S3C64XX_PA_AC97, SZ_256), - [1] = DEFINE_RES_DMA(DMACH_AC97_PCMOUT), - [2] = DEFINE_RES_DMA(DMACH_AC97_PCMIN), - [3] = DEFINE_RES_DMA(DMACH_AC97_MICIN), - [4] = DEFINE_RES_IRQ(IRQ_AC97), + [1] = DEFINE_RES_IRQ(IRQ_AC97), }; -static struct s3c_audio_pdata s3c_ac97_pdata; +static struct s3c_audio_pdata s3c_ac97_pdata = { + .dma_playback = DMACH_AC97_PCMOUT, + .dma_capture = DMACH_AC97_PCMIN, + .dma_capture_mic = DMACH_AC97_MICIN, +}; static u64 s3c64xx_ac97_dmamask = DMA_BIT_MASK(32); diff --git a/kernel/arch/arm/mach-s3c64xx/include/mach/dma.h b/kernel/arch/arm/mach-s3c64xx/include/mach/dma.h index 096e14073..9c739eafe 100644 --- a/kernel/arch/arm/mach-s3c64xx/include/mach/dma.h +++ b/kernel/arch/arm/mach-s3c64xx/include/mach/dma.h @@ -14,38 +14,38 @@ #define S3C64XX_DMA_CHAN(name) ((unsigned long)(name)) /* DMA0/SDMA0 */ -#define DMACH_UART0 S3C64XX_DMA_CHAN("uart0_tx") -#define DMACH_UART0_SRC2 S3C64XX_DMA_CHAN("uart0_rx") -#define DMACH_UART1 S3C64XX_DMA_CHAN("uart1_tx") -#define DMACH_UART1_SRC2 S3C64XX_DMA_CHAN("uart1_rx") -#define DMACH_UART2 S3C64XX_DMA_CHAN("uart2_tx") -#define DMACH_UART2_SRC2 S3C64XX_DMA_CHAN("uart2_rx") -#define DMACH_UART3 S3C64XX_DMA_CHAN("uart3_tx") -#define DMACH_UART3_SRC2 S3C64XX_DMA_CHAN("uart3_rx") -#define DMACH_PCM0_TX S3C64XX_DMA_CHAN("pcm0_tx") -#define DMACH_PCM0_RX S3C64XX_DMA_CHAN("pcm0_rx") -#define DMACH_I2S0_OUT S3C64XX_DMA_CHAN("i2s0_tx") -#define DMACH_I2S0_IN S3C64XX_DMA_CHAN("i2s0_rx") +#define DMACH_UART0 "uart0_tx" +#define DMACH_UART0_SRC2 "uart0_rx" +#define DMACH_UART1 "uart1_tx" +#define DMACH_UART1_SRC2 "uart1_rx" +#define DMACH_UART2 "uart2_tx" +#define DMACH_UART2_SRC2 "uart2_rx" +#define DMACH_UART3 "uart3_tx" +#define DMACH_UART3_SRC2 "uart3_rx" +#define DMACH_PCM0_TX "pcm0_tx" +#define DMACH_PCM0_RX "pcm0_rx" +#define DMACH_I2S0_OUT "i2s0_tx" +#define DMACH_I2S0_IN "i2s0_rx" #define DMACH_SPI0_TX S3C64XX_DMA_CHAN("spi0_tx") #define DMACH_SPI0_RX S3C64XX_DMA_CHAN("spi0_rx") -#define DMACH_HSI_I2SV40_TX S3C64XX_DMA_CHAN("i2s2_tx") -#define DMACH_HSI_I2SV40_RX S3C64XX_DMA_CHAN("i2s2_rx") +#define DMACH_HSI_I2SV40_TX "i2s2_tx" +#define DMACH_HSI_I2SV40_RX "i2s2_rx" /* DMA1/SDMA1 */ -#define DMACH_PCM1_TX S3C64XX_DMA_CHAN("pcm1_tx") -#define DMACH_PCM1_RX S3C64XX_DMA_CHAN("pcm1_rx") -#define DMACH_I2S1_OUT S3C64XX_DMA_CHAN("i2s1_tx") -#define DMACH_I2S1_IN S3C64XX_DMA_CHAN("i2s1_rx") +#define DMACH_PCM1_TX "pcm1_tx" +#define DMACH_PCM1_RX "pcm1_rx" +#define DMACH_I2S1_OUT "i2s1_tx" +#define DMACH_I2S1_IN "i2s1_rx" #define DMACH_SPI1_TX S3C64XX_DMA_CHAN("spi1_tx") #define DMACH_SPI1_RX S3C64XX_DMA_CHAN("spi1_rx") -#define DMACH_AC97_PCMOUT S3C64XX_DMA_CHAN("ac97_out") -#define DMACH_AC97_PCMIN S3C64XX_DMA_CHAN("ac97_in") -#define DMACH_AC97_MICIN S3C64XX_DMA_CHAN("ac97_mic") -#define DMACH_PWM S3C64XX_DMA_CHAN("pwm") -#define DMACH_IRDA S3C64XX_DMA_CHAN("irda") -#define DMACH_EXTERNAL S3C64XX_DMA_CHAN("external") -#define DMACH_SECURITY_RX S3C64XX_DMA_CHAN("sec_rx") -#define DMACH_SECURITY_TX S3C64XX_DMA_CHAN("sec_tx") +#define DMACH_AC97_PCMOUT "ac97_out" +#define DMACH_AC97_PCMIN "ac97_in" +#define DMACH_AC97_MICIN "ac97_mic" +#define DMACH_PWM "pwm" +#define DMACH_IRDA "irda" +#define DMACH_EXTERNAL "external" +#define DMACH_SECURITY_RX "sec_rx" +#define DMACH_SECURITY_TX "sec_tx" enum dma_ch { DMACH_MAX = 32 diff --git a/kernel/arch/arm/mach-sa1100/clock.c b/kernel/arch/arm/mach-sa1100/clock.c index cbf53bb9c..0db46895c 100644 --- a/kernel/arch/arm/mach-sa1100/clock.c +++ b/kernel/arch/arm/mach-sa1100/clock.c @@ -125,6 +125,8 @@ static unsigned long clk_36864_get_rate(struct clk *clk) } static struct clkops clk_36864_ops = { + .enable = clk_cpu_enable, + .disable = clk_cpu_disable, .get_rate = clk_36864_get_rate, }; @@ -140,9 +142,8 @@ static struct clk_lookup sa11xx_clkregs[] = { CLKDEV_INIT(NULL, "OSTIMER0", &clk_36864), }; -static int __init sa11xx_clk_init(void) +int __init sa11xx_clk_init(void) { clkdev_add_table(sa11xx_clkregs, ARRAY_SIZE(sa11xx_clkregs)); return 0; } -core_initcall(sa11xx_clk_init); diff --git a/kernel/arch/arm/mach-sa1100/generic.c b/kernel/arch/arm/mach-sa1100/generic.c index 345e63f4e..3e09beddb 100644 --- a/kernel/arch/arm/mach-sa1100/generic.c +++ b/kernel/arch/arm/mach-sa1100/generic.c @@ -34,6 +34,7 @@ #include #include +#include #include "generic.h" #include @@ -95,6 +96,8 @@ static void sa1100_power_off(void) void sa11x0_restart(enum reboot_mode mode, const char *cmd) { + clear_reset_status(RESET_STATUS_ALL); + if (mode == REBOOT_SOFT) { /* Jump into ROM at address 0 */ soft_restart(0); @@ -388,6 +391,7 @@ void __init sa1100_init_irq(void) sa11x0_init_irq_nodt(IRQ_GPIO0_SC, irq_resource.start); sa1100_init_gpio(); + sa11xx_clk_init(); } /* diff --git a/kernel/arch/arm/mach-sa1100/generic.h b/kernel/arch/arm/mach-sa1100/generic.h index 0d92e119b..68199b603 100644 --- a/kernel/arch/arm/mach-sa1100/generic.h +++ b/kernel/arch/arm/mach-sa1100/generic.h @@ -44,3 +44,5 @@ int sa11x0_pm_init(void); #else static inline int sa11x0_pm_init(void) { return 0; } #endif + +int sa11xx_clk_init(void); diff --git a/kernel/arch/arm/mach-sa1100/pleb.c b/kernel/arch/arm/mach-sa1100/pleb.c index 1525d7b5f..88149f85b 100644 --- a/kernel/arch/arm/mach-sa1100/pleb.c +++ b/kernel/arch/arm/mach-sa1100/pleb.c @@ -45,7 +45,7 @@ static struct resource smc91x_resources[] = { }; static struct smc91x_platdata smc91x_platdata = { - .flags = SMC91X_USE_16BIT | SMC91X_NOWAIT, + .flags = SMC91X_USE_16BIT | SMC91X_USE_8BIT | SMC91X_NOWAIT, }; static struct platform_device smc91x_device = { diff --git a/kernel/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c b/kernel/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c index 62437b578..73e3adbc1 100644 --- a/kernel/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c +++ b/kernel/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c @@ -41,39 +41,26 @@ #define REGULATOR_IRQ_MASK BIT(2) /* IRQ2, active low */ -static void __iomem *irqc; - -static const u8 da9063_mask_regs[] = { - DA9063_REG_IRQ_MASK_A, - DA9063_REG_IRQ_MASK_B, - DA9063_REG_IRQ_MASK_C, - DA9063_REG_IRQ_MASK_D, -}; - -/* DA9210 System Control and Event Registers */ +/* start of DA9210 System Control and Event Registers */ #define DA9210_REG_MASK_A 0x54 -#define DA9210_REG_MASK_B 0x55 - -static const u8 da9210_mask_regs[] = { - DA9210_REG_MASK_A, - DA9210_REG_MASK_B, -}; - -static void da9xxx_mask_irqs(struct i2c_client *client, const u8 regs[], - unsigned int nregs) -{ - unsigned int i; - dev_info(&client->dev, "Masking %s interrupt sources\n", client->name); +static void __iomem *irqc; - for (i = 0; i < nregs; i++) { - int error = i2c_smbus_write_byte_data(client, regs[i], ~0); - if (error) { - dev_err(&client->dev, "i2c error %d\n", error); - return; - } - } -} +/* first byte sets the memory pointer, following are consecutive reg values */ +static u8 da9063_irq_clr[] = { DA9063_REG_IRQ_MASK_A, 0xff, 0xff, 0xff, 0xff }; +static u8 da9210_irq_clr[] = { DA9210_REG_MASK_A, 0xff, 0xff }; + +static struct i2c_msg da9xxx_msgs[2] = { + { + .addr = 0x58, + .len = ARRAY_SIZE(da9063_irq_clr), + .buf = da9063_irq_clr, + }, { + .addr = 0x68, + .len = ARRAY_SIZE(da9210_irq_clr), + .buf = da9210_irq_clr, + }, +}; static int regulator_quirk_notify(struct notifier_block *nb, unsigned long action, void *data) @@ -93,12 +80,15 @@ static int regulator_quirk_notify(struct notifier_block *nb, client = to_i2c_client(dev); dev_dbg(dev, "Detected %s\n", client->name); - if ((client->addr == 0x58 && !strcmp(client->name, "da9063"))) - da9xxx_mask_irqs(client, da9063_mask_regs, - ARRAY_SIZE(da9063_mask_regs)); - else if (client->addr == 0x68 && !strcmp(client->name, "da9210")) - da9xxx_mask_irqs(client, da9210_mask_regs, - ARRAY_SIZE(da9210_mask_regs)); + if ((client->addr == 0x58 && !strcmp(client->name, "da9063")) || + (client->addr == 0x68 && !strcmp(client->name, "da9210"))) { + int ret; + + dev_info(&client->dev, "clearing da9063/da9210 interrupts\n"); + ret = i2c_transfer(client->adapter, da9xxx_msgs, ARRAY_SIZE(da9xxx_msgs)); + if (ret != ARRAY_SIZE(da9xxx_msgs)) + dev_err(&client->dev, "i2c error %d\n", ret); + } mon = ioread32(irqc + IRQC_MONITOR); if (mon & REGULATOR_IRQ_MASK) diff --git a/kernel/arch/arm/mach-socfpga/headsmp.S b/kernel/arch/arm/mach-socfpga/headsmp.S index 5d94b7a2f..c160fa300 100644 --- a/kernel/arch/arm/mach-socfpga/headsmp.S +++ b/kernel/arch/arm/mach-socfpga/headsmp.S @@ -13,6 +13,7 @@ #include .arch armv7-a + .arm ENTRY(secondary_trampoline) /* CPU1 will always fetch from 0x0 when it is brought out of reset. diff --git a/kernel/arch/arm/mach-ux500/pm.c b/kernel/arch/arm/mach-ux500/pm.c index 8538910db..a970e7fcb 100644 --- a/kernel/arch/arm/mach-ux500/pm.c +++ b/kernel/arch/arm/mach-ux500/pm.c @@ -134,8 +134,8 @@ bool prcmu_pending_irq(void) */ bool prcmu_is_cpu_in_wfi(int cpu) { - return readl(PRCM_ARM_WFI_STANDBY) & cpu ? PRCM_ARM_WFI_STANDBY_WFI1 : - PRCM_ARM_WFI_STANDBY_WFI0; + return readl(PRCM_ARM_WFI_STANDBY) & + (cpu ? PRCM_ARM_WFI_STANDBY_WFI1 : PRCM_ARM_WFI_STANDBY_WFI0); } /* diff --git a/kernel/arch/arm/mach-zynq/common.c b/kernel/arch/arm/mach-zynq/common.c index 6f39d03cc..0a43143e9 100644 --- a/kernel/arch/arm/mach-zynq/common.c +++ b/kernel/arch/arm/mach-zynq/common.c @@ -59,7 +59,7 @@ void __iomem *zynq_scu_base; static void __init zynq_memory_init(void) { if (!__pa(PAGE_OFFSET)) - memblock_reserve(__pa(PAGE_OFFSET), __pa(swapper_pg_dir)); + memblock_reserve(__pa(PAGE_OFFSET), 0x80000); } static struct platform_device zynq_cpuidle_device = { diff --git a/kernel/arch/arm/mm/fault.c b/kernel/arch/arm/mm/fault.c index b8aa1e9ee..276ac73f8 100644 --- a/kernel/arch/arm/mm/fault.c +++ b/kernel/arch/arm/mm/fault.c @@ -616,9 +616,9 @@ static int __init early_abort_handler(unsigned long addr, unsigned int fsr, void __init early_abt_enable(void) { - fsr_info[22].fn = early_abort_handler; + fsr_info[FSR_FS_AEA].fn = early_abort_handler; local_abt_enable(); - fsr_info[22].fn = do_bad; + fsr_info[FSR_FS_AEA].fn = do_bad; } #ifndef CONFIG_ARM_LPAE diff --git a/kernel/arch/arm/mm/fault.h b/kernel/arch/arm/mm/fault.h index 05ec5e0df..78830657c 100644 --- a/kernel/arch/arm/mm/fault.h +++ b/kernel/arch/arm/mm/fault.h @@ -11,11 +11,15 @@ #define FSR_FS5_0 (0x3f) #ifdef CONFIG_ARM_LPAE +#define FSR_FS_AEA 17 + static inline int fsr_fs(unsigned int fsr) { return fsr & FSR_FS5_0; } #else +#define FSR_FS_AEA 22 + static inline int fsr_fs(unsigned int fsr) { return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6; diff --git a/kernel/arch/arm/plat-samsung/devs.c b/kernel/arch/arm/plat-samsung/devs.c index 82074625d..e212f9d80 100644 --- a/kernel/arch/arm/plat-samsung/devs.c +++ b/kernel/arch/arm/plat-samsung/devs.c @@ -65,6 +65,7 @@ #include #include #include +#include #include static u64 samsung_device_dma_mask = DMA_BIT_MASK(32); @@ -74,9 +75,12 @@ static u64 samsung_device_dma_mask = DMA_BIT_MASK(32); static struct resource s3c_ac97_resource[] = { [0] = DEFINE_RES_MEM(S3C2440_PA_AC97, S3C2440_SZ_AC97), [1] = DEFINE_RES_IRQ(IRQ_S3C244X_AC97), - [2] = DEFINE_RES_DMA_NAMED(DMACH_PCM_OUT, "PCM out"), - [3] = DEFINE_RES_DMA_NAMED(DMACH_PCM_IN, "PCM in"), - [4] = DEFINE_RES_DMA_NAMED(DMACH_MIC_IN, "Mic in"), +}; + +static struct s3c_audio_pdata s3c_ac97_pdata = { + .dma_playback = (void *)DMACH_PCM_OUT, + .dma_capture = (void *)DMACH_PCM_IN, + .dma_capture_mic = (void *)DMACH_MIC_IN, }; struct platform_device s3c_device_ac97 = { @@ -87,6 +91,7 @@ struct platform_device s3c_device_ac97 = { .dev = { .dma_mask = &samsung_device_dma_mask, .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &s3c_ac97_pdata, } }; #endif /* CONFIG_CPU_S3C2440 */ diff --git a/kernel/arch/arm/xen/enlighten.c b/kernel/arch/arm/xen/enlighten.c index fc7ea529f..52c8c1f64 100644 --- a/kernel/arch/arm/xen/enlighten.c +++ b/kernel/arch/arm/xen/enlighten.c @@ -239,8 +239,7 @@ static int __init xen_guest_init(void) * for secondary CPUs as they are brought up. * For uniformity we use VCPUOP_register_vcpu_info even on cpu0. */ - xen_vcpu_info = __alloc_percpu(sizeof(struct vcpu_info), - sizeof(struct vcpu_info)); + xen_vcpu_info = alloc_percpu(struct vcpu_info); if (xen_vcpu_info == NULL) return -ENOMEM; diff --git a/kernel/arch/arm64/Kconfig b/kernel/arch/arm64/Kconfig index 1baa6537c..9196cf82f 100644 --- a/kernel/arch/arm64/Kconfig +++ b/kernel/arch/arm64/Kconfig @@ -392,6 +392,15 @@ config CAVIUM_ERRATUM_22375 If unsure, say Y. +config CAVIUM_ERRATUM_23144 + bool "Cavium erratum 23144: ITS SYNC hang on dual socket system" + depends on NUMA + default y + help + ITS SYNC command hang for cross node io and collections/cpu mapping. + + If unsure, say Y. + config CAVIUM_ERRATUM_23154 bool "Cavium erratum 23154: Access to ICC_IAR1_EL1 is not sync'ed" default y @@ -402,6 +411,17 @@ config CAVIUM_ERRATUM_23154 If unsure, say Y. +config CAVIUM_ERRATUM_27456 + bool "Cavium erratum 27456: Broadcast TLBI instructions may cause icache corruption" + default y + help + On ThunderX T88 pass 1.x through 2.1 parts, broadcast TLBI + instructions may cause the icache to become corrupted if it + contains data for a non-current ASID. The fix is to + invalidate the icache when changing the mm context. + + If unsure, say Y. + endmenu diff --git a/kernel/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/kernel/arch/arm64/boot/dts/rockchip/rk3368.dtsi index cc093a482..e0ee2b00d 100644 --- a/kernel/arch/arm64/boot/dts/rockchip/rk3368.dtsi +++ b/kernel/arch/arm64/boot/dts/rockchip/rk3368.dtsi @@ -262,6 +262,8 @@ #io-channel-cells = <1>; clocks = <&cru SCLK_SARADC>, <&cru PCLK_SARADC>; clock-names = "saradc", "apb_pclk"; + resets = <&cru SRST_SARADC>; + reset-names = "saradc-apb"; status = "disabled"; }; @@ -517,7 +519,7 @@ #address-cells = <0>; reg = <0x0 0xffb71000 0x0 0x1000>, - <0x0 0xffb72000 0x0 0x1000>, + <0x0 0xffb72000 0x0 0x2000>, <0x0 0xffb74000 0x0 0x2000>, <0x0 0xffb76000 0x0 0x2000>; interrupts = +#include .text .arch armv8-a+crypto @@ -19,7 +20,7 @@ */ ENTRY(ce_aes_ccm_auth_data) ldr w8, [x3] /* leftover from prev round? */ - ld1 {v0.2d}, [x0] /* load mac */ + ld1 {v0.16b}, [x0] /* load mac */ cbz w8, 1f sub w8, w8, #16 eor v1.16b, v1.16b, v1.16b @@ -31,7 +32,7 @@ ENTRY(ce_aes_ccm_auth_data) beq 8f /* out of input? */ cbnz w8, 0b eor v0.16b, v0.16b, v1.16b -1: ld1 {v3.2d}, [x4] /* load first round key */ +1: ld1 {v3.16b}, [x4] /* load first round key */ prfm pldl1strm, [x1] cmp w5, #12 /* which key size? */ add x6, x4, #16 @@ -41,17 +42,17 @@ ENTRY(ce_aes_ccm_auth_data) mov v5.16b, v3.16b b 4f 2: mov v4.16b, v3.16b - ld1 {v5.2d}, [x6], #16 /* load 2nd round key */ + ld1 {v5.16b}, [x6], #16 /* load 2nd round key */ 3: aese v0.16b, v4.16b aesmc v0.16b, v0.16b -4: ld1 {v3.2d}, [x6], #16 /* load next round key */ +4: ld1 {v3.16b}, [x6], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b -5: ld1 {v4.2d}, [x6], #16 /* load next round key */ +5: ld1 {v4.16b}, [x6], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b - ld1 {v5.2d}, [x6], #16 /* load next round key */ + ld1 {v5.16b}, [x6], #16 /* load next round key */ bpl 3b aese v0.16b, v4.16b subs w2, w2, #16 /* last data? */ @@ -60,7 +61,7 @@ ENTRY(ce_aes_ccm_auth_data) ld1 {v1.16b}, [x1], #16 /* load next input block */ eor v0.16b, v0.16b, v1.16b /* xor with mac */ bne 1b -6: st1 {v0.2d}, [x0] /* store mac */ +6: st1 {v0.16b}, [x0] /* store mac */ beq 10f adds w2, w2, #16 beq 10f @@ -79,7 +80,7 @@ ENTRY(ce_aes_ccm_auth_data) adds w7, w7, #1 bne 9b eor v0.16b, v0.16b, v1.16b - st1 {v0.2d}, [x0] + st1 {v0.16b}, [x0] 10: str w8, [x3] ret ENDPROC(ce_aes_ccm_auth_data) @@ -89,27 +90,27 @@ ENDPROC(ce_aes_ccm_auth_data) * u32 rounds); */ ENTRY(ce_aes_ccm_final) - ld1 {v3.2d}, [x2], #16 /* load first round key */ - ld1 {v0.2d}, [x0] /* load mac */ + ld1 {v3.16b}, [x2], #16 /* load first round key */ + ld1 {v0.16b}, [x0] /* load mac */ cmp w3, #12 /* which key size? */ sub w3, w3, #2 /* modified # of rounds */ - ld1 {v1.2d}, [x1] /* load 1st ctriv */ + ld1 {v1.16b}, [x1] /* load 1st ctriv */ bmi 0f bne 3f mov v5.16b, v3.16b b 2f 0: mov v4.16b, v3.16b -1: ld1 {v5.2d}, [x2], #16 /* load next round key */ +1: ld1 {v5.16b}, [x2], #16 /* load next round key */ aese v0.16b, v4.16b aesmc v0.16b, v0.16b aese v1.16b, v4.16b aesmc v1.16b, v1.16b -2: ld1 {v3.2d}, [x2], #16 /* load next round key */ +2: ld1 {v3.16b}, [x2], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b aese v1.16b, v5.16b aesmc v1.16b, v1.16b -3: ld1 {v4.2d}, [x2], #16 /* load next round key */ +3: ld1 {v4.16b}, [x2], #16 /* load next round key */ subs w3, w3, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b @@ -120,47 +121,47 @@ ENTRY(ce_aes_ccm_final) aese v1.16b, v4.16b /* final round key cancels out */ eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */ - st1 {v0.2d}, [x0] /* store result */ + st1 {v0.16b}, [x0] /* store result */ ret ENDPROC(ce_aes_ccm_final) .macro aes_ccm_do_crypt,enc ldr x8, [x6, #8] /* load lower ctr */ - ld1 {v0.2d}, [x5] /* load mac */ - rev x8, x8 /* keep swabbed ctr in reg */ + ld1 {v0.16b}, [x5] /* load mac */ +CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ 0: /* outer loop */ - ld1 {v1.1d}, [x6] /* load upper ctr */ + ld1 {v1.8b}, [x6] /* load upper ctr */ prfm pldl1strm, [x1] add x8, x8, #1 rev x9, x8 cmp w4, #12 /* which key size? */ sub w7, w4, #2 /* get modified # of rounds */ ins v1.d[1], x9 /* no carry in lower ctr */ - ld1 {v3.2d}, [x3] /* load first round key */ + ld1 {v3.16b}, [x3] /* load first round key */ add x10, x3, #16 bmi 1f bne 4f mov v5.16b, v3.16b b 3f 1: mov v4.16b, v3.16b - ld1 {v5.2d}, [x10], #16 /* load 2nd round key */ + ld1 {v5.16b}, [x10], #16 /* load 2nd round key */ 2: /* inner loop: 3 rounds, 2x interleaved */ aese v0.16b, v4.16b aesmc v0.16b, v0.16b aese v1.16b, v4.16b aesmc v1.16b, v1.16b -3: ld1 {v3.2d}, [x10], #16 /* load next round key */ +3: ld1 {v3.16b}, [x10], #16 /* load next round key */ aese v0.16b, v5.16b aesmc v0.16b, v0.16b aese v1.16b, v5.16b aesmc v1.16b, v1.16b -4: ld1 {v4.2d}, [x10], #16 /* load next round key */ +4: ld1 {v4.16b}, [x10], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b aesmc v0.16b, v0.16b aese v1.16b, v3.16b aesmc v1.16b, v1.16b - ld1 {v5.2d}, [x10], #16 /* load next round key */ + ld1 {v5.16b}, [x10], #16 /* load next round key */ bpl 2b aese v0.16b, v4.16b aese v1.16b, v4.16b @@ -177,14 +178,14 @@ ENDPROC(ce_aes_ccm_final) eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ st1 {v1.16b}, [x0], #16 /* write output block */ bne 0b - rev x8, x8 - st1 {v0.2d}, [x5] /* store mac */ +CPU_LE( rev x8, x8 ) + st1 {v0.16b}, [x5] /* store mac */ str x8, [x6, #8] /* store lsb end of ctr (BE) */ 5: ret 6: eor v0.16b, v0.16b, v5.16b /* final round mac */ eor v1.16b, v1.16b, v5.16b /* final round enc */ - st1 {v0.2d}, [x5] /* store mac */ + st1 {v0.16b}, [x5] /* store mac */ add w2, w2, #16 /* process partial tail block */ 7: ldrb w9, [x1], #1 /* get 1 byte of input */ umov w6, v1.b[0] /* get top crypted ctr byte */ diff --git a/kernel/arch/arm64/crypto/aes-ce-cipher.c b/kernel/arch/arm64/crypto/aes-ce-cipher.c index f7bd9bf0b..50d9fe11d 100644 --- a/kernel/arch/arm64/crypto/aes-ce-cipher.c +++ b/kernel/arch/arm64/crypto/aes-ce-cipher.c @@ -47,24 +47,24 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) kernel_neon_begin_partial(4); __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.2d}, [%[key]], #16 ;" + " ld1 {v1.16b}, [%[key]], #16 ;" " cmp %w[rounds], #10 ;" " bmi 0f ;" " bne 3f ;" " mov v3.16b, v1.16b ;" " b 2f ;" "0: mov v2.16b, v1.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" "1: aese v0.16b, v2.16b ;" " aesmc v0.16b, v0.16b ;" - "2: ld1 {v1.2d}, [%[key]], #16 ;" + "2: ld1 {v1.16b}, [%[key]], #16 ;" " aese v0.16b, v3.16b ;" " aesmc v0.16b, v0.16b ;" - "3: ld1 {v2.2d}, [%[key]], #16 ;" + "3: ld1 {v2.16b}, [%[key]], #16 ;" " subs %w[rounds], %w[rounds], #3 ;" " aese v0.16b, v1.16b ;" " aesmc v0.16b, v0.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" " bpl 1b ;" " aese v0.16b, v2.16b ;" " eor v0.16b, v0.16b, v3.16b ;" @@ -92,24 +92,24 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) kernel_neon_begin_partial(4); __asm__(" ld1 {v0.16b}, %[in] ;" - " ld1 {v1.2d}, [%[key]], #16 ;" + " ld1 {v1.16b}, [%[key]], #16 ;" " cmp %w[rounds], #10 ;" " bmi 0f ;" " bne 3f ;" " mov v3.16b, v1.16b ;" " b 2f ;" "0: mov v2.16b, v1.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" "1: aesd v0.16b, v2.16b ;" " aesimc v0.16b, v0.16b ;" - "2: ld1 {v1.2d}, [%[key]], #16 ;" + "2: ld1 {v1.16b}, [%[key]], #16 ;" " aesd v0.16b, v3.16b ;" " aesimc v0.16b, v0.16b ;" - "3: ld1 {v2.2d}, [%[key]], #16 ;" + "3: ld1 {v2.16b}, [%[key]], #16 ;" " subs %w[rounds], %w[rounds], #3 ;" " aesd v0.16b, v1.16b ;" " aesimc v0.16b, v0.16b ;" - " ld1 {v3.2d}, [%[key]], #16 ;" + " ld1 {v3.16b}, [%[key]], #16 ;" " bpl 1b ;" " aesd v0.16b, v2.16b ;" " eor v0.16b, v0.16b, v3.16b ;" @@ -173,7 +173,12 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, u32 *rki = ctx->key_enc + (i * kwords); u32 *rko = rki + kwords; +#ifndef CONFIG_CPU_BIG_ENDIAN rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; +#else + rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^ + rki[0]; +#endif rko[1] = rko[0] ^ rki[1]; rko[2] = rko[1] ^ rki[2]; rko[3] = rko[2] ^ rki[3]; diff --git a/kernel/arch/arm64/crypto/aes-ce.S b/kernel/arch/arm64/crypto/aes-ce.S index 78f3cfe92..b46093d56 100644 --- a/kernel/arch/arm64/crypto/aes-ce.S +++ b/kernel/arch/arm64/crypto/aes-ce.S @@ -10,6 +10,7 @@ */ #include +#include #define AES_ENTRY(func) ENTRY(ce_ ## func) #define AES_ENDPROC(func) ENDPROC(ce_ ## func) diff --git a/kernel/arch/arm64/crypto/aes-glue.c b/kernel/arch/arm64/crypto/aes-glue.c index 05d9e16c0..6a51dfccf 100644 --- a/kernel/arch/arm64/crypto/aes-glue.c +++ b/kernel/arch/arm64/crypto/aes-glue.c @@ -211,7 +211,7 @@ static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } - if (nbytes) { + if (walk.nbytes % AES_BLOCK_SIZE) { u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; u8 __aligned(8) tail[AES_BLOCK_SIZE]; diff --git a/kernel/arch/arm64/crypto/aes-modes.S b/kernel/arch/arm64/crypto/aes-modes.S index f6e372c52..838dad5c2 100644 --- a/kernel/arch/arm64/crypto/aes-modes.S +++ b/kernel/arch/arm64/crypto/aes-modes.S @@ -193,15 +193,16 @@ AES_ENTRY(aes_cbc_encrypt) cbz w6, .Lcbcencloop ld1 {v0.16b}, [x5] /* get iv */ - enc_prepare w3, x2, x5 + enc_prepare w3, x2, x6 .Lcbcencloop: ld1 {v1.16b}, [x1], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with iv */ - encrypt_block v0, w3, x2, x5, w6 + encrypt_block v0, w3, x2, x6, w7 st1 {v0.16b}, [x0], #16 subs w4, w4, #1 bne .Lcbcencloop + st1 {v0.16b}, [x5] /* return iv */ ret AES_ENDPROC(aes_cbc_encrypt) @@ -211,7 +212,7 @@ AES_ENTRY(aes_cbc_decrypt) cbz w6, .LcbcdecloopNx ld1 {v7.16b}, [x5] /* get iv */ - dec_prepare w3, x2, x5 + dec_prepare w3, x2, x6 .LcbcdecloopNx: #if INTERLEAVE >= 2 @@ -248,7 +249,7 @@ AES_ENTRY(aes_cbc_decrypt) .Lcbcdecloop: ld1 {v1.16b}, [x1], #16 /* get next ct block */ mov v0.16b, v1.16b /* ...and copy to v0 */ - decrypt_block v0, w3, x2, x5, w6 + decrypt_block v0, w3, x2, x6, w7 eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ mov v7.16b, v1.16b /* ct is next iv */ st1 {v0.16b}, [x0], #16 @@ -256,6 +257,7 @@ AES_ENTRY(aes_cbc_decrypt) bne .Lcbcdecloop .Lcbcdecout: FRAME_POP + st1 {v7.16b}, [x5] /* return iv */ ret AES_ENDPROC(aes_cbc_decrypt) @@ -267,24 +269,15 @@ AES_ENDPROC(aes_cbc_decrypt) AES_ENTRY(aes_ctr_encrypt) FRAME_PUSH - cbnz w6, .Lctrfirst /* 1st time around? */ - umov x5, v4.d[1] /* keep swabbed ctr in reg */ - rev x5, x5 -#if INTERLEAVE >= 2 - cmn w5, w4 /* 32 bit overflow? */ - bcs .Lctrinc - add x5, x5, #1 /* increment BE ctr */ - b .LctrincNx -#else - b .Lctrinc -#endif -.Lctrfirst: + cbz w6, .Lctrnotfirst /* 1st time around? */ enc_prepare w3, x2, x6 ld1 {v4.16b}, [x5] - umov x5, v4.d[1] /* keep swabbed ctr in reg */ - rev x5, x5 + +.Lctrnotfirst: + umov x8, v4.d[1] /* keep swabbed ctr in reg */ + rev x8, x8 #if INTERLEAVE >= 2 - cmn w5, w4 /* 32 bit overflow? */ + cmn w8, w4 /* 32 bit overflow? */ bcs .Lctrloop .LctrloopNx: subs w4, w4, #INTERLEAVE @@ -292,11 +285,11 @@ AES_ENTRY(aes_ctr_encrypt) #if INTERLEAVE == 2 mov v0.8b, v4.8b mov v1.8b, v4.8b - rev x7, x5 - add x5, x5, #1 + rev x7, x8 + add x8, x8, #1 ins v0.d[1], x7 - rev x7, x5 - add x5, x5, #1 + rev x7, x8 + add x8, x8, #1 ins v1.d[1], x7 ld1 {v2.16b-v3.16b}, [x1], #32 /* get 2 input blocks */ do_encrypt_block2x @@ -305,7 +298,7 @@ AES_ENTRY(aes_ctr_encrypt) st1 {v0.16b-v1.16b}, [x0], #32 #else ldr q8, =0x30000000200000001 /* addends 1,2,3[,0] */ - dup v7.4s, w5 + dup v7.4s, w8 mov v0.16b, v4.16b add v7.4s, v7.4s, v8.4s mov v1.16b, v4.16b @@ -323,18 +316,12 @@ AES_ENTRY(aes_ctr_encrypt) eor v2.16b, v7.16b, v2.16b eor v3.16b, v5.16b, v3.16b st1 {v0.16b-v3.16b}, [x0], #64 - add x5, x5, #INTERLEAVE + add x8, x8, #INTERLEAVE #endif - cbz w4, .LctroutNx -.LctrincNx: - rev x7, x5 + rev x7, x8 ins v4.d[1], x7 + cbz w4, .Lctrout b .LctrloopNx -.LctroutNx: - sub x5, x5, #1 - rev x7, x5 - ins v4.d[1], x7 - b .Lctrout .Lctr1x: adds w4, w4, #INTERLEAVE beq .Lctrout @@ -342,30 +329,39 @@ AES_ENTRY(aes_ctr_encrypt) .Lctrloop: mov v0.16b, v4.16b encrypt_block v0, w3, x2, x6, w7 + + adds x8, x8, #1 /* increment BE ctr */ + rev x7, x8 + ins v4.d[1], x7 + bcs .Lctrcarry /* overflow? */ + +.Lctrcarrydone: subs w4, w4, #1 bmi .Lctrhalfblock /* blocks < 0 means 1/2 block */ ld1 {v3.16b}, [x1], #16 eor v3.16b, v0.16b, v3.16b st1 {v3.16b}, [x0], #16 - beq .Lctrout -.Lctrinc: - adds x5, x5, #1 /* increment BE ctr */ - rev x7, x5 - ins v4.d[1], x7 - bcc .Lctrloop /* no overflow? */ - umov x7, v4.d[0] /* load upper word of ctr */ - rev x7, x7 /* ... to handle the carry */ - add x7, x7, #1 - rev x7, x7 - ins v4.d[0], x7 - b .Lctrloop + bne .Lctrloop + +.Lctrout: + st1 {v4.16b}, [x5] /* return next CTR value */ + FRAME_POP + ret + .Lctrhalfblock: ld1 {v3.8b}, [x1] eor v3.8b, v0.8b, v3.8b st1 {v3.8b}, [x0] -.Lctrout: FRAME_POP ret + +.Lctrcarry: + umov x7, v4.d[0] /* load upper word of ctr */ + rev x7, x7 /* ... to handle the carry */ + add x7, x7, #1 + rev x7, x7 + ins v4.d[0], x7 + b .Lctrcarrydone AES_ENDPROC(aes_ctr_encrypt) .ltorg @@ -386,7 +382,8 @@ AES_ENDPROC(aes_ctr_encrypt) .endm .Lxts_mul_x: - .word 1, 0, 0x87, 0 +CPU_LE( .quad 1, 0x87 ) +CPU_BE( .quad 0x87, 1 ) AES_ENTRY(aes_xts_encrypt) FRAME_PUSH diff --git a/kernel/arch/arm64/crypto/aes-neon.S b/kernel/arch/arm64/crypto/aes-neon.S index b93170e1c..85f07ead7 100644 --- a/kernel/arch/arm64/crypto/aes-neon.S +++ b/kernel/arch/arm64/crypto/aes-neon.S @@ -9,6 +9,7 @@ */ #include +#include #define AES_ENTRY(func) ENTRY(neon_ ## func) #define AES_ENDPROC(func) ENDPROC(neon_ ## func) @@ -83,13 +84,13 @@ .endm .macro do_block, enc, in, rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in\().16b, \in\().16b, v15.16b /* ^round key */ tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */ sub_bytes \in - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -229,7 +230,7 @@ .endm .macro do_block_2x, enc, in0, in1 rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ @@ -237,7 +238,7 @@ sub_bytes_2x \in0, \in1 tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -254,7 +255,7 @@ .endm .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i - ld1 {v15.16b}, [\rk] + ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 mov \i, \rounds 1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ @@ -266,7 +267,7 @@ tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */ tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */ - ld1 {v15.16b}, [\rkp], #16 + ld1 {v15.4s}, [\rkp], #16 subs \i, \i, #1 beq 2222f .if \enc == 1 @@ -306,12 +307,16 @@ .text .align 4 .LForward_ShiftRows: - .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 - .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb +CPU_LE( .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 ) +CPU_LE( .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb ) +CPU_BE( .byte 0xb, 0x6, 0x1, 0xc, 0x7, 0x2, 0xd, 0x8 ) +CPU_BE( .byte 0x3, 0xe, 0x9, 0x4, 0xf, 0xa, 0x5, 0x0 ) .LReverse_ShiftRows: - .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb - .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 +CPU_LE( .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb ) +CPU_LE( .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 ) +CPU_BE( .byte 0x3, 0x6, 0x9, 0xc, 0xf, 0x2, 0x5, 0x8 ) +CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 ) .LForward_Sbox: .byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 diff --git a/kernel/arch/arm64/crypto/ghash-ce-core.S b/kernel/arch/arm64/crypto/ghash-ce-core.S index dc4570158..f0bb9f0b5 100644 --- a/kernel/arch/arm64/crypto/ghash-ce-core.S +++ b/kernel/arch/arm64/crypto/ghash-ce-core.S @@ -29,8 +29,8 @@ * struct ghash_key const *k, const char *head) */ ENTRY(pmull_ghash_update) - ld1 {SHASH.16b}, [x3] - ld1 {XL.16b}, [x1] + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] movi MASK.16b, #0xe1 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 shl MASK.2d, MASK.2d, #57 @@ -74,6 +74,6 @@ CPU_LE( rev64 T1.16b, T1.16b ) cbnz w0, 0b - st1 {XL.16b}, [x1] + st1 {XL.2d}, [x1] ret ENDPROC(pmull_ghash_update) diff --git a/kernel/arch/arm64/crypto/sha1-ce-core.S b/kernel/arch/arm64/crypto/sha1-ce-core.S index 033aae6d7..c98e7e849 100644 --- a/kernel/arch/arm64/crypto/sha1-ce-core.S +++ b/kernel/arch/arm64/crypto/sha1-ce-core.S @@ -78,7 +78,7 @@ ENTRY(sha1_ce_transform) ld1r {k3.4s}, [x6] /* load state */ - ldr dga, [x0] + ld1 {dgav.4s}, [x0] ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ @@ -144,7 +144,7 @@ CPU_LE( rev32 v11.16b, v11.16b ) b 1b /* store new state */ -3: str dga, [x0] +3: st1 {dgav.4s}, [x0] str dgb, [x0, #16] ret ENDPROC(sha1_ce_transform) diff --git a/kernel/arch/arm64/crypto/sha2-ce-core.S b/kernel/arch/arm64/crypto/sha2-ce-core.S index 5df9d9d47..01cfee066 100644 --- a/kernel/arch/arm64/crypto/sha2-ce-core.S +++ b/kernel/arch/arm64/crypto/sha2-ce-core.S @@ -85,7 +85,7 @@ ENTRY(sha2_ce_transform) ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ldp dga, dgb, [x0] + ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] @@ -148,6 +148,6 @@ CPU_LE( rev32 v19.16b, v19.16b ) b 1b /* store new state */ -3: stp dga, dgb, [x0] +3: st1 {dgav.4s, dgbv.4s}, [x0] ret ENDPROC(sha2_ce_transform) diff --git a/kernel/arch/arm64/include/asm/arch_gicv3.h b/kernel/arch/arm64/include/asm/arch_gicv3.h index 2731d3b25..8ec88e5b2 100644 --- a/kernel/arch/arm64/include/asm/arch_gicv3.h +++ b/kernel/arch/arm64/include/asm/arch_gicv3.h @@ -103,6 +103,7 @@ static inline u64 gic_read_iar_common(void) u64 irqstat; asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat)); + dsb(sy); return irqstat; } diff --git a/kernel/arch/arm64/include/asm/cpufeature.h b/kernel/arch/arm64/include/asm/cpufeature.h index 8f271b83f..8884b5d5f 100644 --- a/kernel/arch/arm64/include/asm/cpufeature.h +++ b/kernel/arch/arm64/include/asm/cpufeature.h @@ -30,8 +30,9 @@ #define ARM64_HAS_LSE_ATOMICS 5 #define ARM64_WORKAROUND_CAVIUM_23154 6 #define ARM64_WORKAROUND_834220 7 +#define ARM64_WORKAROUND_CAVIUM_27456 8 -#define ARM64_NCAPS 8 +#define ARM64_NCAPS 9 #ifndef __ASSEMBLY__ @@ -76,7 +77,7 @@ struct arm64_cpu_capabilities { const char *desc; u16 capability; bool (*matches)(const struct arm64_cpu_capabilities *); - void (*enable)(void *); /* Called on all active CPUs */ + int (*enable)(void *); /* Called on all active CPUs */ union { struct { /* To be used for erratum handling only */ u32 midr_model; diff --git a/kernel/arch/arm64/include/asm/elf.h b/kernel/arch/arm64/include/asm/elf.h index faad6df49..44dd892a4 100644 --- a/kernel/arch/arm64/include/asm/elf.h +++ b/kernel/arch/arm64/include/asm/elf.h @@ -136,6 +136,7 @@ typedef struct user_fpsimd_state elf_fpregset_t; #define SET_PERSONALITY(ex) clear_thread_flag(TIF_32BIT); +/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ #define ARCH_DLINFO \ do { \ NEW_AUX_ENT(AT_SYSINFO_EHDR, \ @@ -156,14 +157,14 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, #define STACK_RND_MASK (0x3ffff >> (PAGE_SHIFT - 12)) #endif -#ifdef CONFIG_COMPAT - #ifdef __AARCH64EB__ #define COMPAT_ELF_PLATFORM ("v8b") #else #define COMPAT_ELF_PLATFORM ("v8l") #endif +#ifdef CONFIG_COMPAT + #define COMPAT_ELF_ET_DYN_BASE (2 * TASK_SIZE_32 / 3) /* AArch32 registers. */ diff --git a/kernel/arch/arm64/include/asm/futex.h b/kernel/arch/arm64/include/asm/futex.h index 007a69fc4..5f3ab8c1d 100644 --- a/kernel/arch/arm64/include/asm/futex.h +++ b/kernel/arch/arm64/include/asm/futex.h @@ -121,6 +121,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; asm volatile("// futex_atomic_cmpxchg_inatomic\n" +ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) " prfm pstl1strm, %2\n" "1: ldxr %w1, %2\n" " sub %w3, %w1, %w4\n" @@ -137,6 +138,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, " .align 3\n" " .quad 1b, 4b, 2b, 4b\n" " .popsection\n" +ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN) : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp) : "r" (oldval), "r" (newval), "Ir" (-EFAULT) : "memory"); diff --git a/kernel/arch/arm64/include/asm/kvm_arm.h b/kernel/arch/arm64/include/asm/kvm_arm.h index 5e6857b6b..2d960f858 100644 --- a/kernel/arch/arm64/include/asm/kvm_arm.h +++ b/kernel/arch/arm64/include/asm/kvm_arm.h @@ -107,8 +107,6 @@ #define TCR_EL2_MASK (TCR_EL2_TG0 | TCR_EL2_SH0 | \ TCR_EL2_ORGN0 | TCR_EL2_IRGN0 | TCR_EL2_T0SZ) -#define TCR_EL2_FLAGS (TCR_EL2_RES1 | TCR_EL2_PS_40B) - /* VTCR_EL2 Registers bits */ #define VTCR_EL2_RES1 (1 << 31) #define VTCR_EL2_PS_MASK (7 << 16) diff --git a/kernel/arch/arm64/include/asm/opcodes.h b/kernel/arch/arm64/include/asm/opcodes.h index 4e603ea36..123f45d92 100644 --- a/kernel/arch/arm64/include/asm/opcodes.h +++ b/kernel/arch/arm64/include/asm/opcodes.h @@ -1 +1,5 @@ +#ifdef CONFIG_CPU_BIG_ENDIAN +#define CONFIG_CPU_ENDIAN_BE8 CONFIG_CPU_BIG_ENDIAN +#endif + #include <../../arm/include/asm/opcodes.h> diff --git a/kernel/arch/arm64/include/asm/percpu.h b/kernel/arch/arm64/include/asm/percpu.h index 0a456bef8..8a336852e 100644 --- a/kernel/arch/arm64/include/asm/percpu.h +++ b/kernel/arch/arm64/include/asm/percpu.h @@ -44,48 +44,44 @@ static inline unsigned long __percpu_##op(void *ptr, \ \ switch (size) { \ case 1: \ - do { \ - asm ("//__per_cpu_" #op "_1\n" \ - "ldxrb %w[ret], %[ptr]\n" \ + asm ("//__per_cpu_" #op "_1\n" \ + "1: ldxrb %w[ret], %[ptr]\n" \ #asm_op " %w[ret], %w[ret], %w[val]\n" \ - "stxrb %w[loop], %w[ret], %[ptr]\n" \ - : [loop] "=&r" (loop), [ret] "=&r" (ret), \ - [ptr] "+Q"(*(u8 *)ptr) \ - : [val] "Ir" (val)); \ - } while (loop); \ + " stxrb %w[loop], %w[ret], %[ptr]\n" \ + " cbnz %w[loop], 1b" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u8 *)ptr) \ + : [val] "Ir" (val)); \ break; \ case 2: \ - do { \ - asm ("//__per_cpu_" #op "_2\n" \ - "ldxrh %w[ret], %[ptr]\n" \ + asm ("//__per_cpu_" #op "_2\n" \ + "1: ldxrh %w[ret], %[ptr]\n" \ #asm_op " %w[ret], %w[ret], %w[val]\n" \ - "stxrh %w[loop], %w[ret], %[ptr]\n" \ - : [loop] "=&r" (loop), [ret] "=&r" (ret), \ - [ptr] "+Q"(*(u16 *)ptr) \ - : [val] "Ir" (val)); \ - } while (loop); \ + " stxrh %w[loop], %w[ret], %[ptr]\n" \ + " cbnz %w[loop], 1b" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u16 *)ptr) \ + : [val] "Ir" (val)); \ break; \ case 4: \ - do { \ - asm ("//__per_cpu_" #op "_4\n" \ - "ldxr %w[ret], %[ptr]\n" \ + asm ("//__per_cpu_" #op "_4\n" \ + "1: ldxr %w[ret], %[ptr]\n" \ #asm_op " %w[ret], %w[ret], %w[val]\n" \ - "stxr %w[loop], %w[ret], %[ptr]\n" \ - : [loop] "=&r" (loop), [ret] "=&r" (ret), \ - [ptr] "+Q"(*(u32 *)ptr) \ - : [val] "Ir" (val)); \ - } while (loop); \ + " stxr %w[loop], %w[ret], %[ptr]\n" \ + " cbnz %w[loop], 1b" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u32 *)ptr) \ + : [val] "Ir" (val)); \ break; \ case 8: \ - do { \ - asm ("//__per_cpu_" #op "_8\n" \ - "ldxr %[ret], %[ptr]\n" \ + asm ("//__per_cpu_" #op "_8\n" \ + "1: ldxr %[ret], %[ptr]\n" \ #asm_op " %[ret], %[ret], %[val]\n" \ - "stxr %w[loop], %[ret], %[ptr]\n" \ - : [loop] "=&r" (loop), [ret] "=&r" (ret), \ - [ptr] "+Q"(*(u64 *)ptr) \ - : [val] "Ir" (val)); \ - } while (loop); \ + " stxr %w[loop], %[ret], %[ptr]\n" \ + " cbnz %w[loop], 1b" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u64 *)ptr) \ + : [val] "Ir" (val)); \ break; \ default: \ BUILD_BUG(); \ @@ -150,44 +146,40 @@ static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, switch (size) { case 1: - do { - asm ("//__percpu_xchg_1\n" - "ldxrb %w[ret], %[ptr]\n" - "stxrb %w[loop], %w[val], %[ptr]\n" - : [loop] "=&r"(loop), [ret] "=&r"(ret), - [ptr] "+Q"(*(u8 *)ptr) - : [val] "r" (val)); - } while (loop); + asm ("//__percpu_xchg_1\n" + "1: ldxrb %w[ret], %[ptr]\n" + " stxrb %w[loop], %w[val], %[ptr]\n" + " cbnz %w[loop], 1b" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u8 *)ptr) + : [val] "r" (val)); break; case 2: - do { - asm ("//__percpu_xchg_2\n" - "ldxrh %w[ret], %[ptr]\n" - "stxrh %w[loop], %w[val], %[ptr]\n" - : [loop] "=&r"(loop), [ret] "=&r"(ret), - [ptr] "+Q"(*(u16 *)ptr) - : [val] "r" (val)); - } while (loop); + asm ("//__percpu_xchg_2\n" + "1: ldxrh %w[ret], %[ptr]\n" + " stxrh %w[loop], %w[val], %[ptr]\n" + " cbnz %w[loop], 1b" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u16 *)ptr) + : [val] "r" (val)); break; case 4: - do { - asm ("//__percpu_xchg_4\n" - "ldxr %w[ret], %[ptr]\n" - "stxr %w[loop], %w[val], %[ptr]\n" - : [loop] "=&r"(loop), [ret] "=&r"(ret), - [ptr] "+Q"(*(u32 *)ptr) - : [val] "r" (val)); - } while (loop); + asm ("//__percpu_xchg_4\n" + "1: ldxr %w[ret], %[ptr]\n" + " stxr %w[loop], %w[val], %[ptr]\n" + " cbnz %w[loop], 1b" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u32 *)ptr) + : [val] "r" (val)); break; case 8: - do { - asm ("//__percpu_xchg_8\n" - "ldxr %[ret], %[ptr]\n" - "stxr %w[loop], %[val], %[ptr]\n" - : [loop] "=&r"(loop), [ret] "=&r"(ret), - [ptr] "+Q"(*(u64 *)ptr) - : [val] "r" (val)); - } while (loop); + asm ("//__percpu_xchg_8\n" + "1: ldxr %[ret], %[ptr]\n" + " stxr %w[loop], %[val], %[ptr]\n" + " cbnz %w[loop], 1b" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u64 *)ptr) + : [val] "r" (val)); break; default: BUILD_BUG(); diff --git a/kernel/arch/arm64/include/asm/pgtable-hwdef.h b/kernel/arch/arm64/include/asm/pgtable-hwdef.h index d6739e836..b9da9545b 100644 --- a/kernel/arch/arm64/include/asm/pgtable-hwdef.h +++ b/kernel/arch/arm64/include/asm/pgtable-hwdef.h @@ -117,7 +117,6 @@ * Section */ #define PMD_SECT_VALID (_AT(pmdval_t, 1) << 0) -#define PMD_SECT_PROT_NONE (_AT(pmdval_t, 1) << 58) #define PMD_SECT_USER (_AT(pmdval_t, 1) << 6) /* AP[1] */ #define PMD_SECT_RDONLY (_AT(pmdval_t, 1) << 7) /* AP[2] */ #define PMD_SECT_S (_AT(pmdval_t, 3) << 8) diff --git a/kernel/arch/arm64/include/asm/pgtable.h b/kernel/arch/arm64/include/asm/pgtable.h index eaa9cabf4..67c2ad6d3 100644 --- a/kernel/arch/arm64/include/asm/pgtable.h +++ b/kernel/arch/arm64/include/asm/pgtable.h @@ -69,11 +69,11 @@ extern void __pgd_error(const char *file, int line, unsigned long val); #define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) #define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) -#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) -#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE)) -#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_NC)) -#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_WT)) -#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) +#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) +#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE)) +#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC)) +#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT)) +#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL)) #define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE)) #define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL)) @@ -83,7 +83,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val); #define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY) -#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY) +#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY) #define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE) #define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT) @@ -155,6 +155,7 @@ extern struct page *empty_zero_page; #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) #define pte_exec(pte) (!(pte_val(pte) & PTE_UXN)) #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) +#define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) #ifdef CONFIG_ARM64_HW_AFDBM #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) @@ -165,8 +166,6 @@ extern struct page *empty_zero_page; #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) -#define pte_valid_user(pte) \ - ((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) #define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID) @@ -264,13 +263,13 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { - if (pte_valid_user(pte)) { - if (!pte_special(pte) && pte_exec(pte)) - __sync_icache_dcache(pte, addr); + if (pte_present(pte)) { if (pte_sw_dirty(pte) && pte_write(pte)) pte_val(pte) &= ~PTE_RDONLY; else pte_val(pte) |= PTE_RDONLY; + if (pte_user(pte) && pte_exec(pte) && !pte_special(pte)) + __sync_icache_dcache(pte, addr); } /* @@ -348,6 +347,7 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#define pmd_present(pmd) pte_present(pmd_pte(pmd)) #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) #define pmd_young(pmd) pte_young(pmd_pte(pmd)) #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) @@ -356,7 +356,7 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) -#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_TYPE_MASK)) +#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID)) #define __HAVE_ARCH_PMD_WRITE #define pmd_write(pmd) pte_write(pmd_pte(pmd)) @@ -395,7 +395,6 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot); #define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_present(pmd) (pmd_val(pmd)) #define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) @@ -539,6 +538,21 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) } #ifdef CONFIG_ARM64_HW_AFDBM +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +extern int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +static inline int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty); +} +#endif + /* * Atomic pte/pmd modifications. */ @@ -591,9 +605,9 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, } #ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define __HAVE_ARCH_PMDP_GET_AND_CLEAR -static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, - unsigned long address, pmd_t *pmdp) +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) { return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp)); } @@ -641,6 +655,7 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; * bits 0-1: present (must be zero) * bits 2-7: swap type * bits 8-57: swap offset + * bit 58: PTE_PROT_NONE (must be zero) */ #define __SWP_TYPE_SHIFT 2 #define __SWP_TYPE_BITS 6 diff --git a/kernel/arch/arm64/include/asm/processor.h b/kernel/arch/arm64/include/asm/processor.h index 4acb7ca94..d08559528 100644 --- a/kernel/arch/arm64/include/asm/processor.h +++ b/kernel/arch/arm64/include/asm/processor.h @@ -186,6 +186,6 @@ static inline void spin_lock_prefetch(const void *x) #endif -void cpu_enable_pan(void *__unused); +int cpu_enable_pan(void *__unused); #endif /* __ASM_PROCESSOR_H */ diff --git a/kernel/arch/arm64/include/asm/ptrace.h b/kernel/arch/arm64/include/asm/ptrace.h index e9e5467e0..7f9475508 100644 --- a/kernel/arch/arm64/include/asm/ptrace.h +++ b/kernel/arch/arm64/include/asm/ptrace.h @@ -58,6 +58,7 @@ #define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ +#define COMPAT_PSR_GE_MASK 0x000f0000 #ifdef CONFIG_CPU_BIG_ENDIAN #define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT @@ -116,6 +117,8 @@ struct pt_regs { }; u64 orig_x0; u64 syscallno; + u64 orig_addr_limit; + u64 unused; // maintain 16 byte alignment }; #define arch_has_single_step() (1) @@ -151,35 +154,9 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->regs[0]; } -/* - * Are the current registers suitable for user mode? (used to maintain - * security in signal handlers) - */ -static inline int valid_user_regs(struct user_pt_regs *regs) -{ - if (user_mode(regs) && (regs->pstate & PSR_I_BIT) == 0) { - regs->pstate &= ~(PSR_F_BIT | PSR_A_BIT); - - /* The T bit is reserved for AArch64 */ - if (!(regs->pstate & PSR_MODE32_BIT)) - regs->pstate &= ~COMPAT_PSR_T_BIT; - - return 1; - } - - /* - * Force PSR to something logical... - */ - regs->pstate &= PSR_f | PSR_s | (PSR_x & ~PSR_A_BIT) | \ - COMPAT_PSR_T_BIT | PSR_MODE32_BIT; - - if (!(regs->pstate & PSR_MODE32_BIT)) { - regs->pstate &= ~COMPAT_PSR_T_BIT; - regs->pstate |= PSR_MODE_EL0t; - } - - return 0; -} +/* We must avoid circular header include via sched.h */ +struct task_struct; +int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task); #define instruction_pointer(regs) ((unsigned long)(regs)->pc) diff --git a/kernel/arch/arm64/include/asm/spinlock.h b/kernel/arch/arm64/include/asm/spinlock.h index c85e96d17..499e8de33 100644 --- a/kernel/arch/arm64/include/asm/spinlock.h +++ b/kernel/arch/arm64/include/asm/spinlock.h @@ -312,4 +312,14 @@ static inline int arch_read_trylock(arch_rwlock_t *rw) #define arch_read_relax(lock) cpu_relax() #define arch_write_relax(lock) cpu_relax() +/* + * Accesses appearing in program order before a spin_lock() operation + * can be reordered with accesses inside the critical section, by virtue + * of arch_spin_lock being constructed using acquire semantics. + * + * In cases where this is problematic (e.g. try_to_wake_up), an + * smp_mb__before_spinlock() can restore the required ordering. + */ +#define smp_mb__before_spinlock() smp_mb() + #endif /* __ASM_SPINLOCK_H */ diff --git a/kernel/arch/arm64/include/uapi/asm/auxvec.h b/kernel/arch/arm64/include/uapi/asm/auxvec.h index 22d6d8885..4cf0c1778 100644 --- a/kernel/arch/arm64/include/uapi/asm/auxvec.h +++ b/kernel/arch/arm64/include/uapi/asm/auxvec.h @@ -19,4 +19,6 @@ /* vDSO location */ #define AT_SYSINFO_EHDR 33 +#define AT_VECTOR_SIZE_ARCH 1 /* entries in ARCH_DLINFO */ + #endif diff --git a/kernel/arch/arm64/include/uapi/asm/ptrace.h b/kernel/arch/arm64/include/uapi/asm/ptrace.h index 208db3df1..3378238b5 100644 --- a/kernel/arch/arm64/include/uapi/asm/ptrace.h +++ b/kernel/arch/arm64/include/uapi/asm/ptrace.h @@ -76,6 +76,7 @@ struct user_fpsimd_state { __uint128_t vregs[32]; __u32 fpsr; __u32 fpcr; + __u32 __reserved[2]; }; struct user_hwdebug_state { diff --git a/kernel/arch/arm64/kernel/asm-offsets.c b/kernel/arch/arm64/kernel/asm-offsets.c index c5038409e..d74475928 100644 --- a/kernel/arch/arm64/kernel/asm-offsets.c +++ b/kernel/arch/arm64/kernel/asm-offsets.c @@ -59,6 +59,7 @@ int main(void) DEFINE(S_PC, offsetof(struct pt_regs, pc)); DEFINE(S_ORIG_X0, offsetof(struct pt_regs, orig_x0)); DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); + DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit)); DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); BLANK(); DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter)); diff --git a/kernel/arch/arm64/kernel/cpu_errata.c b/kernel/arch/arm64/kernel/cpu_errata.c index feb6b4efa..a3e846a28 100644 --- a/kernel/arch/arm64/kernel/cpu_errata.c +++ b/kernel/arch/arm64/kernel/cpu_errata.c @@ -99,6 +99,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .capability = ARM64_WORKAROUND_CAVIUM_23154, MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01), }, +#endif +#ifdef CONFIG_CAVIUM_ERRATUM_27456 + { + /* Cavium ThunderX, T88 pass 1.x - 2.1 */ + .desc = "Cavium erratum 27456", + .capability = ARM64_WORKAROUND_CAVIUM_27456, + MIDR_RANGE(MIDR_THUNDERX, 0x00, + (1 << MIDR_VARIANT_SHIFT) | 1), + }, #endif { } diff --git a/kernel/arch/arm64/kernel/cpufeature.c b/kernel/arch/arm64/kernel/cpufeature.c index 0669c6328..2735bf814 100644 --- a/kernel/arch/arm64/kernel/cpufeature.c +++ b/kernel/arch/arm64/kernel/cpufeature.c @@ -19,7 +19,9 @@ #define pr_fmt(fmt) "CPU features: " fmt #include +#include #include +#include #include #include #include @@ -764,7 +766,13 @@ static void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) for (i = 0; caps[i].desc; i++) if (caps[i].enable && cpus_have_cap(caps[i].capability)) - on_each_cpu(caps[i].enable, NULL, true); + /* + * Use stop_machine() as it schedules the work allowing + * us to modify PSTATE, instead of on_each_cpu() which + * uses an IPI, giving us a PSTATE that disappears when + * we return. + */ + stop_machine(caps[i].enable, NULL, cpu_online_mask); } #ifdef CONFIG_HOTPLUG_CPU diff --git a/kernel/arch/arm64/kernel/cpuinfo.c b/kernel/arch/arm64/kernel/cpuinfo.c index 212ae6361..0166cfbc8 100644 --- a/kernel/arch/arm64/kernel/cpuinfo.c +++ b/kernel/arch/arm64/kernel/cpuinfo.c @@ -22,6 +22,8 @@ #include #include +#include +#include #include #include #include @@ -85,7 +87,8 @@ static const char *const compat_hwcap_str[] = { "idivt", "vfpd32", "lpae", - "evtstrm" + "evtstrm", + NULL }; static const char *const compat_hwcap2_str[] = { @@ -101,6 +104,7 @@ static const char *const compat_hwcap2_str[] = { static int c_show(struct seq_file *m, void *v) { int i, j; + bool compat = personality(current->personality) == PER_LINUX32; for_each_online_cpu(i) { struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i); @@ -112,6 +116,9 @@ static int c_show(struct seq_file *m, void *v) * "processor". Give glibc what it expects. */ seq_printf(m, "processor\t: %d\n", i); + if (compat) + seq_printf(m, "model name\t: ARMv8 Processor rev %d (%s)\n", + MIDR_REVISION(midr), COMPAT_ELF_PLATFORM); seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", loops_per_jiffy / (500000UL/HZ), @@ -124,7 +131,7 @@ static int c_show(struct seq_file *m, void *v) * software which does already (at least for 32-bit). */ seq_puts(m, "Features\t:"); - if (personality(current->personality) == PER_LINUX32) { + if (compat) { #ifdef CONFIG_COMPAT for (j = 0; compat_hwcap_str[j]; j++) if (compat_elf_hwcap & (1 << j)) diff --git a/kernel/arch/arm64/kernel/debug-monitors.c b/kernel/arch/arm64/kernel/debug-monitors.c index c1492ba1f..c8875b64b 100644 --- a/kernel/arch/arm64/kernel/debug-monitors.c +++ b/kernel/arch/arm64/kernel/debug-monitors.c @@ -152,7 +152,6 @@ static int debug_monitors_init(void) /* Clear the OS lock. */ on_each_cpu(clear_os_lock, NULL, 1); isb(); - local_dbg_enable(); /* Register hotplug handler. */ __register_cpu_notifier(&os_lock_nb); @@ -423,8 +422,10 @@ int kernel_active_single_step(void) /* ptrace API */ void user_enable_single_step(struct task_struct *task) { - set_ti_thread_flag(task_thread_info(task), TIF_SINGLESTEP); - set_regs_spsr_ss(task_pt_regs(task)); + struct thread_info *ti = task_thread_info(task); + + if (!test_and_set_ti_thread_flag(ti, TIF_SINGLESTEP)) + set_regs_spsr_ss(task_pt_regs(task)); } void user_disable_single_step(struct task_struct *task) diff --git a/kernel/arch/arm64/kernel/entry.S b/kernel/arch/arm64/kernel/entry.S index dd8fd31f8..cf92d4ee5 100644 --- a/kernel/arch/arm64/kernel/entry.S +++ b/kernel/arch/arm64/kernel/entry.S @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -93,7 +94,13 @@ disable_step_tsk x19, x20 // exceptions when scheduling. .else add x21, sp, #S_FRAME_SIZE - .endif + get_thread_info tsk + /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */ + ldr x20, [tsk, #TI_ADDR_LIMIT] + str x20, [sp, #S_ORIG_ADDR_LIMIT] + mov x20, #TASK_SIZE_64 + str x20, [tsk, #TI_ADDR_LIMIT] + .endif /* \el == 0 */ mrs x22, elr_el1 mrs x23, spsr_el1 stp lr, x21, [sp, #S_LR] @@ -117,6 +124,12 @@ .endm .macro kernel_exit, el + .if \el != 0 + /* Restore the task's original addr_limit. */ + ldr x20, [sp, #S_ORIG_ADDR_LIMIT] + str x20, [tsk, #TI_ADDR_LIMIT] + .endif + ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ct_user_enter @@ -555,7 +568,7 @@ el0_inv: mov x0, sp mov x1, #BAD_SYNC mov x2, x25 - bl bad_mode + bl bad_el0_sync b ret_to_user ENDPROC(el0_sync) diff --git a/kernel/arch/arm64/kernel/head.S b/kernel/arch/arm64/kernel/head.S index b68525792..20ceb5edf 100644 --- a/kernel/arch/arm64/kernel/head.S +++ b/kernel/arch/arm64/kernel/head.S @@ -518,8 +518,9 @@ CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems b.lt 4f // Skip if no PMU present mrs x0, pmcr_el0 // Disable debug access traps ubfx x0, x0, #11, #5 // to EL2 and allow access to - msr mdcr_el2, x0 // all PMU counters from EL1 4: + csel x0, xzr, x0, lt // all PMU counters from EL1 + msr mdcr_el2, x0 // (if they exist) /* Stage-2 translation */ msr vttbr_el2, xzr diff --git a/kernel/arch/arm64/kernel/ptrace.c b/kernel/arch/arm64/kernel/ptrace.c index ff7f13239..55909b220 100644 --- a/kernel/arch/arm64/kernel/ptrace.c +++ b/kernel/arch/arm64/kernel/ptrace.c @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -449,6 +450,8 @@ static int hw_break_set(struct task_struct *target, /* (address, ctrl) registers */ limit = regset->n * regset->size; while (count && offset < limit) { + if (count < PTRACE_HBP_ADDR_SZ) + return -EINVAL; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr, offset, offset + PTRACE_HBP_ADDR_SZ); if (ret) @@ -458,6 +461,8 @@ static int hw_break_set(struct task_struct *target, return ret; offset += PTRACE_HBP_ADDR_SZ; + if (!count) + break; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, offset, offset + PTRACE_HBP_CTRL_SZ); if (ret) @@ -494,13 +499,13 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - struct user_pt_regs newregs; + struct user_pt_regs newregs = task_pt_regs(target)->user_regs; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1); if (ret) return ret; - if (!valid_user_regs(&newregs)) + if (!valid_user_regs(&newregs, target)) return -EINVAL; task_pt_regs(target)->user_regs = newregs; @@ -524,7 +529,8 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - struct user_fpsimd_state newstate; + struct user_fpsimd_state newstate = + target->thread.fpsimd_state.user_fpsimd; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1); if (ret) @@ -548,7 +554,7 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - unsigned long tls; + unsigned long tls = target->thread.tp_value; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); if (ret) @@ -574,7 +580,8 @@ static int system_call_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - int syscallno, ret; + int syscallno = task_pt_regs(target)->syscallno; + int ret; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1); if (ret) @@ -770,7 +777,7 @@ static int compat_gpr_set(struct task_struct *target, } - if (valid_user_regs(&newregs.user_regs)) + if (valid_user_regs(&newregs.user_regs, target)) *task_pt_regs(target) = newregs; else ret = -EINVAL; @@ -846,7 +853,7 @@ static int compat_tls_set(struct task_struct *target, const void __user *ubuf) { int ret; - compat_ulong_t tls; + compat_ulong_t tls = target->thread.tp_value; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); if (ret) @@ -1272,3 +1279,79 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) if (test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT); } + +/* + * Bits which are always architecturally RES0 per ARM DDI 0487A.h + * Userspace cannot use these until they have an architectural meaning. + * We also reserve IL for the kernel; SS is handled dynamically. + */ +#define SPSR_EL1_AARCH64_RES0_BITS \ + (GENMASK_ULL(63,32) | GENMASK_ULL(27, 22) | GENMASK_ULL(20, 10) | \ + GENMASK_ULL(5, 5)) +#define SPSR_EL1_AARCH32_RES0_BITS \ + (GENMASK_ULL(63,32) | GENMASK_ULL(24, 22) | GENMASK_ULL(20,20)) + +static int valid_compat_regs(struct user_pt_regs *regs) +{ + regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS; + + if (!system_supports_mixed_endian_el0()) { + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + regs->pstate |= COMPAT_PSR_E_BIT; + else + regs->pstate &= ~COMPAT_PSR_E_BIT; + } + + if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) && + (regs->pstate & COMPAT_PSR_A_BIT) == 0 && + (regs->pstate & COMPAT_PSR_I_BIT) == 0 && + (regs->pstate & COMPAT_PSR_F_BIT) == 0) { + return 1; + } + + /* + * Force PSR to a valid 32-bit EL0t, preserving the same bits as + * arch/arm. + */ + regs->pstate &= COMPAT_PSR_N_BIT | COMPAT_PSR_Z_BIT | + COMPAT_PSR_C_BIT | COMPAT_PSR_V_BIT | + COMPAT_PSR_Q_BIT | COMPAT_PSR_IT_MASK | + COMPAT_PSR_GE_MASK | COMPAT_PSR_E_BIT | + COMPAT_PSR_T_BIT; + regs->pstate |= PSR_MODE32_BIT; + + return 0; +} + +static int valid_native_regs(struct user_pt_regs *regs) +{ + regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS; + + if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) && + (regs->pstate & PSR_D_BIT) == 0 && + (regs->pstate & PSR_A_BIT) == 0 && + (regs->pstate & PSR_I_BIT) == 0 && + (regs->pstate & PSR_F_BIT) == 0) { + return 1; + } + + /* Force PSR to a valid 64-bit EL0t */ + regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; + + return 0; +} + +/* + * Are the current registers suitable for user mode? (used to maintain + * security in signal handlers) + */ +int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task) +{ + if (!test_tsk_thread_flag(task, TIF_SINGLESTEP)) + regs->pstate &= ~DBG_SPSR_SS; + + if (is_compat_thread(task_thread_info(task))) + return valid_compat_regs(regs); + else + return valid_native_regs(regs); +} diff --git a/kernel/arch/arm64/kernel/signal.c b/kernel/arch/arm64/kernel/signal.c index e18c48cb6..a8eafdbc7 100644 --- a/kernel/arch/arm64/kernel/signal.c +++ b/kernel/arch/arm64/kernel/signal.c @@ -115,7 +115,7 @@ static int restore_sigframe(struct pt_regs *regs, */ regs->syscallno = ~0UL; - err |= !valid_user_regs(®s->user_regs); + err |= !valid_user_regs(®s->user_regs, current); if (err == 0) { struct fpsimd_context *fpsimd_ctx = @@ -307,7 +307,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) /* * Check that the resulting registers are actually sane. */ - ret |= !valid_user_regs(®s->user_regs); + ret |= !valid_user_regs(®s->user_regs, current); /* * Fast forward the stepping logic so we step into the signal diff --git a/kernel/arch/arm64/kernel/signal32.c b/kernel/arch/arm64/kernel/signal32.c index 71ef6dc89..107335637 100644 --- a/kernel/arch/arm64/kernel/signal32.c +++ b/kernel/arch/arm64/kernel/signal32.c @@ -356,7 +356,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, */ regs->syscallno = ~0UL; - err |= !valid_user_regs(®s->user_regs); + err |= !valid_user_regs(®s->user_regs, current); aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace; if (err == 0) diff --git a/kernel/arch/arm64/kernel/smp.c b/kernel/arch/arm64/kernel/smp.c index b1adc51b2..f3c3d8fee 100644 --- a/kernel/arch/arm64/kernel/smp.c +++ b/kernel/arch/arm64/kernel/smp.c @@ -188,7 +188,6 @@ asmlinkage void secondary_start_kernel(void) set_cpu_online(cpu, true); complete(&cpu_running); - local_dbg_enable(); local_irq_enable(); local_async_enable(); @@ -334,8 +333,8 @@ void __init smp_cpus_done(unsigned int max_cpus) void __init smp_prepare_boot_cpu(void) { - cpuinfo_store_boot_cpu(); set_my_cpu_offset(per_cpu_offset(smp_processor_id())); + cpuinfo_store_boot_cpu(); } static u64 __init of_get_cpu_mpidr(struct device_node *dn) diff --git a/kernel/arch/arm64/kernel/suspend.c b/kernel/arch/arm64/kernel/suspend.c index 1095aa483..00c1372bf 100644 --- a/kernel/arch/arm64/kernel/suspend.c +++ b/kernel/arch/arm64/kernel/suspend.c @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -110,6 +112,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ set_my_cpu_offset(per_cpu_offset(smp_processor_id())); + /* + * PSTATE was not saved over suspend/resume, re-enable any + * detected features that might not have been set correctly. + */ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, + CONFIG_ARM64_PAN)); + /* * Restore HW breakpoint registers to sane values * before debug exceptions are possibly reenabled diff --git a/kernel/arch/arm64/kernel/traps.c b/kernel/arch/arm64/kernel/traps.c index e9b9b5364..ca7f0ac5f 100644 --- a/kernel/arch/arm64/kernel/traps.c +++ b/kernel/arch/arm64/kernel/traps.c @@ -434,16 +434,33 @@ const char *esr_get_class_string(u32 esr) } /* - * bad_mode handles the impossible case in the exception vector. + * bad_mode handles the impossible case in the exception vector. This is always + * fatal. */ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) { - siginfo_t info; - void __user *pc = (void __user *)instruction_pointer(regs); console_verbose(); pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n", handler[reason], esr, esr_get_class_string(esr)); + + die("Oops - bad mode", regs, 0); + local_irq_disable(); + panic("bad mode"); +} + +/* + * bad_el0_sync handles unexpected, but potentially recoverable synchronous + * exceptions taken from EL0. Unlike bad_mode, this returns. + */ +asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) +{ + siginfo_t info; + void __user *pc = (void __user *)instruction_pointer(regs); + console_verbose(); + + pr_crit("Bad EL0 synchronous exception detected on CPU%d, code 0x%08x -- %s\n", + smp_processor_id(), esr, esr_get_class_string(esr)); __show_regs(regs); info.si_signo = SIGILL; @@ -451,7 +468,10 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) info.si_code = ILL_ILLOPC; info.si_addr = pc; - arm64_notify_die("Oops - bad mode", regs, &info, 0); + current->thread.fault_address = 0; + current->thread.fault_code = 0; + + force_sig_info(info.si_signo, &info, current); } void __pte_error(const char *file, int line, unsigned long val) diff --git a/kernel/arch/arm64/kvm/hyp-init.S b/kernel/arch/arm64/kvm/hyp-init.S index 178ba2248..84c338f01 100644 --- a/kernel/arch/arm64/kvm/hyp-init.S +++ b/kernel/arch/arm64/kvm/hyp-init.S @@ -64,7 +64,7 @@ __do_hyp_init: mrs x4, tcr_el1 ldr x5, =TCR_EL2_MASK and x4, x4, x5 - ldr x5, =TCR_EL2_FLAGS + mov x5, #TCR_EL2_RES1 orr x4, x4, x5 #ifndef CONFIG_ARM64_VA_BITS_48 @@ -85,15 +85,18 @@ __do_hyp_init: ldr_l x5, idmap_t0sz bfi x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH #endif - msr tcr_el2, x4 - - ldr x4, =VTCR_EL2_FLAGS /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in - * VTCR_EL2. + * TCR_EL2 and VTCR_EL2. */ mrs x5, ID_AA64MMFR0_EL1 bfi x4, x5, #16, #3 + + msr tcr_el2, x4 + + ldr x4, =VTCR_EL2_FLAGS + bfi x4, x5, #16, #3 + msr vtcr_el2, x4 mrs x4, mair_el1 diff --git a/kernel/arch/arm64/kvm/inject_fault.c b/kernel/arch/arm64/kvm/inject_fault.c index 648112e90..3972e65fb 100644 --- a/kernel/arch/arm64/kvm/inject_fault.c +++ b/kernel/arch/arm64/kvm/inject_fault.c @@ -130,7 +130,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); if (!is_iabt) - esr |= ESR_ELx_EC_DABT_LOW; + esr |= ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT; vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT; } diff --git a/kernel/arch/arm64/mm/fault.c b/kernel/arch/arm64/mm/fault.c index 92ddac1e8..247bae758 100644 --- a/kernel/arch/arm64/mm/fault.c +++ b/kernel/arch/arm64/mm/fault.c @@ -29,7 +29,9 @@ #include #include #include +#include +#include #include #include #include @@ -81,6 +83,56 @@ void show_pte(struct mm_struct *mm, unsigned long addr) printk("\n"); } +#ifdef CONFIG_ARM64_HW_AFDBM +/* + * This function sets the access flags (dirty, accessed), as well as write + * permission, and only to a more permissive setting. + * + * It needs to cope with hardware update of the accessed/dirty state by other + * agents in the system and can safely skip the __sync_icache_dcache() call as, + * like set_pte_at(), the PTE is never changed from no-exec to exec here. + * + * Returns whether or not the PTE actually changed. + */ +int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty) +{ + pteval_t old_pteval; + unsigned int tmp; + + if (pte_same(*ptep, entry)) + return 0; + + /* only preserve the access flags and write permission */ + pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY; + + /* + * PTE_RDONLY is cleared by default in the asm below, so set it in + * back if necessary (read-only or clean PTE). + */ + if (!pte_write(entry) || !pte_sw_dirty(entry)) + pte_val(entry) |= PTE_RDONLY; + + /* + * Setting the flags must be done atomically to avoid racing with the + * hardware update of the access/dirty state. + */ + asm volatile("// ptep_set_access_flags\n" + " prfm pstl1strm, %2\n" + "1: ldxr %0, %2\n" + " and %0, %0, %3 // clear PTE_RDONLY\n" + " orr %0, %0, %4 // set flags\n" + " stxr %w1, %0, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) + : "L" (~PTE_RDONLY), "r" (pte_val(entry))); + + flush_tlb_fix_spurious_fault(vma, address); + return 1; +} +#endif + /* * The kernel tried to access some page that wasn't present. */ @@ -556,8 +608,16 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, } #ifdef CONFIG_ARM64_PAN -void cpu_enable_pan(void *__unused) +int cpu_enable_pan(void *__unused) { + /* + * We modify PSTATE. This won't work from irq context as the PSTATE + * is discarded once we return from the exception. + */ + WARN_ON_ONCE(in_interrupt()); + config_sctlr_el1(SCTLR_EL1_SPAN, 0); + asm(SET_PSTATE_PAN(1)); + return 0; } #endif /* CONFIG_ARM64_PAN */ diff --git a/kernel/arch/arm64/mm/mmu.c b/kernel/arch/arm64/mm/mmu.c index 116ad654d..653735a8c 100644 --- a/kernel/arch/arm64/mm/mmu.c +++ b/kernel/arch/arm64/mm/mmu.c @@ -652,9 +652,9 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys) /* * Check whether the physical FDT address is set and meets the minimum * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be - * at least 8 bytes so that we can always access the size field of the - * FDT header after mapping the first chunk, double check here if that - * is indeed the case. + * at least 8 bytes so that we can always access the magic and size + * fields of the FDT header after mapping the first chunk, double check + * here if that is indeed the case. */ BUILD_BUG_ON(MIN_FDT_ALIGN < 8); if (!dt_phys || dt_phys % MIN_FDT_ALIGN) @@ -682,7 +682,7 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys) create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, SWAPPER_BLOCK_SIZE, prot); - if (fdt_check_header(dt_virt) != 0) + if (fdt_magic(dt_virt) != FDT_MAGIC) return NULL; size = fdt_totalsize(dt_virt); diff --git a/kernel/arch/arm64/mm/proc.S b/kernel/arch/arm64/mm/proc.S index b8f04b3f2..18201e9e8 100644 --- a/kernel/arch/arm64/mm/proc.S +++ b/kernel/arch/arm64/mm/proc.S @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "proc-macros.S" @@ -137,7 +139,17 @@ ENTRY(cpu_do_switch_mm) bfi x0, x1, #48, #16 // set the ASID msr ttbr0_el1, x0 // set TTBR0 isb +alternative_if_not ARM64_WORKAROUND_CAVIUM_27456 ret + nop + nop + nop +alternative_else + ic iallu + dsb nsh + isb + ret +alternative_endif ENDPROC(cpu_do_switch_mm) .section ".text.init", #alloc, #execinstr @@ -156,6 +168,8 @@ ENTRY(__cpu_setup) msr cpacr_el1, x0 // Enable FP/ASIMD mov x0, #1 << 12 // Reset mdscr_el1 and disable msr mdscr_el1, x0 // access to the DCC from EL0 + isb // Unmask debug exceptions now, + enable_dbg // since this is per-cpu reset_pmuserenr_el0 x0 // Disable PMU access from EL0 /* * Memory region attributes for LPAE: diff --git a/kernel/arch/avr32/include/asm/uaccess.h b/kernel/arch/avr32/include/asm/uaccess.h index 68cf638fa..b1ec1fa06 100644 --- a/kernel/arch/avr32/include/asm/uaccess.h +++ b/kernel/arch/avr32/include/asm/uaccess.h @@ -74,7 +74,7 @@ extern __kernel_size_t __copy_user(void *to, const void *from, extern __kernel_size_t copy_to_user(void __user *to, const void *from, __kernel_size_t n); -extern __kernel_size_t copy_from_user(void *to, const void __user *from, +extern __kernel_size_t ___copy_from_user(void *to, const void __user *from, __kernel_size_t n); static inline __kernel_size_t __copy_to_user(void __user *to, const void *from, @@ -88,6 +88,15 @@ static inline __kernel_size_t __copy_from_user(void *to, { return __copy_user(to, (const void __force *)from, n); } +static inline __kernel_size_t copy_from_user(void *to, + const void __user *from, + __kernel_size_t n) +{ + size_t res = ___copy_from_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; +} #define __copy_to_user_inatomic __copy_to_user #define __copy_from_user_inatomic __copy_from_user diff --git a/kernel/arch/avr32/kernel/avr32_ksyms.c b/kernel/arch/avr32/kernel/avr32_ksyms.c index d93ead02d..7c6cf14f0 100644 --- a/kernel/arch/avr32/kernel/avr32_ksyms.c +++ b/kernel/arch/avr32/kernel/avr32_ksyms.c @@ -36,7 +36,7 @@ EXPORT_SYMBOL(copy_page); /* * Userspace access stuff. */ -EXPORT_SYMBOL(copy_from_user); +EXPORT_SYMBOL(___copy_from_user); EXPORT_SYMBOL(copy_to_user); EXPORT_SYMBOL(__copy_user); EXPORT_SYMBOL(strncpy_from_user); diff --git a/kernel/arch/avr32/lib/copy_user.S b/kernel/arch/avr32/lib/copy_user.S index ea59c04b0..075373471 100644 --- a/kernel/arch/avr32/lib/copy_user.S +++ b/kernel/arch/avr32/lib/copy_user.S @@ -23,13 +23,13 @@ */ .text .align 1 - .global copy_from_user - .type copy_from_user, @function -copy_from_user: + .global ___copy_from_user + .type ___copy_from_user, @function +___copy_from_user: branch_if_kernel r8, __copy_user ret_if_privileged r8, r11, r10, r10 rjmp __copy_user - .size copy_from_user, . - copy_from_user + .size ___copy_from_user, . - ___copy_from_user .global copy_to_user .type copy_to_user, @function diff --git a/kernel/arch/avr32/mach-at32ap/pio.c b/kernel/arch/avr32/mach-at32ap/pio.c index 4f61378c3..456128174 100644 --- a/kernel/arch/avr32/mach-at32ap/pio.c +++ b/kernel/arch/avr32/mach-at32ap/pio.c @@ -435,7 +435,7 @@ void __init at32_init_pio(struct platform_device *pdev) struct resource *regs; struct pio_device *pio; - if (pdev->id > MAX_NR_PIO_DEVICES) { + if (pdev->id >= MAX_NR_PIO_DEVICES) { dev_err(&pdev->dev, "only %d PIO devices supported\n", MAX_NR_PIO_DEVICES); return; diff --git a/kernel/arch/blackfin/include/asm/uaccess.h b/kernel/arch/blackfin/include/asm/uaccess.h index 90612a7f2..8cd0184ea 100644 --- a/kernel/arch/blackfin/include/asm/uaccess.h +++ b/kernel/arch/blackfin/include/asm/uaccess.h @@ -177,11 +177,12 @@ static inline int bad_user_access_length(void) static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) + if (likely(access_ok(VERIFY_READ, from, n))) { memcpy(to, (const void __force *)from, n); - else - return n; - return 0; + return 0; + } + memset(to, 0, n); + return n; } static inline unsigned long __must_check diff --git a/kernel/arch/blackfin/mach-bf561/boards/cm_bf561.c b/kernel/arch/blackfin/mach-bf561/boards/cm_bf561.c index c6db52ba3..10c577718 100644 --- a/kernel/arch/blackfin/mach-bf561/boards/cm_bf561.c +++ b/kernel/arch/blackfin/mach-bf561/boards/cm_bf561.c @@ -146,7 +146,8 @@ static struct platform_device hitachi_fb_device = { #include static struct smc91x_platdata smc91x_info = { - .flags = SMC91X_USE_32BIT | SMC91X_NOWAIT, + .flags = SMC91X_USE_8BIT | SMC91X_USE_16BIT | SMC91X_USE_32BIT | + SMC91X_NOWAIT, .leda = RPC_LED_100_10, .ledb = RPC_LED_TX_RX, }; diff --git a/kernel/arch/blackfin/mach-bf561/boards/ezkit.c b/kernel/arch/blackfin/mach-bf561/boards/ezkit.c index 2de71e8c1..93c22468c 100644 --- a/kernel/arch/blackfin/mach-bf561/boards/ezkit.c +++ b/kernel/arch/blackfin/mach-bf561/boards/ezkit.c @@ -134,7 +134,8 @@ static struct platform_device net2272_bfin_device = { #include static struct smc91x_platdata smc91x_info = { - .flags = SMC91X_USE_32BIT | SMC91X_NOWAIT, + .flags = SMC91X_USE_8BIT | SMC91X_USE_16BIT | SMC91X_USE_32BIT | + SMC91X_NOWAIT, .leda = RPC_LED_100_10, .ledb = RPC_LED_TX_RX, }; diff --git a/kernel/arch/cris/boot/rescue/Makefile b/kernel/arch/cris/boot/rescue/Makefile index 52bd0bd1d..d98edbb30 100644 --- a/kernel/arch/cris/boot/rescue/Makefile +++ b/kernel/arch/cris/boot/rescue/Makefile @@ -10,6 +10,9 @@ asflags-y += $(LINUXINCLUDE) ccflags-y += -O2 $(LINUXINCLUDE) + +ifdef CONFIG_ETRAX_AXISFLASHMAP + arch-$(CONFIG_ETRAX_ARCH_V10) = v10 arch-$(CONFIG_ETRAX_ARCH_V32) = v32 @@ -28,6 +31,11 @@ $(obj)/rescue.bin: $(obj)/rescue.o FORCE $(call if_changed,objcopy) cp -p $(obj)/rescue.bin $(objtree) +else +$(obj)/rescue.bin: + +endif + $(obj)/testrescue.bin: $(obj)/testrescue.o $(OBJCOPY) $(OBJCOPYFLAGS) $(obj)/testrescue.o tr.bin # Pad it to 784 bytes diff --git a/kernel/arch/cris/include/asm/uaccess.h b/kernel/arch/cris/include/asm/uaccess.h index e3530d0f1..56c7d5750 100644 --- a/kernel/arch/cris/include/asm/uaccess.h +++ b/kernel/arch/cris/include/asm/uaccess.h @@ -194,30 +194,6 @@ extern unsigned long __copy_user(void __user *to, const void *from, unsigned lon extern unsigned long __copy_user_zeroing(void *to, const void __user *from, unsigned long n); extern unsigned long __do_clear_user(void __user *to, unsigned long n); -static inline unsigned long -__generic_copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_user(to, from, n); - return n; -} - -static inline unsigned long -__generic_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (access_ok(VERIFY_READ, from, n)) - return __copy_user_zeroing(to, from, n); - return n; -} - -static inline unsigned long -__generic_clear_user(void __user *to, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - return __do_clear_user(to, n); - return n; -} - static inline long __strncpy_from_user(char *dst, const char __user *src, long count) { @@ -282,7 +258,7 @@ __constant_copy_from_user(void *to, const void __user *from, unsigned long n) else if (n == 24) __asm_copy_from_user_24(to, from, ret); else - ret = __generic_copy_from_user(to, from, n); + ret = __copy_user_zeroing(to, from, n); return ret; } @@ -333,7 +309,7 @@ __constant_copy_to_user(void __user *to, const void *from, unsigned long n) else if (n == 24) __asm_copy_to_user_24(to, from, ret); else - ret = __generic_copy_to_user(to, from, n); + ret = __copy_user(to, from, n); return ret; } @@ -366,26 +342,43 @@ __constant_clear_user(void __user *to, unsigned long n) else if (n == 24) __asm_clear_24(to, ret); else - ret = __generic_clear_user(to, n); + ret = __do_clear_user(to, n); return ret; } -#define clear_user(to, n) \ - (__builtin_constant_p(n) ? \ - __constant_clear_user(to, n) : \ - __generic_clear_user(to, n)) +static inline size_t clear_user(void __user *to, size_t n) +{ + if (unlikely(!access_ok(VERIFY_WRITE, to, n))) + return n; + if (__builtin_constant_p(n)) + return __constant_clear_user(to, n); + else + return __do_clear_user(to, n); +} -#define copy_from_user(to, from, n) \ - (__builtin_constant_p(n) ? \ - __constant_copy_from_user(to, from, n) : \ - __generic_copy_from_user(to, from, n)) +static inline size_t copy_from_user(void *to, const void __user *from, size_t n) +{ + if (unlikely(!access_ok(VERIFY_READ, from, n))) { + memset(to, 0, n); + return n; + } + if (__builtin_constant_p(n)) + return __constant_copy_from_user(to, from, n); + else + return __copy_user_zeroing(to, from, n); +} -#define copy_to_user(to, from, n) \ - (__builtin_constant_p(n) ? \ - __constant_copy_to_user(to, from, n) : \ - __generic_copy_to_user(to, from, n)) +static inline size_t copy_to_user(void __user *to, const void *from, size_t n) +{ + if (unlikely(!access_ok(VERIFY_WRITE, to, n))) + return n; + if (__builtin_constant_p(n)) + return __constant_copy_to_user(to, from, n); + else + return __copy_user(to, from, n); +} /* We let the __ versions of copy_from/to_user inline, because they're often * used in fast paths and have only a small space overhead. diff --git a/kernel/arch/frv/include/asm/uaccess.h b/kernel/arch/frv/include/asm/uaccess.h index 3ac9a59d6..87d9e34c5 100644 --- a/kernel/arch/frv/include/asm/uaccess.h +++ b/kernel/arch/frv/include/asm/uaccess.h @@ -263,19 +263,25 @@ do { \ extern long __memset_user(void *dst, unsigned long count); extern long __memcpy_user(void *dst, const void *src, unsigned long count); -#define clear_user(dst,count) __memset_user(____force(dst), (count)) +#define __clear_user(dst,count) __memset_user(____force(dst), (count)) #define __copy_from_user_inatomic(to, from, n) __memcpy_user((to), ____force(from), (n)) #define __copy_to_user_inatomic(to, from, n) __memcpy_user(____force(to), (from), (n)) #else -#define clear_user(dst,count) (memset(____force(dst), 0, (count)), 0) +#define __clear_user(dst,count) (memset(____force(dst), 0, (count)), 0) #define __copy_from_user_inatomic(to, from, n) (memcpy((to), ____force(from), (n)), 0) #define __copy_to_user_inatomic(to, from, n) (memcpy(____force(to), (from), (n)), 0) #endif -#define __clear_user clear_user +static inline unsigned long __must_check +clear_user(void __user *to, unsigned long n) +{ + if (likely(__access_ok(to, n))) + n = __clear_user(to, n); + return n; +} static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) diff --git a/kernel/arch/h8300/include/asm/thread_info.h b/kernel/arch/h8300/include/asm/thread_info.h index b408fe660..3cef06875 100644 --- a/kernel/arch/h8300/include/asm/thread_info.h +++ b/kernel/arch/h8300/include/asm/thread_info.h @@ -31,7 +31,6 @@ struct thread_info { int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ mm_segment_t addr_limit; - struct restart_block restart_block; }; /* @@ -44,9 +43,6 @@ struct thread_info { .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ } #define init_thread_info (init_thread_union.thread_info) diff --git a/kernel/arch/h8300/kernel/signal.c b/kernel/arch/h8300/kernel/signal.c index 380fffd08..036ad04ed 100644 --- a/kernel/arch/h8300/kernel/signal.c +++ b/kernel/arch/h8300/kernel/signal.c @@ -79,7 +79,7 @@ restore_sigcontext(struct sigcontext *usc, int *pd0) unsigned int er0; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; /* restore passed registers */ #define COPY(r) do { err |= get_user(regs->r, &usc->sc_##r); } while (0) diff --git a/kernel/arch/hexagon/include/asm/uaccess.h b/kernel/arch/hexagon/include/asm/uaccess.h index f000a382b..f61cfb28e 100644 --- a/kernel/arch/hexagon/include/asm/uaccess.h +++ b/kernel/arch/hexagon/include/asm/uaccess.h @@ -103,7 +103,8 @@ static inline long hexagon_strncpy_from_user(char *dst, const char __user *src, { long res = __strnlen_user(src, n); - /* return from strnlen can't be zero -- that would be rubbish. */ + if (unlikely(!res)) + return -EFAULT; if (res > n) { copy_from_user(dst, src, n); diff --git a/kernel/arch/ia64/include/asm/io.h b/kernel/arch/ia64/include/asm/io.h index 9041bbe2b..8fdb9c7ee 100644 --- a/kernel/arch/ia64/include/asm/io.h +++ b/kernel/arch/ia64/include/asm/io.h @@ -436,6 +436,7 @@ static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned lo return ioremap(phys_addr, size); } #define ioremap_cache ioremap_cache +#define ioremap_uc ioremap_nocache /* diff --git a/kernel/arch/ia64/include/asm/uaccess.h b/kernel/arch/ia64/include/asm/uaccess.h index 4f3fb6ccb..40c2027a2 100644 --- a/kernel/arch/ia64/include/asm/uaccess.h +++ b/kernel/arch/ia64/include/asm/uaccess.h @@ -263,17 +263,15 @@ __copy_from_user (void *to, const void __user *from, unsigned long count) __cu_len; \ }) -#define copy_from_user(to, from, n) \ -({ \ - void *__cu_to = (to); \ - const void __user *__cu_from = (from); \ - long __cu_len = (n); \ - \ - __chk_user_ptr(__cu_from); \ - if (__access_ok(__cu_from, __cu_len, get_fs())) \ - __cu_len = __copy_user((__force void __user *) __cu_to, __cu_from, __cu_len); \ - __cu_len; \ -}) +static inline unsigned long +copy_from_user(void *to, const void __user *from, unsigned long n) +{ + if (likely(__access_ok(from, n, get_fs()))) + n = __copy_user((__force void __user *) to, from, n); + else + memset(to, 0, n); + return n; +} #define __copy_in_user(to, from, size) __copy_user((to), (from), (size)) diff --git a/kernel/arch/m32r/include/asm/uaccess.h b/kernel/arch/m32r/include/asm/uaccess.h index cac7014da..6f8982157 100644 --- a/kernel/arch/m32r/include/asm/uaccess.h +++ b/kernel/arch/m32r/include/asm/uaccess.h @@ -219,7 +219,7 @@ extern int fixup_exception(struct pt_regs *regs); #define __get_user_nocheck(x, ptr, size) \ ({ \ long __gu_err = 0; \ - unsigned long __gu_val; \ + unsigned long __gu_val = 0; \ might_fault(); \ __get_user_size(__gu_val, (ptr), (size), __gu_err); \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ diff --git a/kernel/arch/m68k/include/asm/delay.h b/kernel/arch/m68k/include/asm/delay.h index d28fa8fe2..c598d847d 100644 --- a/kernel/arch/m68k/include/asm/delay.h +++ b/kernel/arch/m68k/include/asm/delay.h @@ -114,6 +114,6 @@ static inline void __udelay(unsigned long usecs) */ #define HZSCALE (268435456 / (1000000 / HZ)) -#define ndelay(n) __delay(DIV_ROUND_UP((n) * ((((HZSCALE) >> 11) * (loops_per_jiffy >> 11)) >> 6), 1000)); +#define ndelay(n) __delay(DIV_ROUND_UP((n) * ((((HZSCALE) >> 11) * (loops_per_jiffy >> 11)) >> 6), 1000)) #endif /* defined(_M68K_DELAY_H) */ diff --git a/kernel/arch/metag/include/asm/atomic.h b/kernel/arch/metag/include/asm/atomic.h index 470e365f0..8ff0a7086 100644 --- a/kernel/arch/metag/include/asm/atomic.h +++ b/kernel/arch/metag/include/asm/atomic.h @@ -39,11 +39,10 @@ #define atomic_dec(v) atomic_sub(1, (v)) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) +#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) #endif -#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) - #include #endif /* __ASM_METAG_ATOMIC_H */ diff --git a/kernel/arch/metag/include/asm/atomic_lnkget.h b/kernel/arch/metag/include/asm/atomic_lnkget.h index a62581815..88fa25fae 100644 --- a/kernel/arch/metag/include/asm/atomic_lnkget.h +++ b/kernel/arch/metag/include/asm/atomic_lnkget.h @@ -61,7 +61,7 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ " CMPT %0, #HI(0x02000000)\n" \ " BNZ 1b\n" \ : "=&d" (temp), "=&da" (result) \ - : "da" (&v->counter), "bd" (i) \ + : "da" (&v->counter), "br" (i) \ : "cc"); \ \ smp_mb(); \ diff --git a/kernel/arch/metag/include/asm/cmpxchg_lnkget.h b/kernel/arch/metag/include/asm/cmpxchg_lnkget.h index 0154e2807..2369ad394 100644 --- a/kernel/arch/metag/include/asm/cmpxchg_lnkget.h +++ b/kernel/arch/metag/include/asm/cmpxchg_lnkget.h @@ -73,7 +73,7 @@ static inline unsigned long __cmpxchg_u32(volatile int *m, unsigned long old, " DCACHE [%2], %0\n" #endif "2:\n" - : "=&d" (temp), "=&da" (retval) + : "=&d" (temp), "=&d" (retval) : "da" (m), "bd" (old), "da" (new) : "cc" ); diff --git a/kernel/arch/metag/include/asm/uaccess.h b/kernel/arch/metag/include/asm/uaccess.h index 8282cbce7..273e61225 100644 --- a/kernel/arch/metag/include/asm/uaccess.h +++ b/kernel/arch/metag/include/asm/uaccess.h @@ -204,8 +204,9 @@ extern unsigned long __must_check __copy_user_zeroing(void *to, static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) + if (likely(access_ok(VERIFY_READ, from, n))) return __copy_user_zeroing(to, from, n); + memset(to, 0, n); return n; } diff --git a/kernel/arch/microblaze/include/asm/uaccess.h b/kernel/arch/microblaze/include/asm/uaccess.h index 331b0d35f..826676778 100644 --- a/kernel/arch/microblaze/include/asm/uaccess.h +++ b/kernel/arch/microblaze/include/asm/uaccess.h @@ -227,7 +227,7 @@ extern long __user_bad(void); #define __get_user(x, ptr) \ ({ \ - unsigned long __gu_val; \ + unsigned long __gu_val = 0; \ /*unsigned long __gu_ptr = (unsigned long)(ptr);*/ \ long __gu_err; \ switch (sizeof(*(ptr))) { \ @@ -373,10 +373,13 @@ extern long __user_bad(void); static inline long copy_from_user(void *to, const void __user *from, unsigned long n) { + unsigned long res = n; might_fault(); - if (access_ok(VERIFY_READ, from, n)) - return __copy_from_user(to, from, n); - return n; + if (likely(access_ok(VERIFY_READ, from, n))) + res = __copy_from_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; } #define __copy_to_user(to, from, n) \ diff --git a/kernel/arch/mips/Kconfig.debug b/kernel/arch/mips/Kconfig.debug index f0e314ceb..7f975b20b 100644 --- a/kernel/arch/mips/Kconfig.debug +++ b/kernel/arch/mips/Kconfig.debug @@ -113,42 +113,6 @@ config SPINLOCK_TEST help Add several files to the debugfs to test spinlock speed. -if CPU_MIPSR6 - -choice - prompt "Compact branch policy" - default MIPS_COMPACT_BRANCHES_OPTIMAL - -config MIPS_COMPACT_BRANCHES_NEVER - bool "Never (force delay slot branches)" - help - Pass the -mcompact-branches=never flag to the compiler in order to - force it to always emit branches with delay slots, and make no use - of the compact branch instructions introduced by MIPSr6. This is - useful if you suspect there may be an issue with compact branches in - either the compiler or the CPU. - -config MIPS_COMPACT_BRANCHES_OPTIMAL - bool "Optimal (use where beneficial)" - help - Pass the -mcompact-branches=optimal flag to the compiler in order for - it to make use of compact branch instructions where it deems them - beneficial, and use branches with delay slots elsewhere. This is the - default compiler behaviour, and should be used unless you have a - reason to choose otherwise. - -config MIPS_COMPACT_BRANCHES_ALWAYS - bool "Always (force compact branches)" - help - Pass the -mcompact-branches=always flag to the compiler in order to - force it to always emit compact branches, making no use of branch - instructions with delay slots. This can result in more compact code - which may be beneficial in some scenarios. - -endchoice - -endif # CPU_MIPSR6 - config SCACHE_DEBUGFS bool "L2 cache debugfs entries" depends on DEBUG_FS diff --git a/kernel/arch/mips/Makefile b/kernel/arch/mips/Makefile index 3f70ba54a..252e34795 100644 --- a/kernel/arch/mips/Makefile +++ b/kernel/arch/mips/Makefile @@ -204,10 +204,6 @@ toolchain-msa := $(call cc-option-yn,$(mips-cflags) -mhard-float -mfp64 -Wa$( cflags-$(toolchain-msa) += -DTOOLCHAIN_SUPPORTS_MSA endif -cflags-$(CONFIG_MIPS_COMPACT_BRANCHES_NEVER) += -mcompact-branches=never -cflags-$(CONFIG_MIPS_COMPACT_BRANCHES_OPTIMAL) += -mcompact-branches=optimal -cflags-$(CONFIG_MIPS_COMPACT_BRANCHES_ALWAYS) += -mcompact-branches=always - # # Firmware support # diff --git a/kernel/arch/mips/alchemy/devboards/db1000.c b/kernel/arch/mips/alchemy/devboards/db1000.c index bdeed9d13..433c4b9a9 100644 --- a/kernel/arch/mips/alchemy/devboards/db1000.c +++ b/kernel/arch/mips/alchemy/devboards/db1000.c @@ -503,15 +503,15 @@ int __init db1000_dev_setup(void) if (board == BCSR_WHOAMI_DB1500) { c0 = AU1500_GPIO2_INT; c1 = AU1500_GPIO5_INT; - d0 = AU1500_GPIO0_INT; - d1 = AU1500_GPIO3_INT; + d0 = 0; /* GPIO number, NOT irq! */ + d1 = 3; /* GPIO number, NOT irq! */ s0 = AU1500_GPIO1_INT; s1 = AU1500_GPIO4_INT; } else if (board == BCSR_WHOAMI_DB1100) { c0 = AU1100_GPIO2_INT; c1 = AU1100_GPIO5_INT; - d0 = AU1100_GPIO0_INT; - d1 = AU1100_GPIO3_INT; + d0 = 0; /* GPIO number, NOT irq! */ + d1 = 3; /* GPIO number, NOT irq! */ s0 = AU1100_GPIO1_INT; s1 = AU1100_GPIO4_INT; @@ -545,15 +545,15 @@ int __init db1000_dev_setup(void) } else if (board == BCSR_WHOAMI_DB1000) { c0 = AU1000_GPIO2_INT; c1 = AU1000_GPIO5_INT; - d0 = AU1000_GPIO0_INT; - d1 = AU1000_GPIO3_INT; + d0 = 0; /* GPIO number, NOT irq! */ + d1 = 3; /* GPIO number, NOT irq! */ s0 = AU1000_GPIO1_INT; s1 = AU1000_GPIO4_INT; platform_add_devices(db1000_devs, ARRAY_SIZE(db1000_devs)); } else if ((board == BCSR_WHOAMI_PB1500) || (board == BCSR_WHOAMI_PB1500R2)) { c0 = AU1500_GPIO203_INT; - d0 = AU1500_GPIO201_INT; + d0 = 1; /* GPIO number, NOT irq! */ s0 = AU1500_GPIO202_INT; twosocks = 0; flashsize = 64; @@ -566,7 +566,7 @@ int __init db1000_dev_setup(void) */ } else if (board == BCSR_WHOAMI_PB1100) { c0 = AU1100_GPIO11_INT; - d0 = AU1100_GPIO9_INT; + d0 = 9; /* GPIO number, NOT irq! */ s0 = AU1100_GPIO10_INT; twosocks = 0; flashsize = 64; @@ -583,7 +583,6 @@ int __init db1000_dev_setup(void) } else return 0; /* unknown board, no further dev setup to do */ - irq_set_irq_type(d0, IRQ_TYPE_EDGE_BOTH); irq_set_irq_type(c0, IRQ_TYPE_LEVEL_LOW); irq_set_irq_type(s0, IRQ_TYPE_LEVEL_LOW); @@ -597,7 +596,6 @@ int __init db1000_dev_setup(void) c0, d0, /*s0*/0, 0, 0); if (twosocks) { - irq_set_irq_type(d1, IRQ_TYPE_EDGE_BOTH); irq_set_irq_type(c1, IRQ_TYPE_LEVEL_LOW); irq_set_irq_type(s1, IRQ_TYPE_LEVEL_LOW); diff --git a/kernel/arch/mips/alchemy/devboards/db1550.c b/kernel/arch/mips/alchemy/devboards/db1550.c index 5740bcfdf..6c37b9326 100644 --- a/kernel/arch/mips/alchemy/devboards/db1550.c +++ b/kernel/arch/mips/alchemy/devboards/db1550.c @@ -514,7 +514,7 @@ static void __init db1550_devices(void) AU1000_PCMCIA_MEM_PHYS_ADDR + 0x000400000 - 1, AU1000_PCMCIA_IO_PHYS_ADDR, AU1000_PCMCIA_IO_PHYS_ADDR + 0x000010000 - 1, - AU1550_GPIO3_INT, AU1550_GPIO0_INT, + AU1550_GPIO3_INT, 0, /*AU1550_GPIO21_INT*/0, 0, 0); db1x_register_pcmcia_socket( @@ -524,7 +524,7 @@ static void __init db1550_devices(void) AU1000_PCMCIA_MEM_PHYS_ADDR + 0x004400000 - 1, AU1000_PCMCIA_IO_PHYS_ADDR + 0x004000000, AU1000_PCMCIA_IO_PHYS_ADDR + 0x004010000 - 1, - AU1550_GPIO5_INT, AU1550_GPIO1_INT, + AU1550_GPIO5_INT, 1, /*AU1550_GPIO22_INT*/0, 0, 1); platform_device_register(&db1550_nand_dev); diff --git a/kernel/arch/mips/ath79/early_printk.c b/kernel/arch/mips/ath79/early_printk.c index b955fafc5..d1adc59af 100644 --- a/kernel/arch/mips/ath79/early_printk.c +++ b/kernel/arch/mips/ath79/early_printk.c @@ -31,13 +31,15 @@ static inline void prom_putchar_wait(void __iomem *reg, u32 mask, u32 val) } while (1); } +#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) + static void prom_putchar_ar71xx(unsigned char ch) { void __iomem *base = (void __iomem *)(KSEG1ADDR(AR71XX_UART_BASE)); - prom_putchar_wait(base + UART_LSR * 4, UART_LSR_THRE, UART_LSR_THRE); + prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); __raw_writel(ch, base + UART_TX * 4); - prom_putchar_wait(base + UART_LSR * 4, UART_LSR_THRE, UART_LSR_THRE); + prom_putchar_wait(base + UART_LSR * 4, BOTH_EMPTY, BOTH_EMPTY); } static void prom_putchar_ar933x(unsigned char ch) diff --git a/kernel/arch/mips/include/asm/asmmacro.h b/kernel/arch/mips/include/asm/asmmacro.h index 867f924b0..8dedee1de 100644 --- a/kernel/arch/mips/include/asm/asmmacro.h +++ b/kernel/arch/mips/include/asm/asmmacro.h @@ -135,6 +135,7 @@ ldc1 $f28, THREAD_FPR28(\thread) ldc1 $f30, THREAD_FPR30(\thread) ctc1 \tmp, fcr31 + .set pop .endm .macro fpu_restore_16odd thread @@ -298,21 +299,21 @@ .set pop .endm - .macro copy_u_w ws, n + .macro copy_s_w ws, n .set push .set mips32r2 .set fp=64 .set msa - copy_u.w $1, $w\ws[\n] + copy_s.w $1, $w\ws[\n] .set pop .endm - .macro copy_u_d ws, n + .macro copy_s_d ws, n .set push .set mips64r2 .set fp=64 .set msa - copy_u.d $1, $w\ws[\n] + copy_s.d $1, $w\ws[\n] .set pop .endm @@ -346,8 +347,8 @@ #define STH_MSA_INSN 0x5800081f #define STW_MSA_INSN 0x5800082f #define STD_MSA_INSN 0x5800083f -#define COPY_UW_MSA_INSN 0x58f00056 -#define COPY_UD_MSA_INSN 0x58f80056 +#define COPY_SW_MSA_INSN 0x58b00056 +#define COPY_SD_MSA_INSN 0x58b80056 #define INSERT_W_MSA_INSN 0x59300816 #define INSERT_D_MSA_INSN 0x59380816 #else @@ -361,8 +362,8 @@ #define STH_MSA_INSN 0x78000825 #define STW_MSA_INSN 0x78000826 #define STD_MSA_INSN 0x78000827 -#define COPY_UW_MSA_INSN 0x78f00059 -#define COPY_UD_MSA_INSN 0x78f80059 +#define COPY_SW_MSA_INSN 0x78b00059 +#define COPY_SD_MSA_INSN 0x78b80059 #define INSERT_W_MSA_INSN 0x79300819 #define INSERT_D_MSA_INSN 0x79380819 #endif @@ -393,7 +394,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word LDB_MSA_INSN | (\wd << 6) .set pop .endm @@ -402,7 +403,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word LDH_MSA_INSN | (\wd << 6) .set pop .endm @@ -411,7 +412,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word LDW_MSA_INSN | (\wd << 6) .set pop .endm @@ -420,7 +421,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word LDD_MSA_INSN | (\wd << 6) .set pop .endm @@ -429,7 +430,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word STB_MSA_INSN | (\wd << 6) .set pop .endm @@ -438,7 +439,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word STH_MSA_INSN | (\wd << 6) .set pop .endm @@ -447,7 +448,7 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word STW_MSA_INSN | (\wd << 6) .set pop .endm @@ -456,26 +457,26 @@ .set push .set noat SET_HARDFLOAT - addu $1, \base, \off + PTR_ADDU $1, \base, \off .word STD_MSA_INSN | (\wd << 6) .set pop .endm - .macro copy_u_w ws, n + .macro copy_s_w ws, n .set push .set noat SET_HARDFLOAT .insn - .word COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11) + .word COPY_SW_MSA_INSN | (\n << 16) | (\ws << 11) .set pop .endm - .macro copy_u_d ws, n + .macro copy_s_d ws, n .set push .set noat SET_HARDFLOAT .insn - .word COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11) + .word COPY_SD_MSA_INSN | (\n << 16) | (\ws << 11) .set pop .endm diff --git a/kernel/arch/mips/include/asm/cacheflush.h b/kernel/arch/mips/include/asm/cacheflush.h index 723229f4c..176de586a 100644 --- a/kernel/arch/mips/include/asm/cacheflush.h +++ b/kernel/arch/mips/include/asm/cacheflush.h @@ -51,7 +51,6 @@ extern void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page, unsigned long pfn); extern void __flush_dcache_page(struct page *page); -extern void __flush_icache_page(struct vm_area_struct *vma, struct page *page); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 static inline void flush_dcache_page(struct page *page) @@ -77,11 +76,6 @@ static inline void flush_anon_page(struct vm_area_struct *vma, static inline void flush_icache_page(struct vm_area_struct *vma, struct page *page) { - if (!cpu_has_ic_fills_f_dc && (vma->vm_flags & VM_EXEC) && - Page_dcache_dirty(page)) { - __flush_icache_page(vma, page); - ClearPageDcacheDirty(page); - } } extern void (*flush_icache_range)(unsigned long start, unsigned long end); diff --git a/kernel/arch/mips/include/asm/kvm_host.h b/kernel/arch/mips/include/asm/kvm_host.h index 6ded8d347..c8c04a1f1 100644 --- a/kernel/arch/mips/include/asm/kvm_host.h +++ b/kernel/arch/mips/include/asm/kvm_host.h @@ -372,6 +372,7 @@ struct kvm_mips_tlb { #define KVM_MIPS_GUEST_TLB_SIZE 64 struct kvm_vcpu_arch { void *host_ebase, *guest_ebase; + int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu); unsigned long host_stack; unsigned long host_gp; @@ -399,7 +400,10 @@ struct kvm_vcpu_arch { /* Host KSEG0 address of the EI/DI offset */ void *kseg0_commpage; - u32 io_gpr; /* GPR used as IO source/target */ + /* Resume PC after MMIO completion */ + unsigned long io_pc; + /* GPR used as IO source/target */ + u32 io_gpr; struct hrtimer comparecount_timer; /* Count timer control KVM register */ @@ -421,8 +425,6 @@ struct kvm_vcpu_arch { /* Bitmask of pending exceptions to be cleared */ unsigned long pending_exceptions_clr; - unsigned long pending_load_cause; - /* Save/Restore the entryhi register when are are preempted/scheduled back in */ unsigned long preempt_entryhi; @@ -784,7 +786,7 @@ extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, uint32_t kvm_mips_read_count(struct kvm_vcpu *vcpu); void kvm_mips_write_count(struct kvm_vcpu *vcpu, uint32_t count); -void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare); +void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare, bool ack); void kvm_mips_init_count(struct kvm_vcpu *vcpu); int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl); int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume); diff --git a/kernel/arch/mips/include/asm/mach-paravirt/kernel-entry-init.h b/kernel/arch/mips/include/asm/mach-paravirt/kernel-entry-init.h index 2f82bfa3a..c9f5769df 100644 --- a/kernel/arch/mips/include/asm/mach-paravirt/kernel-entry-init.h +++ b/kernel/arch/mips/include/asm/mach-paravirt/kernel-entry-init.h @@ -11,11 +11,13 @@ #define CP0_EBASE $15, 1 .macro kernel_entry_setup +#ifdef CONFIG_SMP mfc0 t0, CP0_EBASE andi t0, t0, 0x3ff # CPUNum beqz t0, 1f # CPUs other than zero goto smp_bootstrap j smp_bootstrap +#endif /* CONFIG_SMP */ 1: .endm diff --git a/kernel/arch/mips/include/asm/msa.h b/kernel/arch/mips/include/asm/msa.h index bbb85fe21..6e4effa6f 100644 --- a/kernel/arch/mips/include/asm/msa.h +++ b/kernel/arch/mips/include/asm/msa.h @@ -147,6 +147,19 @@ static inline void restore_msa(struct task_struct *t) _restore_msa(t); } +static inline void init_msa_upper(void) +{ + /* + * Check cpu_has_msa only if it's a constant. This will allow the + * compiler to optimise out code for CPUs without MSA without adding + * an extra redundant check for CPUs with MSA. + */ + if (__builtin_constant_p(cpu_has_msa) && !cpu_has_msa) + return; + + _init_msa_upper(); +} + #ifdef TOOLCHAIN_SUPPORTS_MSA #define __BUILD_MSA_CTL_REG(name, cs) \ diff --git a/kernel/arch/mips/include/asm/pgtable.h b/kernel/arch/mips/include/asm/pgtable.h index 18826aa15..4e68c644a 100644 --- a/kernel/arch/mips/include/asm/pgtable.h +++ b/kernel/arch/mips/include/asm/pgtable.h @@ -127,10 +127,14 @@ do { \ } \ } while(0) +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval); + #if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32) #define pte_none(pte) (!(((pte).pte_high) & ~_PAGE_GLOBAL)) #define pte_present(pte) ((pte).pte_low & _PAGE_PRESENT) +#define pte_no_exec(pte) ((pte).pte_low & _PAGE_NO_EXEC) static inline void set_pte(pte_t *ptep, pte_t pte) { @@ -148,7 +152,6 @@ static inline void set_pte(pte_t *ptep, pte_t pte) buddy->pte_high |= _PAGE_GLOBAL; } } -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval) static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { @@ -166,6 +169,7 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt #define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) #define pte_present(pte) (pte_val(pte) & _PAGE_PRESENT) +#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC) /* * Certain architectures need to do special things when pte's @@ -218,7 +222,6 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) } #endif } -#define set_pte_at(mm, addr, ptep, pteval) set_pte(ptep, pteval) static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { @@ -234,6 +237,22 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt } #endif +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + extern void __update_cache(unsigned long address, pte_t pte); + + if (!pte_present(pteval)) + goto cache_sync_done; + + if (pte_present(*ptep) && (pte_pfn(*ptep) == pte_pfn(pteval))) + goto cache_sync_done; + + __update_cache(addr, pteval); +cache_sync_done: + set_pte(ptep, pteval); +} + /* * (pmds are folded into puds so this doesn't get actually called, * but the define is needed for a generic inline function.) @@ -430,15 +449,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) extern void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte); -extern void __update_cache(struct vm_area_struct *vma, unsigned long address, - pte_t pte); static inline void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { pte_t pte = *ptep; __update_tlb(vma, address, pte); - __update_cache(vma, address, pte); } static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, diff --git a/kernel/arch/mips/include/asm/processor.h b/kernel/arch/mips/include/asm/processor.h index 3f832c3dd..041153f5c 100644 --- a/kernel/arch/mips/include/asm/processor.h +++ b/kernel/arch/mips/include/asm/processor.h @@ -45,7 +45,7 @@ extern unsigned int vced_count, vcei_count; * User space process size: 2GB. This is hardcoded into a few places, * so don't change it unless you know what you are doing. */ -#define TASK_SIZE 0x7fff8000UL +#define TASK_SIZE 0x80000000UL #endif #define STACK_TOP_MAX TASK_SIZE diff --git a/kernel/arch/mips/include/asm/ptrace.h b/kernel/arch/mips/include/asm/ptrace.h index f6fc6aac5..b6578611d 100644 --- a/kernel/arch/mips/include/asm/ptrace.h +++ b/kernel/arch/mips/include/asm/ptrace.h @@ -152,7 +152,7 @@ static inline int is_syscall_success(struct pt_regs *regs) static inline long regs_return_value(struct pt_regs *regs) { - if (is_syscall_success(regs)) + if (is_syscall_success(regs) || !user_mode(regs)) return regs->regs[2]; else return -regs->regs[2]; diff --git a/kernel/arch/mips/include/asm/switch_to.h b/kernel/arch/mips/include/asm/switch_to.h index 28b5d84a5..ebb5c0f2f 100644 --- a/kernel/arch/mips/include/asm/switch_to.h +++ b/kernel/arch/mips/include/asm/switch_to.h @@ -105,7 +105,7 @@ do { \ __clear_software_ll_bit(); \ if (cpu_has_userlocal) \ write_c0_userlocal(task_thread_info(next)->tp_value); \ - __restore_watch(); \ + __restore_watch(next); \ (last) = resume(prev, next, task_thread_info(next)); \ } while (0) diff --git a/kernel/arch/mips/include/asm/uaccess.h b/kernel/arch/mips/include/asm/uaccess.h index 095ecafe6..c74c32ccc 100644 --- a/kernel/arch/mips/include/asm/uaccess.h +++ b/kernel/arch/mips/include/asm/uaccess.h @@ -14,6 +14,7 @@ #include #include #include +#include #include /* @@ -1170,6 +1171,8 @@ extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); __cu_len = __invoke_copy_from_user(__cu_to, \ __cu_from, \ __cu_len); \ + } else { \ + memset(__cu_to, 0, __cu_len); \ } \ } \ __cu_len; \ diff --git a/kernel/arch/mips/include/asm/uprobes.h b/kernel/arch/mips/include/asm/uprobes.h index 34c325c67..70a4a2f17 100644 --- a/kernel/arch/mips/include/asm/uprobes.h +++ b/kernel/arch/mips/include/asm/uprobes.h @@ -36,7 +36,6 @@ struct arch_uprobe { unsigned long resume_epc; u32 insn[2]; u32 ixol[2]; - union mips_instruction orig_inst[MAX_UINSN_BYTES / 4]; }; struct arch_uprobe_task { diff --git a/kernel/arch/mips/include/asm/watch.h b/kernel/arch/mips/include/asm/watch.h index 20126ec79..6ffe3eadf 100644 --- a/kernel/arch/mips/include/asm/watch.h +++ b/kernel/arch/mips/include/asm/watch.h @@ -12,21 +12,21 @@ #include -void mips_install_watch_registers(void); +void mips_install_watch_registers(struct task_struct *t); void mips_read_watch_registers(void); void mips_clear_watch_registers(void); void mips_probe_watch_registers(struct cpuinfo_mips *c); #ifdef CONFIG_HARDWARE_WATCHPOINTS -#define __restore_watch() do { \ +#define __restore_watch(task) do { \ if (unlikely(test_bit(TIF_LOAD_WATCH, \ - ¤t_thread_info()->flags))) { \ - mips_install_watch_registers(); \ + &task_thread_info(task)->flags))) { \ + mips_install_watch_registers(task); \ } \ } while (0) #else -#define __restore_watch() do {} while (0) +#define __restore_watch(task) do {} while (0) #endif #endif /* _ASM_WATCH_H */ diff --git a/kernel/arch/mips/include/uapi/asm/siginfo.h b/kernel/arch/mips/include/uapi/asm/siginfo.h index 2cb7fdead..e2b5337e8 100644 --- a/kernel/arch/mips/include/uapi/asm/siginfo.h +++ b/kernel/arch/mips/include/uapi/asm/siginfo.h @@ -28,7 +28,7 @@ #define __ARCH_SIGSYS -#include +#include /* We can't use generic siginfo_t, because our si_code and si_errno are swapped */ typedef struct siginfo { @@ -42,13 +42,13 @@ typedef struct siginfo { /* kill() */ struct { - pid_t _pid; /* sender's pid */ + __kernel_pid_t _pid; /* sender's pid */ __ARCH_SI_UID_T _uid; /* sender's uid */ } _kill; /* POSIX.1b timers */ struct { - timer_t _tid; /* timer id */ + __kernel_timer_t _tid; /* timer id */ int _overrun; /* overrun count */ char _pad[sizeof( __ARCH_SI_UID_T) - sizeof(int)]; sigval_t _sigval; /* same as below */ @@ -57,26 +57,26 @@ typedef struct siginfo { /* POSIX.1b signals */ struct { - pid_t _pid; /* sender's pid */ + __kernel_pid_t _pid; /* sender's pid */ __ARCH_SI_UID_T _uid; /* sender's uid */ sigval_t _sigval; } _rt; /* SIGCHLD */ struct { - pid_t _pid; /* which child */ + __kernel_pid_t _pid; /* which child */ __ARCH_SI_UID_T _uid; /* sender's uid */ int _status; /* exit code */ - clock_t _utime; - clock_t _stime; + __kernel_clock_t _utime; + __kernel_clock_t _stime; } _sigchld; /* IRIX SIGCHLD */ struct { - pid_t _pid; /* which child */ - clock_t _utime; + __kernel_pid_t _pid; /* which child */ + __kernel_clock_t _utime; int _status; /* exit code */ - clock_t _stime; + __kernel_clock_t _stime; } _irix_sigchld; /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */ @@ -118,6 +118,4 @@ typedef struct siginfo { #define SI_TIMER __SI_CODE(__SI_TIMER, -3) /* sent by timer expiration */ #define SI_MESGQ __SI_CODE(__SI_MESGQ, -4) /* sent by real time mesq state change */ -#include - #endif /* _UAPI_ASM_SIGINFO_H */ diff --git a/kernel/arch/mips/kernel/csrc-r4k.c b/kernel/arch/mips/kernel/csrc-r4k.c index 1f910563f..d76275da5 100644 --- a/kernel/arch/mips/kernel/csrc-r4k.c +++ b/kernel/arch/mips/kernel/csrc-r4k.c @@ -23,7 +23,7 @@ static struct clocksource clocksource_mips = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static u64 notrace r4k_read_sched_clock(void) +static u64 __maybe_unused notrace r4k_read_sched_clock(void) { return read_c0_count(); } @@ -82,7 +82,9 @@ int __init init_r4k_clocksource(void) clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); +#ifndef CONFIG_CPU_FREQ sched_clock_register(r4k_read_sched_clock, 32, mips_hpt_frequency); +#endif return 0; } diff --git a/kernel/arch/mips/kernel/mips-r2-to-r6-emul.c b/kernel/arch/mips/kernel/mips-r2-to-r6-emul.c index 1f5aac7f9..af27334d6 100644 --- a/kernel/arch/mips/kernel/mips-r2-to-r6-emul.c +++ b/kernel/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -1163,7 +1164,9 @@ fpu_emul: regs->regs[31] = r31; regs->cp0_epc = epc; if (!used_math()) { /* First time FPU user. */ + preempt_disable(); err = init_fpu(); + preempt_enable(); set_used_math(); } lose_fpu(1); /* Save FPU state for the emulator. */ @@ -1251,10 +1254,10 @@ fpu_emul: " j 10b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1326,10 +1329,10 @@ fpu_emul: " j 10b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1397,10 +1400,10 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1467,10 +1470,10 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1582,14 +1585,14 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" - " .word 5b,8b\n" - " .word 6b,8b\n" - " .word 7b,8b\n" - " .word 0b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" + STR(PTR) " 5b,8b\n" + STR(PTR) " 6b,8b\n" + STR(PTR) " 7b,8b\n" + STR(PTR) " 0b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1701,14 +1704,14 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" - " .word 5b,8b\n" - " .word 6b,8b\n" - " .word 7b,8b\n" - " .word 0b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" + STR(PTR) " 5b,8b\n" + STR(PTR) " 6b,8b\n" + STR(PTR) " 7b,8b\n" + STR(PTR) " 0b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1820,14 +1823,14 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" - " .word 5b,8b\n" - " .word 6b,8b\n" - " .word 7b,8b\n" - " .word 0b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" + STR(PTR) " 5b,8b\n" + STR(PTR) " 6b,8b\n" + STR(PTR) " 7b,8b\n" + STR(PTR) " 0b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -1938,14 +1941,14 @@ fpu_emul: " j 9b\n" " .previous\n" " .section __ex_table,\"a\"\n" - " .word 1b,8b\n" - " .word 2b,8b\n" - " .word 3b,8b\n" - " .word 4b,8b\n" - " .word 5b,8b\n" - " .word 6b,8b\n" - " .word 7b,8b\n" - " .word 0b,8b\n" + STR(PTR) " 1b,8b\n" + STR(PTR) " 2b,8b\n" + STR(PTR) " 3b,8b\n" + STR(PTR) " 4b,8b\n" + STR(PTR) " 5b,8b\n" + STR(PTR) " 6b,8b\n" + STR(PTR) " 7b,8b\n" + STR(PTR) " 0b,8b\n" " .previous\n" " .set pop\n" : "+&r"(rt), "=&r"(rs), @@ -2000,7 +2003,7 @@ fpu_emul: "j 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" - ".word 1b, 3b\n" + STR(PTR) " 1b,3b\n" ".previous\n" : "=&r"(res), "+&r"(err) : "r"(vaddr), "i"(SIGSEGV) @@ -2058,7 +2061,7 @@ fpu_emul: "j 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" - ".word 1b, 3b\n" + STR(PTR) " 1b,3b\n" ".previous\n" : "+&r"(res), "+&r"(err) : "r"(vaddr), "i"(SIGSEGV)); @@ -2119,7 +2122,7 @@ fpu_emul: "j 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" - ".word 1b, 3b\n" + STR(PTR) " 1b,3b\n" ".previous\n" : "=&r"(res), "+&r"(err) : "r"(vaddr), "i"(SIGSEGV) @@ -2182,7 +2185,7 @@ fpu_emul: "j 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" - ".word 1b, 3b\n" + STR(PTR) " 1b,3b\n" ".previous\n" : "+&r"(res), "+&r"(err) : "r"(vaddr), "i"(SIGSEGV)); diff --git a/kernel/arch/mips/kernel/pm.c b/kernel/arch/mips/kernel/pm.c index fefdf39d3..dc8148921 100644 --- a/kernel/arch/mips/kernel/pm.c +++ b/kernel/arch/mips/kernel/pm.c @@ -56,7 +56,7 @@ static void mips_cpu_restore(void) write_c0_userlocal(current_thread_info()->tp_value); /* Restore watch registers */ - __restore_watch(); + __restore_watch(current); } /** diff --git a/kernel/arch/mips/kernel/process.c b/kernel/arch/mips/kernel/process.c index f2975d4d1..44a6f25e9 100644 --- a/kernel/arch/mips/kernel/process.c +++ b/kernel/arch/mips/kernel/process.c @@ -457,7 +457,7 @@ unsigned long notrace unwind_stack_by_address(unsigned long stack_page, *sp + sizeof(*regs) <= stack_page + THREAD_SIZE - 32) { regs = (struct pt_regs *)*sp; pc = regs->cp0_epc; - if (__kernel_text_address(pc)) { + if (!user_mode(regs) && __kernel_text_address(pc)) { *sp = regs->regs[29]; *ra = regs->regs[31]; return pc; @@ -593,16 +593,19 @@ int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) return -EOPNOTSUPP; /* Avoid inadvertently triggering emulation */ - if ((value & PR_FP_MODE_FR) && cpu_has_fpu && - !(current_cpu_data.fpu_id & MIPS_FPIR_F64)) + if ((value & PR_FP_MODE_FR) && raw_cpu_has_fpu && + !(raw_current_cpu_data.fpu_id & MIPS_FPIR_F64)) return -EOPNOTSUPP; - if ((value & PR_FP_MODE_FRE) && cpu_has_fpu && !cpu_has_fre) + if ((value & PR_FP_MODE_FRE) && raw_cpu_has_fpu && !cpu_has_fre) return -EOPNOTSUPP; /* FR = 0 not supported in MIPS R6 */ - if (!(value & PR_FP_MODE_FR) && cpu_has_fpu && cpu_has_mips_r6) + if (!(value & PR_FP_MODE_FR) && raw_cpu_has_fpu && cpu_has_mips_r6) return -EOPNOTSUPP; + /* Proceed with the mode switch */ + preempt_disable(); + /* Save FP & vector context, then disable FPU & MSA */ if (task->signal == current->signal) lose_fpu(1); @@ -661,6 +664,7 @@ int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) /* Allow threads to use FP again */ atomic_set(&task->mm->context.fp_mode_switching, 0); + preempt_enable(); return 0; } diff --git a/kernel/arch/mips/kernel/ptrace.c b/kernel/arch/mips/kernel/ptrace.c index 4f0ac78d1..74d581569 100644 --- a/kernel/arch/mips/kernel/ptrace.c +++ b/kernel/arch/mips/kernel/ptrace.c @@ -57,8 +57,7 @@ static void init_fp_ctx(struct task_struct *target) /* Begin with data registers set to all 1s... */ memset(&target->thread.fpu.fpr, ~0, sizeof(target->thread.fpu.fpr)); - /* ...and FCSR zeroed */ - target->thread.fpu.fcr31 = 0; + /* FCSR has been preset by `mips_set_personality_nan'. */ /* * Record that the target has "used" math, such that the context @@ -79,6 +78,22 @@ void ptrace_disable(struct task_struct *child) clear_tsk_thread_flag(child, TIF_LOAD_WATCH); } +/* + * Poke at FCSR according to its mask. Don't set the cause bits as + * this is currently not handled correctly in FP context restoration + * and will cause an oops if a corresponding enable bit is set. + */ +static void ptrace_setfcr31(struct task_struct *child, u32 value) +{ + u32 fcr31; + u32 mask; + + value &= ~FPU_CSR_ALL_X; + fcr31 = child->thread.fpu.fcr31; + mask = boot_cpu_data.fpu_msk31; + child->thread.fpu.fcr31 = (value & ~mask) | (fcr31 & mask); +} + /* * Read a general register set. We always use the 64-bit format, even * for 32-bit kernels and for 32-bit processes on a 64-bit kernel. @@ -159,9 +174,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) { union fpureg *fregs; u64 fpr_val; - u32 fcr31; u32 value; - u32 mask; int i; if (!access_ok(VERIFY_READ, data, 33 * 8)) @@ -176,9 +189,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) } __get_user(value, data + 64); - fcr31 = child->thread.fpu.fcr31; - mask = boot_cpu_data.fpu_msk31; - child->thread.fpu.fcr31 = (value & ~mask) | (fcr31 & mask); + ptrace_setfcr31(child, value); /* FIR may not be written. */ @@ -808,7 +819,7 @@ long arch_ptrace(struct task_struct *child, long request, break; #endif case FPC_CSR: - child->thread.fpu.fcr31 = data & ~FPU_CSR_ALL_X; + ptrace_setfcr31(child, data); break; case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; diff --git a/kernel/arch/mips/kernel/r4k_fpu.S b/kernel/arch/mips/kernel/r4k_fpu.S index f09546ee2..bc74485ec 100644 --- a/kernel/arch/mips/kernel/r4k_fpu.S +++ b/kernel/arch/mips/kernel/r4k_fpu.S @@ -244,17 +244,17 @@ LEAF(\name) .set push .set noat #ifdef CONFIG_64BIT - copy_u_d \wr, 1 + copy_s_d \wr, 1 EX sd $1, \off(\base) #elif defined(CONFIG_CPU_LITTLE_ENDIAN) - copy_u_w \wr, 2 + copy_s_w \wr, 2 EX sw $1, \off(\base) - copy_u_w \wr, 3 + copy_s_w \wr, 3 EX sw $1, (\off+4)(\base) #else /* CONFIG_CPU_BIG_ENDIAN */ - copy_u_w \wr, 2 + copy_s_w \wr, 2 EX sw $1, (\off+4)(\base) - copy_u_w \wr, 3 + copy_s_w \wr, 3 EX sw $1, \off(\base) #endif .set pop diff --git a/kernel/arch/mips/kernel/scall64-n32.S b/kernel/arch/mips/kernel/scall64-n32.S index 5a69eb48d..ee93d5fe6 100644 --- a/kernel/arch/mips/kernel/scall64-n32.S +++ b/kernel/arch/mips/kernel/scall64-n32.S @@ -344,7 +344,7 @@ EXPORT(sysn32_call_table) PTR sys_ni_syscall /* available, was setaltroot */ PTR sys_add_key PTR sys_request_key - PTR sys_keyctl /* 6245 */ + PTR compat_sys_keyctl /* 6245 */ PTR sys_set_thread_area PTR sys_inotify_init PTR sys_inotify_add_watch diff --git a/kernel/arch/mips/kernel/scall64-o32.S b/kernel/arch/mips/kernel/scall64-o32.S index e4b6d7c97..b77052ec6 100644 --- a/kernel/arch/mips/kernel/scall64-o32.S +++ b/kernel/arch/mips/kernel/scall64-o32.S @@ -500,7 +500,7 @@ EXPORT(sys32_call_table) PTR sys_ni_syscall /* available, was setaltroot */ PTR sys_add_key /* 4280 */ PTR sys_request_key - PTR sys_keyctl + PTR compat_sys_keyctl PTR sys_set_thread_area PTR sys_inotify_init PTR sys_inotify_add_watch /* 4285 */ diff --git a/kernel/arch/mips/kernel/setup.c b/kernel/arch/mips/kernel/setup.c index 66aac55df..8acae316f 100644 --- a/kernel/arch/mips/kernel/setup.c +++ b/kernel/arch/mips/kernel/setup.c @@ -706,6 +706,9 @@ static void __init arch_mem_init(char **cmdline_p) for_each_memblock(reserved, reg) if (reg->size != 0) reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT); + + reserve_bootmem_region(__pa_symbol(&__nosave_begin), + __pa_symbol(&__nosave_end)); /* Reserve for hibernation */ } static void __init resource_init(void) diff --git a/kernel/arch/mips/kernel/signal.c b/kernel/arch/mips/kernel/signal.c index bf792e283..9e35b6b26 100644 --- a/kernel/arch/mips/kernel/signal.c +++ b/kernel/arch/mips/kernel/signal.c @@ -195,6 +195,9 @@ static int restore_msa_extcontext(void __user *buf, unsigned int size) unsigned int csr; int i, err; + if (!config_enabled(CONFIG_CPU_HAS_MSA)) + return SIGSYS; + if (size != sizeof(*msa)) return -EINVAL; @@ -398,8 +401,8 @@ int protected_restore_fp_context(void __user *sc) } fp_done: - if (used & USED_EXTCONTEXT) - err |= restore_extcontext(sc_to_extcontext(sc)); + if (!err && (used & USED_EXTCONTEXT)) + err = restore_extcontext(sc_to_extcontext(sc)); return err ?: sig; } @@ -767,15 +770,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) sigset_t *oldset = sigmask_to_save(); int ret; struct mips_abi *abi = current->thread.abi; -#ifdef CONFIG_CPU_MICROMIPS - void *vdso; - unsigned long tmp = (unsigned long)current->mm->context.vdso; - - set_isa16_mode(tmp); - vdso = (void *)tmp; -#else void *vdso = current->mm->context.vdso; -#endif if (regs->regs[0]) { switch(regs->regs[2]) { diff --git a/kernel/arch/mips/kernel/smp.c b/kernel/arch/mips/kernel/smp.c index 2b521e07b..7fef02a9e 100644 --- a/kernel/arch/mips/kernel/smp.c +++ b/kernel/arch/mips/kernel/smp.c @@ -174,6 +174,9 @@ asmlinkage void start_secondary(void) cpumask_set_cpu(cpu, &cpu_coherent_mask); notify_cpu_starting(cpu); + cpumask_set_cpu(cpu, &cpu_callin_map); + synchronise_count_slave(cpu); + set_cpu_online(cpu, true); set_cpu_sibling_map(cpu); @@ -181,10 +184,6 @@ asmlinkage void start_secondary(void) calculate_cpu_foreign_map(); - cpumask_set_cpu(cpu, &cpu_callin_map); - - synchronise_count_slave(cpu); - /* * irq will be enabled in ->smp_finish(), enabling it too early * is dangerous. diff --git a/kernel/arch/mips/kernel/traps.c b/kernel/arch/mips/kernel/traps.c index ca9a81007..99a402231 100644 --- a/kernel/arch/mips/kernel/traps.c +++ b/kernel/arch/mips/kernel/traps.c @@ -144,7 +144,7 @@ static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) if (!task) task = current; - if (raw_show_trace || !__kernel_text_address(pc)) { + if (raw_show_trace || user_mode(regs) || !__kernel_text_address(pc)) { show_raw_backtrace(sp); return; } @@ -1241,7 +1241,7 @@ static int enable_restore_fp_context(int msa) err = init_fpu(); if (msa && !err) { enable_msa(); - _init_msa_upper(); + init_msa_upper(); set_thread_flag(TIF_USEDMSA); set_thread_flag(TIF_MSA_CTX_LIVE); } @@ -1304,7 +1304,7 @@ static int enable_restore_fp_context(int msa) */ prior_msa = test_and_set_thread_flag(TIF_MSA_CTX_LIVE); if (!prior_msa && was_fpu_owner) { - _init_msa_upper(); + init_msa_upper(); goto out; } @@ -1321,7 +1321,7 @@ static int enable_restore_fp_context(int msa) * of each vector register such that it cannot see data left * behind by another task. */ - _init_msa_upper(); + init_msa_upper(); } else { /* We need to restore the vector context. */ restore_msa(current); diff --git a/kernel/arch/mips/kernel/unaligned.c b/kernel/arch/mips/kernel/unaligned.c index 490cea569..5c62065cb 100644 --- a/kernel/arch/mips/kernel/unaligned.c +++ b/kernel/arch/mips/kernel/unaligned.c @@ -885,7 +885,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, { union mips_instruction insn; unsigned long value; - unsigned int res; + unsigned int res, preempted; unsigned long origpc; unsigned long orig31; void __user *fault_addr = NULL; @@ -1226,27 +1226,36 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_READ, addr, sizeof(*fpr))) goto sigbus; - /* - * Disable preemption to avoid a race between copying - * state from userland, migrating to another CPU and - * updating the hardware vector register below. - */ - preempt_disable(); - - res = __copy_from_user_inatomic(fpr, addr, - sizeof(*fpr)); - if (res) - goto fault; - - /* - * Update the hardware register if it is in use by the - * task in this quantum, in order to avoid having to - * save & restore the whole vector context. - */ - if (test_thread_flag(TIF_USEDMSA)) - write_msa_wr(wd, fpr, df); + do { + /* + * If we have live MSA context keep track of + * whether we get preempted in order to avoid + * the register context we load being clobbered + * by the live context as it's saved during + * preemption. If we don't have live context + * then it can't be saved to clobber the value + * we load. + */ + preempted = test_thread_flag(TIF_USEDMSA); + + res = __copy_from_user_inatomic(fpr, addr, + sizeof(*fpr)); + if (res) + goto fault; - preempt_enable(); + /* + * Update the hardware register if it is in use + * by the task in this quantum, in order to + * avoid having to save & restore the whole + * vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) { + write_msa_wr(wd, fpr, df); + preempted = 0; + } + preempt_enable(); + } while (preempted); break; case msa_st_op: diff --git a/kernel/arch/mips/kernel/uprobes.c b/kernel/arch/mips/kernel/uprobes.c index 8452d933a..4e7b89f2e 100644 --- a/kernel/arch/mips/kernel/uprobes.c +++ b/kernel/arch/mips/kernel/uprobes.c @@ -157,7 +157,6 @@ bool is_trap_insn(uprobe_opcode_t *insn) int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) { struct uprobe_task *utask = current->utask; - union mips_instruction insn; /* * Now find the EPC where to resume after the breakpoint has been @@ -168,10 +167,10 @@ int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) unsigned long epc; epc = regs->cp0_epc; - __compute_return_epc_for_insn(regs, insn); + __compute_return_epc_for_insn(regs, + (union mips_instruction) aup->insn[0]); aup->resume_epc = regs->cp0_epc; } - utask->autask.saved_trap_nr = current->thread.trap_nr; current->thread.trap_nr = UPROBE_TRAP_NR; regs->cp0_epc = current->utask->xol_vaddr; @@ -257,7 +256,7 @@ unsigned long arch_uretprobe_hijack_return_addr( ra = regs->regs[31]; /* Replace the return address with the trampoline address */ - regs->regs[31] = ra; + regs->regs[31] = trampoline_vaddr; return ra; } @@ -280,24 +279,6 @@ int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN); } -/** - * set_orig_insn - Restore the original instruction. - * @mm: the probed process address space. - * @auprobe: arch specific probepoint information. - * @vaddr: the virtual address to insert the opcode. - * - * For mm @mm, restore the original opcode (opcode) at @vaddr. - * Return 0 (success) or a negative errno. - * - * This overrides the weak version in kernel/events/uprobes.c. - */ -int set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, - unsigned long vaddr) -{ - return uprobe_write_opcode(mm, vaddr, - *(uprobe_opcode_t *)&auprobe->orig_inst[0].word); -} - void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, void *src, unsigned long len) { diff --git a/kernel/arch/mips/kernel/vdso.c b/kernel/arch/mips/kernel/vdso.c index 975e99759..5649a9e42 100644 --- a/kernel/arch/mips/kernel/vdso.c +++ b/kernel/arch/mips/kernel/vdso.c @@ -39,16 +39,16 @@ static struct vm_special_mapping vdso_vvar_mapping = { static void __init init_vdso_image(struct mips_vdso_image *image) { unsigned long num_pages, i; + unsigned long data_pfn; BUG_ON(!PAGE_ALIGNED(image->data)); BUG_ON(!PAGE_ALIGNED(image->size)); num_pages = image->size / PAGE_SIZE; - for (i = 0; i < num_pages; i++) { - image->mapping.pages[i] = - virt_to_page(image->data + (i * PAGE_SIZE)); - } + data_pfn = __phys_to_pfn(__pa_symbol(image->data)); + for (i = 0; i < num_pages; i++) + image->mapping.pages[i] = pfn_to_page(data_pfn + i); } static int __init init_vdso(void) diff --git a/kernel/arch/mips/kernel/watch.c b/kernel/arch/mips/kernel/watch.c index 2a03abb5b..9b78e3751 100644 --- a/kernel/arch/mips/kernel/watch.c +++ b/kernel/arch/mips/kernel/watch.c @@ -15,10 +15,9 @@ * Install the watch registers for the current thread. A maximum of * four registers are installed although the machine may have more. */ -void mips_install_watch_registers(void) +void mips_install_watch_registers(struct task_struct *t) { - struct mips3264_watch_reg_state *watches = - ¤t->thread.watch.mips3264; + struct mips3264_watch_reg_state *watches = &t->thread.watch.mips3264; switch (current_cpu_data.watch_reg_use_cnt) { default: BUG(); diff --git a/kernel/arch/mips/kvm/emulate.c b/kernel/arch/mips/kvm/emulate.c index 41b1b090f..4c85ab808 100644 --- a/kernel/arch/mips/kvm/emulate.c +++ b/kernel/arch/mips/kvm/emulate.c @@ -302,12 +302,31 @@ static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu) */ static uint32_t kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now) { - ktime_t expires; + struct mips_coproc *cop0 = vcpu->arch.cop0; + ktime_t expires, threshold; + uint32_t count, compare; int running; - /* Is the hrtimer pending? */ + /* Calculate the biased and scaled guest CP0_Count */ + count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now); + compare = kvm_read_c0_guest_compare(cop0); + + /* + * Find whether CP0_Count has reached the closest timer interrupt. If + * not, we shouldn't inject it. + */ + if ((int32_t)(count - compare) < 0) + return count; + + /* + * The CP0_Count we're going to return has already reached the closest + * timer interrupt. Quickly check if it really is a new interrupt by + * looking at whether the interval until the hrtimer expiry time is + * less than 1/4 of the timer period. + */ expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer); - if (ktime_compare(now, expires) >= 0) { + threshold = ktime_add_ns(now, vcpu->arch.count_period / 4); + if (ktime_before(expires, threshold)) { /* * Cancel it while we handle it so there's no chance of * interference with the timeout handler. @@ -329,8 +348,7 @@ static uint32_t kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now) } } - /* Return the biased and scaled guest CP0_Count */ - return vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now); + return count; } /** @@ -419,32 +437,6 @@ static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu, hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS); } -/** - * kvm_mips_update_hrtimer() - Update next expiry time of hrtimer. - * @vcpu: Virtual CPU. - * - * Recalculates and updates the expiry time of the hrtimer. This can be used - * after timer parameters have been altered which do not depend on the time that - * the change occurs (in those cases kvm_mips_freeze_hrtimer() and - * kvm_mips_resume_hrtimer() are used directly). - * - * It is guaranteed that no timer interrupts will be lost in the process. - * - * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). - */ -static void kvm_mips_update_hrtimer(struct kvm_vcpu *vcpu) -{ - ktime_t now; - uint32_t count; - - /* - * freeze_hrtimer takes care of a timer interrupts <= count, and - * resume_hrtimer the hrtimer takes care of a timer interrupts > count. - */ - now = kvm_mips_freeze_hrtimer(vcpu, &count); - kvm_mips_resume_hrtimer(vcpu, now, count); -} - /** * kvm_mips_write_count() - Modify the count and update timer. * @vcpu: Virtual CPU. @@ -540,23 +532,42 @@ int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz) * kvm_mips_write_compare() - Modify compare and update timer. * @vcpu: Virtual CPU. * @compare: New CP0_Compare value. + * @ack: Whether to acknowledge timer interrupt. * * Update CP0_Compare to a new value and update the timeout. + * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure + * any pending timer interrupt is preserved. */ -void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare) +void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare, bool ack) { struct mips_coproc *cop0 = vcpu->arch.cop0; + int dc; + u32 old_compare = kvm_read_c0_guest_compare(cop0); + ktime_t now; + uint32_t count; /* if unchanged, must just be an ack */ - if (kvm_read_c0_guest_compare(cop0) == compare) + if (old_compare == compare) { + if (!ack) + return; + kvm_mips_callbacks->dequeue_timer_int(vcpu); + kvm_write_c0_guest_compare(cop0, compare); return; + } + + /* freeze_hrtimer() takes care of timer interrupts <= count */ + dc = kvm_mips_count_disabled(vcpu); + if (!dc) + now = kvm_mips_freeze_hrtimer(vcpu, &count); + + if (ack) + kvm_mips_callbacks->dequeue_timer_int(vcpu); - /* Update compare */ kvm_write_c0_guest_compare(cop0, compare); - /* Update timeout if count enabled */ - if (!kvm_mips_count_disabled(vcpu)) - kvm_mips_update_hrtimer(vcpu); + /* resume_hrtimer() takes care of timer interrupts > count */ + if (!dc) + kvm_mips_resume_hrtimer(vcpu, now, count); } /** @@ -741,15 +752,15 @@ enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu) struct mips_coproc *cop0 = vcpu->arch.cop0; enum emulation_result er = EMULATE_DONE; - if (kvm_read_c0_guest_status(cop0) & ST0_EXL) { + if (kvm_read_c0_guest_status(cop0) & ST0_ERL) { + kvm_clear_c0_guest_status(cop0, ST0_ERL); + vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0); + } else if (kvm_read_c0_guest_status(cop0) & ST0_EXL) { kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc, kvm_read_c0_guest_epc(cop0)); kvm_clear_c0_guest_status(cop0, ST0_EXL); vcpu->arch.pc = kvm_read_c0_guest_epc(cop0); - } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) { - kvm_clear_c0_guest_status(cop0, ST0_ERL); - vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0); } else { kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n", vcpu->arch.pc); @@ -796,6 +807,47 @@ enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu) return EMULATE_FAIL; } +/** + * kvm_mips_invalidate_guest_tlb() - Indicates a change in guest MMU map. + * @vcpu: VCPU with changed mappings. + * @tlb: TLB entry being removed. + * + * This is called to indicate a single change in guest MMU mappings, so that we + * can arrange TLB flushes on this and other CPUs. + */ +static void kvm_mips_invalidate_guest_tlb(struct kvm_vcpu *vcpu, + struct kvm_mips_tlb *tlb) +{ + int cpu, i; + bool user; + + /* No need to flush for entries which are already invalid */ + if (!((tlb->tlb_lo0 | tlb->tlb_lo1) & MIPS3_PG_V)) + return; + /* User address space doesn't need flushing for KSeg2/3 changes */ + user = tlb->tlb_hi < KVM_GUEST_KSEG0; + + preempt_disable(); + + /* + * Probe the shadow host TLB for the entry being overwritten, if one + * matches, invalidate it + */ + kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi); + + /* Invalidate the whole ASID on other CPUs */ + cpu = smp_processor_id(); + for_each_possible_cpu(i) { + if (i == cpu) + continue; + if (user) + vcpu->arch.guest_user_asid[i] = 0; + vcpu->arch.guest_kernel_asid[i] = 0; + } + + preempt_enable(); +} + /* Write Guest TLB Entry @ Index */ enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu) { @@ -815,11 +867,8 @@ enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu) } tlb = &vcpu->arch.guest_tlb[index]; - /* - * Probe the shadow host TLB for the entry being overwritten, if one - * matches, invalidate it - */ - kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi); + + kvm_mips_invalidate_guest_tlb(vcpu, tlb); tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0); tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0); @@ -848,11 +897,7 @@ enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu) tlb = &vcpu->arch.guest_tlb[index]; - /* - * Probe the shadow host TLB for the entry being overwritten, if one - * matches, invalidate it - */ - kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi); + kvm_mips_invalidate_guest_tlb(vcpu, tlb); tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0); tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0); @@ -971,6 +1016,7 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, int32_t rt, rd, copz, sel, co_bit, op; uint32_t pc = vcpu->arch.pc; unsigned long curr_pc; + int cpu, i; /* * Update PC and hold onto current PC in case there is @@ -1078,8 +1124,16 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, vcpu->arch.gprs[rt] & ASID_MASK); + preempt_disable(); /* Blow away the shadow host TLBs */ kvm_mips_flush_host_tlb(1); + cpu = smp_processor_id(); + for_each_possible_cpu(i) + if (i != cpu) { + vcpu->arch.guest_user_asid[i] = 0; + vcpu->arch.guest_kernel_asid[i] = 0; + } + preempt_enable(); } kvm_write_c0_guest_entryhi(cop0, vcpu->arch.gprs[rt]); @@ -1095,9 +1149,9 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, /* If we are writing to COMPARE */ /* Clear pending timer interrupt, if any */ - kvm_mips_callbacks->dequeue_timer_int(vcpu); kvm_mips_write_compare(vcpu, - vcpu->arch.gprs[rt]); + vcpu->arch.gprs[rt], + true); } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { unsigned int old_val, val, change; @@ -1419,6 +1473,7 @@ enum emulation_result kvm_mips_emulate_load(uint32_t inst, uint32_t cause, struct kvm_vcpu *vcpu) { enum emulation_result er = EMULATE_DO_MMIO; + unsigned long curr_pc; int32_t op, base, rt, offset; uint32_t bytes; @@ -1427,7 +1482,18 @@ enum emulation_result kvm_mips_emulate_load(uint32_t inst, uint32_t cause, offset = inst & 0xffff; op = (inst >> 26) & 0x3f; - vcpu->arch.pending_load_cause = cause; + /* + * Find the resume PC now while we have safe and easy access to the + * prior branch instruction, and save it for + * kvm_mips_complete_mmio_load() to restore later. + */ + curr_pc = vcpu->arch.pc; + er = update_pc(vcpu, cause); + if (er == EMULATE_FAIL) + return er; + vcpu->arch.io_pc = vcpu->arch.pc; + vcpu->arch.pc = curr_pc; + vcpu->arch.io_gpr = rt; switch (op) { @@ -1618,8 +1684,14 @@ enum emulation_result kvm_mips_emulate_cache(uint32_t inst, uint32_t *opc, preempt_disable(); if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) { - if (kvm_mips_host_tlb_lookup(vcpu, va) < 0) - kvm_mips_handle_kseg0_tlb_fault(va, vcpu); + if (kvm_mips_host_tlb_lookup(vcpu, va) < 0 && + kvm_mips_handle_kseg0_tlb_fault(va, vcpu)) { + kvm_err("%s: handling mapped kseg0 tlb fault for %lx, vcpu: %p, ASID: %#lx\n", + __func__, va, vcpu, read_c0_entryhi()); + er = EMULATE_FAIL; + preempt_enable(); + goto done; + } } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) || KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) { int index; @@ -1654,14 +1726,19 @@ enum emulation_result kvm_mips_emulate_cache(uint32_t inst, uint32_t *opc, run, vcpu); preempt_enable(); goto dont_update_pc; - } else { - /* - * We fault an entry from the guest tlb to the - * shadow host TLB - */ - kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, - NULL, - NULL); + } + /* + * We fault an entry from the guest tlb to the + * shadow host TLB + */ + if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, + NULL, NULL)) { + kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n", + __func__, va, index, vcpu, + read_c0_entryhi()); + er = EMULATE_FAIL; + preempt_enable(); + goto done; } } } else { @@ -2396,9 +2473,8 @@ enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, goto done; } - er = update_pc(vcpu, vcpu->arch.pending_load_cause); - if (er == EMULATE_FAIL) - return er; + /* Restore saved resume PC */ + vcpu->arch.pc = vcpu->arch.io_pc; switch (run->mmio.len) { case 4: @@ -2420,11 +2496,6 @@ enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, break; } - if (vcpu->arch.pending_load_cause & CAUSEF_BD) - kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n", - vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr, - vcpu->mmio_needed); - done: return er; } @@ -2622,8 +2693,13 @@ enum emulation_result kvm_mips_handle_tlbmiss(unsigned long cause, * OK we have a Guest TLB entry, now inject it into the * shadow host TLB */ - kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, NULL, - NULL); + if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, + NULL, NULL)) { + kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n", + __func__, va, index, vcpu, + read_c0_entryhi()); + er = EMULATE_FAIL; + } } } diff --git a/kernel/arch/mips/kvm/interrupt.h b/kernel/arch/mips/kvm/interrupt.h index 4ab4bdfad..214388470 100644 --- a/kernel/arch/mips/kvm/interrupt.h +++ b/kernel/arch/mips/kvm/interrupt.h @@ -28,6 +28,7 @@ #define MIPS_EXC_MAX 12 /* XXXSL More to follow */ +extern char __kvm_mips_vcpu_run_end[]; extern char mips32_exception[], mips32_exceptionEnd[]; extern char mips32_GuestException[], mips32_GuestExceptionEnd[]; diff --git a/kernel/arch/mips/kvm/locore.S b/kernel/arch/mips/kvm/locore.S index 7e2210846..777064336 100644 --- a/kernel/arch/mips/kvm/locore.S +++ b/kernel/arch/mips/kvm/locore.S @@ -227,6 +227,7 @@ FEXPORT(__kvm_mips_load_k0k1) /* Jump to guest */ eret +EXPORT(__kvm_mips_vcpu_run_end) VECTOR(MIPSX(exception), unknown) /* Find out what mode we came from and jump to the proper handler. */ diff --git a/kernel/arch/mips/kvm/mips.c b/kernel/arch/mips/kvm/mips.c index 8a113298b..8d4d92701 100644 --- a/kernel/arch/mips/kvm/mips.c +++ b/kernel/arch/mips/kvm/mips.c @@ -314,9 +314,18 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) memcpy(gebase + offset, mips32_GuestException, mips32_GuestExceptionEnd - mips32_GuestException); +#ifdef MODULE + offset += mips32_GuestExceptionEnd - mips32_GuestException; + memcpy(gebase + offset, (char *)__kvm_mips_vcpu_run, + __kvm_mips_vcpu_run_end - (char *)__kvm_mips_vcpu_run); + vcpu->arch.vcpu_run = gebase + offset; +#else + vcpu->arch.vcpu_run = __kvm_mips_vcpu_run; +#endif + /* Invalidate the icache for these ranges */ - local_flush_icache_range((unsigned long)gebase, - (unsigned long)gebase + ALIGN(size, PAGE_SIZE)); + flush_icache_range((unsigned long)gebase, + (unsigned long)gebase + ALIGN(size, PAGE_SIZE)); /* * Allocate comm page for guest kernel, a TLB will be reserved for @@ -403,7 +412,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) /* Disable hardware page table walking while in guest */ htw_stop(); - r = __kvm_mips_vcpu_run(run, vcpu); + r = vcpu->arch.vcpu_run(run, vcpu); /* Re-enable HTW before enabling interrupts */ htw_start(); diff --git a/kernel/arch/mips/kvm/tlb.c b/kernel/arch/mips/kvm/tlb.c index aed0ac2a4..eff71c75d 100644 --- a/kernel/arch/mips/kvm/tlb.c +++ b/kernel/arch/mips/kvm/tlb.c @@ -152,7 +152,7 @@ static int kvm_mips_map_page(struct kvm *kvm, gfn_t gfn) srcu_idx = srcu_read_lock(&kvm->srcu); pfn = kvm_mips_gfn_to_pfn(kvm, gfn); - if (kvm_mips_is_error_pfn(pfn)) { + if (is_error_noslot_pfn(pfn)) { kvm_err("Couldn't get pfn for gfn %#" PRIx64 "!\n", gfn); err = -EFAULT; goto out; @@ -276,7 +276,7 @@ int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr, } gfn = (KVM_GUEST_CPHYSADDR(badvaddr) >> PAGE_SHIFT); - if (gfn >= kvm->arch.guest_pmap_npages) { + if ((gfn | 1) >= kvm->arch.guest_pmap_npages) { kvm_err("%s: Invalid gfn: %#llx, BadVaddr: %#lx\n", __func__, gfn, badvaddr); kvm_mips_dump_host_tlbs(); @@ -361,25 +361,39 @@ int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu, unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0; struct kvm *kvm = vcpu->kvm; pfn_t pfn0, pfn1; - - if ((tlb->tlb_hi & VPN2_MASK) == 0) { - pfn0 = 0; - pfn1 = 0; - } else { - if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo0) - >> PAGE_SHIFT) < 0) - return -1; - - if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo1) - >> PAGE_SHIFT) < 0) - return -1; - - pfn0 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo0) - >> PAGE_SHIFT]; - pfn1 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo1) - >> PAGE_SHIFT]; + gfn_t gfn0, gfn1; + long tlb_lo[2]; + + tlb_lo[0] = tlb->tlb_lo0; + tlb_lo[1] = tlb->tlb_lo1; + + /* + * The commpage address must not be mapped to anything else if the guest + * TLB contains entries nearby, or commpage accesses will break. + */ + if (!((tlb->tlb_hi ^ KVM_GUEST_COMMPAGE_ADDR) & + VPN2_MASK & (PAGE_MASK << 1))) + tlb_lo[(KVM_GUEST_COMMPAGE_ADDR >> PAGE_SHIFT) & 1] = 0; + + gfn0 = mips3_tlbpfn_to_paddr(tlb_lo[0]) >> PAGE_SHIFT; + gfn1 = mips3_tlbpfn_to_paddr(tlb_lo[1]) >> PAGE_SHIFT; + if (gfn0 >= kvm->arch.guest_pmap_npages || + gfn1 >= kvm->arch.guest_pmap_npages) { + kvm_err("%s: Invalid gfn: [%#llx, %#llx], EHi: %#lx\n", + __func__, gfn0, gfn1, tlb->tlb_hi); + kvm_mips_dump_guest_tlbs(vcpu); + return -1; } + if (kvm_mips_map_page(kvm, gfn0) < 0) + return -1; + + if (kvm_mips_map_page(kvm, gfn1) < 0) + return -1; + + pfn0 = kvm->arch.guest_pmap[gfn0]; + pfn1 = kvm->arch.guest_pmap[gfn1]; + if (hpa0) *hpa0 = pfn0 << PAGE_SHIFT; @@ -391,9 +405,9 @@ int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu, kvm_mips_get_kernel_asid(vcpu) : kvm_mips_get_user_asid(vcpu)); entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | - (tlb->tlb_lo0 & MIPS3_PG_D) | (tlb->tlb_lo0 & MIPS3_PG_V); + (tlb_lo[0] & MIPS3_PG_D) | (tlb_lo[0] & MIPS3_PG_V); entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | - (tlb->tlb_lo1 & MIPS3_PG_D) | (tlb->tlb_lo1 & MIPS3_PG_V); + (tlb_lo[1] & MIPS3_PG_D) | (tlb_lo[1] & MIPS3_PG_V); kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc, tlb->tlb_lo0, tlb->tlb_lo1); @@ -794,10 +808,16 @@ uint32_t kvm_get_inst(uint32_t *opc, struct kvm_vcpu *vcpu) local_irq_restore(flags); return KVM_INVALID_INST; } - kvm_mips_handle_mapped_seg_tlb_fault(vcpu, - &vcpu->arch. - guest_tlb[index], - NULL, NULL); + if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, + &vcpu->arch.guest_tlb[index], + NULL, NULL)) { + kvm_err("%s: handling mapped seg tlb fault failed for %p, index: %u, vcpu: %p, ASID: %#lx\n", + __func__, opc, index, vcpu, + read_c0_entryhi()); + kvm_mips_dump_guest_tlbs(vcpu); + local_irq_restore(flags); + return KVM_INVALID_INST; + } inst = *(opc); } local_irq_restore(flags); diff --git a/kernel/arch/mips/kvm/trap_emul.c b/kernel/arch/mips/kvm/trap_emul.c index d836ed5b0..307cc4c98 100644 --- a/kernel/arch/mips/kvm/trap_emul.c +++ b/kernel/arch/mips/kvm/trap_emul.c @@ -547,7 +547,7 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu, kvm_mips_write_count(vcpu, v); break; case KVM_REG_MIPS_CP0_COMPARE: - kvm_mips_write_compare(vcpu, v); + kvm_mips_write_compare(vcpu, v, false); break; case KVM_REG_MIPS_CP0_CAUSE: /* diff --git a/kernel/arch/mips/lib/ashldi3.c b/kernel/arch/mips/lib/ashldi3.c index beb80f316..927dc94a0 100644 --- a/kernel/arch/mips/lib/ashldi3.c +++ b/kernel/arch/mips/lib/ashldi3.c @@ -2,7 +2,7 @@ #include "libgcc.h" -long long __ashldi3(long long u, word_type b) +long long notrace __ashldi3(long long u, word_type b) { DWunion uu, w; word_type bm; diff --git a/kernel/arch/mips/lib/ashrdi3.c b/kernel/arch/mips/lib/ashrdi3.c index c884a912b..9fdf1a598 100644 --- a/kernel/arch/mips/lib/ashrdi3.c +++ b/kernel/arch/mips/lib/ashrdi3.c @@ -2,7 +2,7 @@ #include "libgcc.h" -long long __ashrdi3(long long u, word_type b) +long long notrace __ashrdi3(long long u, word_type b) { DWunion uu, w; word_type bm; diff --git a/kernel/arch/mips/lib/bswapdi.c b/kernel/arch/mips/lib/bswapdi.c index 77e5f9c1f..e3e77aa52 100644 --- a/kernel/arch/mips/lib/bswapdi.c +++ b/kernel/arch/mips/lib/bswapdi.c @@ -1,6 +1,6 @@ #include -unsigned long long __bswapdi2(unsigned long long u) +unsigned long long notrace __bswapdi2(unsigned long long u) { return (((u) & 0xff00000000000000ull) >> 56) | (((u) & 0x00ff000000000000ull) >> 40) | diff --git a/kernel/arch/mips/lib/bswapsi.c b/kernel/arch/mips/lib/bswapsi.c index 2b302ff12..530a8afe6 100644 --- a/kernel/arch/mips/lib/bswapsi.c +++ b/kernel/arch/mips/lib/bswapsi.c @@ -1,6 +1,6 @@ #include -unsigned int __bswapsi2(unsigned int u) +unsigned int notrace __bswapsi2(unsigned int u) { return (((u) & 0xff000000) >> 24) | (((u) & 0x00ff0000) >> 8) | diff --git a/kernel/arch/mips/lib/cmpdi2.c b/kernel/arch/mips/lib/cmpdi2.c index 8c1306437..06857da96 100644 --- a/kernel/arch/mips/lib/cmpdi2.c +++ b/kernel/arch/mips/lib/cmpdi2.c @@ -2,7 +2,7 @@ #include "libgcc.h" -word_type __cmpdi2(long long a, long long b) +word_type notrace __cmpdi2(long long a, long long b) { const DWunion au = { .ll = a diff --git a/kernel/arch/mips/lib/lshrdi3.c b/kernel/arch/mips/lib/lshrdi3.c index dcf8d6810..364547449 100644 --- a/kernel/arch/mips/lib/lshrdi3.c +++ b/kernel/arch/mips/lib/lshrdi3.c @@ -2,7 +2,7 @@ #include "libgcc.h" -long long __lshrdi3(long long u, word_type b) +long long notrace __lshrdi3(long long u, word_type b) { DWunion uu, w; word_type bm; diff --git a/kernel/arch/mips/lib/ucmpdi2.c b/kernel/arch/mips/lib/ucmpdi2.c index bb4cb2f82..bd599f582 100644 --- a/kernel/arch/mips/lib/ucmpdi2.c +++ b/kernel/arch/mips/lib/ucmpdi2.c @@ -2,7 +2,7 @@ #include "libgcc.h" -word_type __ucmpdi2(unsigned long long a, unsigned long long b) +word_type notrace __ucmpdi2(unsigned long long a, unsigned long long b) { const DWunion au = {.ll = a}; const DWunion bu = {.ll = b}; diff --git a/kernel/arch/mips/loongson64/loongson-3/hpet.c b/kernel/arch/mips/loongson64/loongson-3/hpet.c index a2631a52c..444802e78 100644 --- a/kernel/arch/mips/loongson64/loongson-3/hpet.c +++ b/kernel/arch/mips/loongson64/loongson-3/hpet.c @@ -13,8 +13,8 @@ #define SMBUS_PCI_REG64 0x64 #define SMBUS_PCI_REGB4 0xb4 -#define HPET_MIN_CYCLES 64 -#define HPET_MIN_PROG_DELTA (HPET_MIN_CYCLES + (HPET_MIN_CYCLES >> 1)) +#define HPET_MIN_CYCLES 16 +#define HPET_MIN_PROG_DELTA (HPET_MIN_CYCLES * 12) static DEFINE_SPINLOCK(hpet_lock); DEFINE_PER_CPU(struct clock_event_device, hpet_clockevent_device); @@ -157,14 +157,14 @@ static int hpet_tick_resume(struct clock_event_device *evt) static int hpet_next_event(unsigned long delta, struct clock_event_device *evt) { - unsigned int cnt; - int res; + u32 cnt; + s32 res; cnt = hpet_read(HPET_COUNTER); - cnt += delta; + cnt += (u32) delta; hpet_write(HPET_T0_CMP, cnt); - res = (int)(cnt - hpet_read(HPET_COUNTER)); + res = (s32)(cnt - hpet_read(HPET_COUNTER)); return res < HPET_MIN_CYCLES ? -ETIME : 0; } @@ -230,7 +230,7 @@ void __init setup_hpet_timer(void) cd = &per_cpu(hpet_clockevent_device, cpu); cd->name = "hpet"; - cd->rating = 320; + cd->rating = 100; cd->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; cd->set_state_shutdown = hpet_set_state_shutdown; cd->set_state_periodic = hpet_set_state_periodic; diff --git a/kernel/arch/mips/loongson64/loongson-3/numa.c b/kernel/arch/mips/loongson64/loongson-3/numa.c index 6f9e010ce..282c5a8c2 100644 --- a/kernel/arch/mips/loongson64/loongson-3/numa.c +++ b/kernel/arch/mips/loongson64/loongson-3/numa.c @@ -213,10 +213,10 @@ static void __init node_mem_init(unsigned int node) BOOTMEM_DEFAULT); if (node == 0 && node_end_pfn(0) >= (0xffffffff >> PAGE_SHIFT)) { - /* Reserve 0xff800000~0xffffffff for RS780E integrated GPU */ + /* Reserve 0xfe000000~0xffffffff for RS780E integrated GPU */ reserve_bootmem_node(NODE_DATA(node), - (node_addrspace_offset | 0xff800000), - 8 << 20, BOOTMEM_DEFAULT); + (node_addrspace_offset | 0xfe000000), + 32 << 20, BOOTMEM_DEFAULT); } sparse_memory_present_with_active_regions(node); diff --git a/kernel/arch/mips/math-emu/cp1emu.c b/kernel/arch/mips/math-emu/cp1emu.c index 32f0e19a0..734a2c766 100644 --- a/kernel/arch/mips/math-emu/cp1emu.c +++ b/kernel/arch/mips/math-emu/cp1emu.c @@ -445,9 +445,11 @@ static int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case spec_op: switch (insn.r_format.func) { case jalr_op: - regs->regs[insn.r_format.rd] = - regs->cp0_epc + dec_insn.pc_inc + - dec_insn.next_pc_inc; + if (insn.r_format.rd != 0) { + regs->regs[insn.r_format.rd] = + regs->cp0_epc + dec_insn.pc_inc + + dec_insn.next_pc_inc; + } /* Fall through */ case jr_op: /* For R6, JR already emulated in jalr_op */ diff --git a/kernel/arch/mips/mm/cache.c b/kernel/arch/mips/mm/cache.c index aab218c36..e87bccd6e 100644 --- a/kernel/arch/mips/mm/cache.c +++ b/kernel/arch/mips/mm/cache.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -83,8 +84,6 @@ void __flush_dcache_page(struct page *page) struct address_space *mapping = page_mapping(page); unsigned long addr; - if (PageHighMem(page)) - return; if (mapping && !mapping_mapped(mapping)) { SetPageDcacheDirty(page); return; @@ -95,8 +94,15 @@ void __flush_dcache_page(struct page *page) * case is for exec env/arg pages and those are %99 certainly going to * get faulted into the tlb (and thus flushed) anyways. */ - addr = (unsigned long) page_address(page); + if (PageHighMem(page)) + addr = (unsigned long)kmap_atomic(page); + else + addr = (unsigned long)page_address(page); + flush_data_cache_page(addr); + + if (PageHighMem(page)) + __kunmap_atomic((void *)addr); } EXPORT_SYMBOL(__flush_dcache_page); @@ -119,33 +125,28 @@ void __flush_anon_page(struct page *page, unsigned long vmaddr) EXPORT_SYMBOL(__flush_anon_page); -void __flush_icache_page(struct vm_area_struct *vma, struct page *page) -{ - unsigned long addr; - - if (PageHighMem(page)) - return; - - addr = (unsigned long) page_address(page); - flush_data_cache_page(addr); -} -EXPORT_SYMBOL_GPL(__flush_icache_page); - -void __update_cache(struct vm_area_struct *vma, unsigned long address, - pte_t pte) +void __update_cache(unsigned long address, pte_t pte) { struct page *page; unsigned long pfn, addr; - int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc; + int exec = !pte_no_exec(pte) && !cpu_has_ic_fills_f_dc; pfn = pte_pfn(pte); if (unlikely(!pfn_valid(pfn))) return; page = pfn_to_page(pfn); - if (page_mapping(page) && Page_dcache_dirty(page)) { - addr = (unsigned long) page_address(page); + if (Page_dcache_dirty(page)) { + if (PageHighMem(page)) + addr = (unsigned long)kmap_atomic(page); + else + addr = (unsigned long)page_address(page); + if (exec || pages_do_alias(addr, address & PAGE_MASK)) flush_data_cache_page(addr); + + if (PageHighMem(page)) + __kunmap_atomic((void *)addr); + ClearPageDcacheDirty(page); } } diff --git a/kernel/arch/mips/mm/uasm-mips.c b/kernel/arch/mips/mm/uasm-mips.c index b4a837893..5abe51cad 100644 --- a/kernel/arch/mips/mm/uasm-mips.c +++ b/kernel/arch/mips/mm/uasm-mips.c @@ -65,7 +65,7 @@ static struct insn insn_table[] = { #ifndef CONFIG_CPU_MIPSR6 { insn_cache, M(cache_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, #else - { insn_cache, M6(cache_op, 0, 0, 0, cache6_op), RS | RT | SIMM9 }, + { insn_cache, M6(spec3_op, 0, 0, 0, cache6_op), RS | RT | SIMM9 }, #endif { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD }, diff --git a/kernel/arch/mips/mti-malta/malta-setup.c b/kernel/arch/mips/mti-malta/malta-setup.c index 4740c82fb..36b09b2ea 100644 --- a/kernel/arch/mips/mti-malta/malta-setup.c +++ b/kernel/arch/mips/mti-malta/malta-setup.c @@ -39,6 +39,9 @@ #include #endif +#define ROCIT_CONFIG_GEN0 0x1f403000 +#define ROCIT_CONFIG_GEN0_PCI_IOCU BIT(7) + extern void malta_be_init(void); extern int malta_be_handler(struct pt_regs *regs, int is_fixup); @@ -107,6 +110,8 @@ static void __init fd_activate(void) static int __init plat_enable_iocoherency(void) { int supported = 0; + u32 cfg; + if (mips_revision_sconid == MIPS_REVISION_SCON_BONITO) { if (BONITO_PCICACHECTRL & BONITO_PCICACHECTRL_CPUCOH_PRES) { BONITO_PCICACHECTRL |= BONITO_PCICACHECTRL_CPUCOH_EN; @@ -129,7 +134,8 @@ static int __init plat_enable_iocoherency(void) } else if (mips_cm_numiocu() != 0) { /* Nothing special needs to be done to enable coherency */ pr_info("CMP IOCU detected\n"); - if ((*(unsigned int *)0xbf403000 & 0x81) != 0x81) { + cfg = __raw_readl((u32 *)CKSEG1ADDR(ROCIT_CONFIG_GEN0)); + if (!(cfg & ROCIT_CONFIG_GEN0_PCI_IOCU)) { pr_crit("IOCU OPERATION DISABLED BY SWITCH - DEFAULTING TO SW IO COHERENCY\n"); return 0; } diff --git a/kernel/arch/mips/vdso/Makefile b/kernel/arch/mips/vdso/Makefile index 14568900f..6c7d78546 100644 --- a/kernel/arch/mips/vdso/Makefile +++ b/kernel/arch/mips/vdso/Makefile @@ -5,10 +5,12 @@ obj-vdso-y := elf.o gettimeofday.o sigreturn.o ccflags-vdso := \ $(filter -I%,$(KBUILD_CFLAGS)) \ $(filter -E%,$(KBUILD_CFLAGS)) \ + $(filter -mmicromips,$(KBUILD_CFLAGS)) \ $(filter -march=%,$(KBUILD_CFLAGS)) cflags-vdso := $(ccflags-vdso) \ $(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \ - -O2 -g -fPIC -fno-common -fno-builtin -G 0 -DDISABLE_BRANCH_PROFILING \ + -O2 -g -fPIC -fno-strict-aliasing -fno-common -fno-builtin -G 0 \ + -DDISABLE_BRANCH_PROFILING \ $(call cc-option, -fno-stack-protector) aflags-vdso := $(ccflags-vdso) \ $(filter -I%,$(KBUILD_CFLAGS)) \ @@ -73,7 +75,7 @@ obj-vdso := $(obj-vdso-y:%.o=$(obj)/%.o) $(obj-vdso): KBUILD_CFLAGS := $(cflags-vdso) $(native-abi) $(obj-vdso): KBUILD_AFLAGS := $(aflags-vdso) $(native-abi) -$(obj)/vdso.lds: KBUILD_CPPFLAGS := $(native-abi) +$(obj)/vdso.lds: KBUILD_CPPFLAGS := $(ccflags-vdso) $(native-abi) $(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE $(call if_changed,vdsold) diff --git a/kernel/arch/mn10300/include/asm/uaccess.h b/kernel/arch/mn10300/include/asm/uaccess.h index 537278746..4af43d9ba 100644 --- a/kernel/arch/mn10300/include/asm/uaccess.h +++ b/kernel/arch/mn10300/include/asm/uaccess.h @@ -181,6 +181,7 @@ struct __large_struct { unsigned long buf[100]; }; "2:\n" \ " .section .fixup,\"ax\"\n" \ "3:\n\t" \ + " mov 0,%1\n" \ " mov %3,%0\n" \ " jmp 2b\n" \ " .previous\n" \ diff --git a/kernel/arch/mn10300/lib/usercopy.c b/kernel/arch/mn10300/lib/usercopy.c index 7826e6c36..ce8899e5e 100644 --- a/kernel/arch/mn10300/lib/usercopy.c +++ b/kernel/arch/mn10300/lib/usercopy.c @@ -9,7 +9,7 @@ * as published by the Free Software Foundation; either version * 2 of the Licence, or (at your option) any later version. */ -#include +#include unsigned long __generic_copy_to_user(void *to, const void *from, unsigned long n) @@ -24,6 +24,8 @@ __generic_copy_from_user(void *to, const void *from, unsigned long n) { if (access_ok(VERIFY_READ, from, n)) __copy_user_zeroing(to, from, n); + else + memset(to, 0, n); return n; } diff --git a/kernel/arch/nios2/include/asm/uaccess.h b/kernel/arch/nios2/include/asm/uaccess.h index caa51ff85..0ab82324c 100644 --- a/kernel/arch/nios2/include/asm/uaccess.h +++ b/kernel/arch/nios2/include/asm/uaccess.h @@ -102,9 +102,12 @@ extern long __copy_to_user(void __user *to, const void *from, unsigned long n); static inline long copy_from_user(void *to, const void __user *from, unsigned long n) { - if (!access_ok(VERIFY_READ, from, n)) - return n; - return __copy_from_user(to, from, n); + unsigned long res = n; + if (access_ok(VERIFY_READ, from, n)) + res = __copy_from_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; } static inline long copy_to_user(void __user *to, const void *from, @@ -139,7 +142,7 @@ extern long strnlen_user(const char __user *s, long n); #define __get_user_unknown(val, size, ptr, err) do { \ err = 0; \ - if (copy_from_user(&(val), ptr, size)) { \ + if (__copy_from_user(&(val), ptr, size)) { \ err = -EFAULT; \ } \ } while (0) @@ -166,7 +169,7 @@ do { \ ({ \ long __gu_err = -EFAULT; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ - unsigned long __gu_val; \ + unsigned long __gu_val = 0; \ __get_user_common(__gu_val, sizeof(*(ptr)), __gu_ptr, __gu_err);\ (x) = (__force __typeof__(x))__gu_val; \ __gu_err; \ diff --git a/kernel/arch/openrisc/include/asm/uaccess.h b/kernel/arch/openrisc/include/asm/uaccess.h index a6bd07ca3..5cc6b4f1b 100644 --- a/kernel/arch/openrisc/include/asm/uaccess.h +++ b/kernel/arch/openrisc/include/asm/uaccess.h @@ -273,28 +273,20 @@ __copy_tofrom_user(void *to, const void *from, unsigned long size); static inline unsigned long copy_from_user(void *to, const void *from, unsigned long n) { - unsigned long over; - - if (access_ok(VERIFY_READ, from, n)) - return __copy_tofrom_user(to, from, n); - if ((unsigned long)from < TASK_SIZE) { - over = (unsigned long)from + n - TASK_SIZE; - return __copy_tofrom_user(to, from, n - over) + over; - } - return n; + unsigned long res = n; + + if (likely(access_ok(VERIFY_READ, from, n))) + res = __copy_tofrom_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; } static inline unsigned long copy_to_user(void *to, const void *from, unsigned long n) { - unsigned long over; - - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_tofrom_user(to, from, n); - if ((unsigned long)to < TASK_SIZE) { - over = (unsigned long)to + n - TASK_SIZE; - return __copy_tofrom_user(to, from, n - over) + over; - } + if (likely(access_ok(VERIFY_WRITE, to, n))) + n = __copy_tofrom_user(to, from, n); return n; } @@ -303,13 +295,8 @@ extern unsigned long __clear_user(void *addr, unsigned long size); static inline __must_check unsigned long clear_user(void *addr, unsigned long size) { - - if (access_ok(VERIFY_WRITE, addr, size)) - return __clear_user(addr, size); - if ((unsigned long)addr < TASK_SIZE) { - unsigned long over = (unsigned long)addr + size - TASK_SIZE; - return __clear_user(addr, size - over) + over; - } + if (likely(access_ok(VERIFY_WRITE, addr, size))) + size = __clear_user(addr, size); return size; } diff --git a/kernel/arch/parisc/include/asm/bitops.h b/kernel/arch/parisc/include/asm/bitops.h index 3f9406d9b..da8794332 100644 --- a/kernel/arch/parisc/include/asm/bitops.h +++ b/kernel/arch/parisc/include/asm/bitops.h @@ -6,7 +6,7 @@ #endif #include -#include /* for BITS_PER_LONG/SHIFT_PER_LONG */ +#include #include #include #include @@ -17,6 +17,12 @@ * to include/asm-i386/bitops.h or kerneldoc */ +#if __BITS_PER_LONG == 64 +#define SHIFT_PER_LONG 6 +#else +#define SHIFT_PER_LONG 5 +#endif + #define CHOP_SHIFTCOUNT(x) (((unsigned long) (x)) & (BITS_PER_LONG - 1)) diff --git a/kernel/arch/parisc/include/asm/pgtable.h b/kernel/arch/parisc/include/asm/pgtable.h index 291cee28c..3a4ed9f91 100644 --- a/kernel/arch/parisc/include/asm/pgtable.h +++ b/kernel/arch/parisc/include/asm/pgtable.h @@ -65,9 +65,9 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) unsigned long flags; \ spin_lock_irqsave(&pa_tlb_lock, flags); \ old_pte = *ptep; \ - set_pte(ptep, pteval); \ if (pte_inserted(old_pte)) \ purge_tlb_entries(mm, addr); \ + set_pte(ptep, pteval); \ spin_unlock_irqrestore(&pa_tlb_lock, flags); \ } while (0) @@ -83,10 +83,10 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e)) /* This is the size of the initially mapped kernel memory */ -#ifdef CONFIG_64BIT -#define KERNEL_INITIAL_ORDER 25 /* 1<<25 = 32MB */ +#if defined(CONFIG_64BIT) +#define KERNEL_INITIAL_ORDER 26 /* 1<<26 = 64MB */ #else -#define KERNEL_INITIAL_ORDER 24 /* 1<<24 = 16MB */ +#define KERNEL_INITIAL_ORDER 25 /* 1<<25 = 32MB */ #endif #define KERNEL_INITIAL_SIZE (1 << KERNEL_INITIAL_ORDER) @@ -478,8 +478,8 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned spin_unlock_irqrestore(&pa_tlb_lock, flags); return 0; } - set_pte(ptep, pte_mkold(pte)); purge_tlb_entries(vma->vm_mm, addr); + set_pte(ptep, pte_mkold(pte)); spin_unlock_irqrestore(&pa_tlb_lock, flags); return 1; } @@ -492,9 +492,9 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, spin_lock_irqsave(&pa_tlb_lock, flags); old_pte = *ptep; - set_pte(ptep, __pte(0)); if (pte_inserted(old_pte)) purge_tlb_entries(mm, addr); + set_pte(ptep, __pte(0)); spin_unlock_irqrestore(&pa_tlb_lock, flags); return old_pte; @@ -504,8 +504,8 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, { unsigned long flags; spin_lock_irqsave(&pa_tlb_lock, flags); - set_pte(ptep, pte_wrprotect(*ptep)); purge_tlb_entries(mm, addr); + set_pte(ptep, pte_wrprotect(*ptep)); spin_unlock_irqrestore(&pa_tlb_lock, flags); } diff --git a/kernel/arch/parisc/include/asm/uaccess.h b/kernel/arch/parisc/include/asm/uaccess.h index 0abdd4c60..4ad514658 100644 --- a/kernel/arch/parisc/include/asm/uaccess.h +++ b/kernel/arch/parisc/include/asm/uaccess.h @@ -10,6 +10,7 @@ #include #include +#include #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -76,6 +77,7 @@ struct exception_table_entry { */ struct exception_data { unsigned long fault_ip; + unsigned long fault_gp; unsigned long fault_space; unsigned long fault_addr; }; @@ -244,13 +246,14 @@ static inline unsigned long __must_check copy_from_user(void *to, unsigned long n) { int sz = __compiletime_object_size(to); - int ret = -EFAULT; + unsigned long ret = n; if (likely(sz == -1 || !__builtin_constant_p(n) || sz >= n)) ret = __copy_from_user(to, from, n); else copy_from_user_overflow(); - + if (unlikely(ret)) + memset(to + (n - ret), 0, ret); return ret; } diff --git a/kernel/arch/parisc/include/uapi/asm/bitsperlong.h b/kernel/arch/parisc/include/uapi/asm/bitsperlong.h index e0a23c7bd..07fa7e50b 100644 --- a/kernel/arch/parisc/include/uapi/asm/bitsperlong.h +++ b/kernel/arch/parisc/include/uapi/asm/bitsperlong.h @@ -3,10 +3,8 @@ #if defined(__LP64__) #define __BITS_PER_LONG 64 -#define SHIFT_PER_LONG 6 #else #define __BITS_PER_LONG 32 -#define SHIFT_PER_LONG 5 #endif #include diff --git a/kernel/arch/parisc/include/uapi/asm/errno.h b/kernel/arch/parisc/include/uapi/asm/errno.h index c0ae62520..274d5bc6e 100644 --- a/kernel/arch/parisc/include/uapi/asm/errno.h +++ b/kernel/arch/parisc/include/uapi/asm/errno.h @@ -97,10 +97,10 @@ #define ENOTCONN 235 /* Transport endpoint is not connected */ #define ESHUTDOWN 236 /* Cannot send after transport endpoint shutdown */ #define ETOOMANYREFS 237 /* Too many references: cannot splice */ -#define EREFUSED ECONNREFUSED /* for HP's NFS apparently */ #define ETIMEDOUT 238 /* Connection timed out */ #define ECONNREFUSED 239 /* Connection refused */ -#define EREMOTERELEASE 240 /* Remote peer released connection */ +#define EREFUSED ECONNREFUSED /* for HP's NFS apparently */ +#define EREMOTERELEASE 240 /* Remote peer released connection */ #define EHOSTDOWN 241 /* Host is down */ #define EHOSTUNREACH 242 /* No route to host */ diff --git a/kernel/arch/parisc/include/uapi/asm/swab.h b/kernel/arch/parisc/include/uapi/asm/swab.h index e78403b12..928e1bbac 100644 --- a/kernel/arch/parisc/include/uapi/asm/swab.h +++ b/kernel/arch/parisc/include/uapi/asm/swab.h @@ -1,6 +1,7 @@ #ifndef _PARISC_SWAB_H #define _PARISC_SWAB_H +#include #include #include @@ -38,7 +39,7 @@ static inline __attribute_const__ __u32 __arch_swab32(__u32 x) } #define __arch_swab32 __arch_swab32 -#if BITS_PER_LONG > 32 +#if __BITS_PER_LONG > 32 /* ** From "PA-RISC 2.0 Architecture", HP Professional Books. ** See Appendix I page 8 , "Endian Byte Swapping". @@ -61,6 +62,6 @@ static inline __attribute_const__ __u64 __arch_swab64(__u64 x) return x; } #define __arch_swab64 __arch_swab64 -#endif /* BITS_PER_LONG > 32 */ +#endif /* __BITS_PER_LONG > 32 */ #endif /* _PARISC_SWAB_H */ diff --git a/kernel/arch/parisc/kernel/asm-offsets.c b/kernel/arch/parisc/kernel/asm-offsets.c index d2f62570a..78d30d2ea 100644 --- a/kernel/arch/parisc/kernel/asm-offsets.c +++ b/kernel/arch/parisc/kernel/asm-offsets.c @@ -299,6 +299,7 @@ int main(void) #endif BLANK(); DEFINE(EXCDATA_IP, offsetof(struct exception_data, fault_ip)); + DEFINE(EXCDATA_GP, offsetof(struct exception_data, fault_gp)); DEFINE(EXCDATA_SPACE, offsetof(struct exception_data, fault_space)); DEFINE(EXCDATA_ADDR, offsetof(struct exception_data, fault_addr)); BLANK(); diff --git a/kernel/arch/parisc/kernel/cache.c b/kernel/arch/parisc/kernel/cache.c index cda6dbbe9..6857a104b 100644 --- a/kernel/arch/parisc/kernel/cache.c +++ b/kernel/arch/parisc/kernel/cache.c @@ -351,6 +351,7 @@ void __init parisc_setup_cache_timing(void) { unsigned long rangetime, alltime; unsigned long size, start; + unsigned long threshold; alltime = mfctl(16); flush_data_cache(); @@ -364,26 +365,30 @@ void __init parisc_setup_cache_timing(void) printk(KERN_DEBUG "Whole cache flush %lu cycles, flushing %lu bytes %lu cycles\n", alltime, size, rangetime); - /* Racy, but if we see an intermediate value, it's ok too... */ - parisc_cache_flush_threshold = size * alltime / rangetime; - - parisc_cache_flush_threshold = L1_CACHE_ALIGN(parisc_cache_flush_threshold); - if (!parisc_cache_flush_threshold) - parisc_cache_flush_threshold = FLUSH_THRESHOLD; - - if (parisc_cache_flush_threshold > cache_info.dc_size) - parisc_cache_flush_threshold = cache_info.dc_size; - - printk(KERN_INFO "Setting cache flush threshold to %lu kB\n", + threshold = L1_CACHE_ALIGN(size * alltime / rangetime); + if (threshold > cache_info.dc_size) + threshold = cache_info.dc_size; + if (threshold) + parisc_cache_flush_threshold = threshold; + printk(KERN_INFO "Cache flush threshold set to %lu KiB\n", parisc_cache_flush_threshold/1024); /* calculate TLB flush threshold */ + /* On SMP machines, skip the TLB measure of kernel text which + * has been mapped as huge pages. */ + if (num_online_cpus() > 1 && !parisc_requires_coherency()) { + threshold = max(cache_info.it_size, cache_info.dt_size); + threshold *= PAGE_SIZE; + threshold /= num_online_cpus(); + goto set_tlb_threshold; + } + alltime = mfctl(16); flush_tlb_all(); alltime = mfctl(16) - alltime; - size = PAGE_SIZE; + size = 0; start = (unsigned long) _text; rangetime = mfctl(16); while (start < (unsigned long) _end) { @@ -396,13 +401,12 @@ void __init parisc_setup_cache_timing(void) printk(KERN_DEBUG "Whole TLB flush %lu cycles, flushing %lu bytes %lu cycles\n", alltime, size, rangetime); - parisc_tlb_flush_threshold = size * alltime / rangetime; - parisc_tlb_flush_threshold *= num_online_cpus(); - parisc_tlb_flush_threshold = PAGE_ALIGN(parisc_tlb_flush_threshold); - if (!parisc_tlb_flush_threshold) - parisc_tlb_flush_threshold = FLUSH_TLB_THRESHOLD; + threshold = PAGE_ALIGN(num_online_cpus() * size * alltime / rangetime); - printk(KERN_INFO "Setting TLB flush threshold to %lu kB\n", +set_tlb_threshold: + if (threshold) + parisc_tlb_flush_threshold = threshold; + printk(KERN_INFO "TLB flush threshold set to %lu KiB\n", parisc_tlb_flush_threshold/1024); } diff --git a/kernel/arch/parisc/kernel/pacache.S b/kernel/arch/parisc/kernel/pacache.S index b743a80ea..a4761b772 100644 --- a/kernel/arch/parisc/kernel/pacache.S +++ b/kernel/arch/parisc/kernel/pacache.S @@ -96,7 +96,7 @@ fitmanyloop: /* Loop if LOOP >= 2 */ fitmanymiddle: /* Loop if LOOP >= 2 */ addib,COND(>) -1, %r31, fitmanymiddle /* Adjusted inner loop decr */ - pitlbe 0(%sr1, %r28) + pitlbe %r0(%sr1, %r28) pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */ addib,COND(>) -1, %r29, fitmanymiddle /* Middle loop decr */ copy %arg3, %r31 /* Re-init inner loop count */ @@ -139,7 +139,7 @@ fdtmanyloop: /* Loop if LOOP >= 2 */ fdtmanymiddle: /* Loop if LOOP >= 2 */ addib,COND(>) -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */ - pdtlbe 0(%sr1, %r28) + pdtlbe %r0(%sr1, %r28) pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */ addib,COND(>) -1, %r29, fdtmanymiddle /* Middle loop decr */ copy %arg3, %r31 /* Re-init inner loop count */ @@ -620,12 +620,12 @@ ENTRY(copy_user_page_asm) /* Purge any old translations */ #ifdef CONFIG_PA20 - pdtlb,l 0(%r28) - pdtlb,l 0(%r29) + pdtlb,l %r0(%r28) + pdtlb,l %r0(%r29) #else tlb_lock %r20,%r21,%r22 - pdtlb 0(%r28) - pdtlb 0(%r29) + pdtlb %r0(%r28) + pdtlb %r0(%r29) tlb_unlock %r20,%r21,%r22 #endif @@ -768,10 +768,10 @@ ENTRY(clear_user_page_asm) /* Purge any old translation */ #ifdef CONFIG_PA20 - pdtlb,l 0(%r28) + pdtlb,l %r0(%r28) #else tlb_lock %r20,%r21,%r22 - pdtlb 0(%r28) + pdtlb %r0(%r28) tlb_unlock %r20,%r21,%r22 #endif @@ -852,10 +852,10 @@ ENTRY(flush_dcache_page_asm) /* Purge any old translation */ #ifdef CONFIG_PA20 - pdtlb,l 0(%r28) + pdtlb,l %r0(%r28) #else tlb_lock %r20,%r21,%r22 - pdtlb 0(%r28) + pdtlb %r0(%r28) tlb_unlock %r20,%r21,%r22 #endif @@ -886,19 +886,10 @@ ENTRY(flush_dcache_page_asm) fdc,m r31(%r28) fdc,m r31(%r28) fdc,m r31(%r28) - cmpb,COND(<<) %r28, %r25,1b + cmpb,COND(<<) %r28, %r25,1b fdc,m r31(%r28) sync - -#ifdef CONFIG_PA20 - pdtlb,l 0(%r25) -#else - tlb_lock %r20,%r21,%r22 - pdtlb 0(%r25) - tlb_unlock %r20,%r21,%r22 -#endif - bv %r0(%r2) nop .exit @@ -925,13 +916,18 @@ ENTRY(flush_icache_page_asm) depwi 0, 31,PAGE_SHIFT, %r28 /* Clear any offset bits */ #endif - /* Purge any old translation */ + /* Purge any old translation. Note that the FIC instruction + * may use either the instruction or data TLB. Given that we + * have a flat address space, it's not clear which TLB will be + * used. So, we purge both entries. */ #ifdef CONFIG_PA20 + pdtlb,l %r0(%r28) pitlb,l %r0(%sr4,%r28) #else tlb_lock %r20,%r21,%r22 - pitlb (%sr4,%r28) + pdtlb %r0(%r28) + pitlb %r0(%sr4,%r28) tlb_unlock %r20,%r21,%r22 #endif @@ -968,15 +964,6 @@ ENTRY(flush_icache_page_asm) fic,m %r31(%sr4,%r28) sync - -#ifdef CONFIG_PA20 - pitlb,l %r0(%sr4,%r25) -#else - tlb_lock %r20,%r21,%r22 - pitlb (%sr4,%r25) - tlb_unlock %r20,%r21,%r22 -#endif - bv %r0(%r2) nop .exit diff --git a/kernel/arch/parisc/kernel/parisc_ksyms.c b/kernel/arch/parisc/kernel/parisc_ksyms.c index 568b2c61e..3cad8aadc 100644 --- a/kernel/arch/parisc/kernel/parisc_ksyms.c +++ b/kernel/arch/parisc/kernel/parisc_ksyms.c @@ -47,11 +47,11 @@ EXPORT_SYMBOL(__cmpxchg_u64); EXPORT_SYMBOL(lclear_user); EXPORT_SYMBOL(lstrnlen_user); -/* Global fixups */ -extern void fixup_get_user_skip_1(void); -extern void fixup_get_user_skip_2(void); -extern void fixup_put_user_skip_1(void); -extern void fixup_put_user_skip_2(void); +/* Global fixups - defined as int to avoid creation of function pointers */ +extern int fixup_get_user_skip_1; +extern int fixup_get_user_skip_2; +extern int fixup_put_user_skip_1; +extern int fixup_put_user_skip_2; EXPORT_SYMBOL(fixup_get_user_skip_1); EXPORT_SYMBOL(fixup_get_user_skip_2); EXPORT_SYMBOL(fixup_put_user_skip_1); diff --git a/kernel/arch/parisc/kernel/pci-dma.c b/kernel/arch/parisc/kernel/pci-dma.c index b9402c9b3..af0d7fae7 100644 --- a/kernel/arch/parisc/kernel/pci-dma.c +++ b/kernel/arch/parisc/kernel/pci-dma.c @@ -95,8 +95,8 @@ static inline int map_pte_uncached(pte_t * pte, if (!pte_none(*pte)) printk(KERN_ERR "map_pte_uncached: page already exists\n"); - set_pte(pte, __mk_pte(*paddr_ptr, PAGE_KERNEL_UNC)); purge_tlb_start(flags); + set_pte(pte, __mk_pte(*paddr_ptr, PAGE_KERNEL_UNC)); pdtlb_kernel(orig_vaddr); purge_tlb_end(flags); vaddr += PAGE_SIZE; diff --git a/kernel/arch/parisc/kernel/setup.c b/kernel/arch/parisc/kernel/setup.c index f7ea626e2..2e66a8877 100644 --- a/kernel/arch/parisc/kernel/setup.c +++ b/kernel/arch/parisc/kernel/setup.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include /* for pa7300lc_init() proto */ @@ -140,6 +141,13 @@ void __init setup_arch(char **cmdline_p) #endif printk(KERN_CONT ".\n"); + /* + * Check if initial kernel page mappings are sufficient. + * panic early if not, else we may access kernel functions + * and variables which can't be reached. + */ + if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) + panic("KERNEL_INITIAL_ORDER too small!"); pdc_console_init(); @@ -326,6 +334,10 @@ static int __init parisc_init(void) /* tell PDC we're Linux. Nevermind failure. */ pdc_stable_write(0x40, &osid, sizeof(osid)); + /* start with known state */ + flush_cache_all_local(); + flush_tlb_all_local(NULL); + processor_init(); #ifdef CONFIG_SMP pr_info("CPU(s): %d out of %d %s at %d.%06d MHz online\n", diff --git a/kernel/arch/parisc/kernel/syscall.S b/kernel/arch/parisc/kernel/syscall.S index fbafa0d0e..a86b19fcc 100644 --- a/kernel/arch/parisc/kernel/syscall.S +++ b/kernel/arch/parisc/kernel/syscall.S @@ -106,8 +106,6 @@ linux_gateway_entry: mtsp %r0,%sr4 /* get kernel space into sr4 */ mtsp %r0,%sr5 /* get kernel space into sr5 */ mtsp %r0,%sr6 /* get kernel space into sr6 */ - mfsp %sr7,%r1 /* save user sr7 */ - mtsp %r1,%sr3 /* and store it in sr3 */ #ifdef CONFIG_64BIT /* for now we can *always* set the W bit on entry to the syscall @@ -133,6 +131,14 @@ linux_gateway_entry: depdi 0, 31, 32, %r21 1: #endif + + /* We use a rsm/ssm pair to prevent sr3 from being clobbered + * by external interrupts. + */ + mfsp %sr7,%r1 /* save user sr7 */ + rsm PSW_SM_I, %r0 /* disable interrupts */ + mtsp %r1,%sr3 /* and store it in sr3 */ + mfctl %cr30,%r1 xor %r1,%r30,%r30 /* ye olde xor trick */ xor %r1,%r30,%r1 @@ -147,6 +153,7 @@ linux_gateway_entry: */ mtsp %r0,%sr7 /* get kernel space into sr7 */ + ssm PSW_SM_I, %r0 /* enable interrupts */ STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ mfctl %cr30,%r1 /* get task ptr in %r1 */ LDREG TI_TASK(%r1),%r1 diff --git a/kernel/arch/parisc/kernel/traps.c b/kernel/arch/parisc/kernel/traps.c index 553b09855..77e2262c9 100644 --- a/kernel/arch/parisc/kernel/traps.c +++ b/kernel/arch/parisc/kernel/traps.c @@ -798,6 +798,9 @@ void notrace handle_interruption(int code, struct pt_regs *regs) if (fault_space == 0 && !faulthandler_disabled()) { + /* Clean up and return if in exception table. */ + if (fixup_exception(regs)) + return; pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); parisc_terminate("Kernel Fault", regs, code, fault_address); } diff --git a/kernel/arch/parisc/kernel/unaligned.c b/kernel/arch/parisc/kernel/unaligned.c index d7c0acb35..8d49614d6 100644 --- a/kernel/arch/parisc/kernel/unaligned.c +++ b/kernel/arch/parisc/kernel/unaligned.c @@ -666,7 +666,7 @@ void handle_unaligned(struct pt_regs *regs) break; } - if (modify && R1(regs->iir)) + if (ret == 0 && modify && R1(regs->iir)) regs->gr[R1(regs->iir)] = newbase; @@ -677,6 +677,14 @@ void handle_unaligned(struct pt_regs *regs) if (ret) { + /* + * The unaligned handler failed. + * If we were called by __get_user() or __put_user() jump + * to it's exception fixup handler instead of crashing. + */ + if (!user_mode(regs) && fixup_exception(regs)) + return; + printk(KERN_CRIT "Unaligned handler failed, ret = %d\n", ret); die_if_kernel("Unaligned data reference", regs, 28); diff --git a/kernel/arch/parisc/kernel/vmlinux.lds.S b/kernel/arch/parisc/kernel/vmlinux.lds.S index 308f29081..60771df10 100644 --- a/kernel/arch/parisc/kernel/vmlinux.lds.S +++ b/kernel/arch/parisc/kernel/vmlinux.lds.S @@ -88,8 +88,9 @@ SECTIONS /* Start of data section */ _sdata = .; - RO_DATA_SECTION(8) - + /* Architecturally we need to keep __gp below 0x1000000 and thus + * in front of RO_DATA_SECTION() which stores lots of tracepoint + * and ftrace symbols. */ #ifdef CONFIG_64BIT . = ALIGN(16); /* Linkage tables */ @@ -104,6 +105,8 @@ SECTIONS } #endif + RO_DATA_SECTION(8) + /* unwind info */ .PARISC.unwind : { __start___unwind = .; diff --git a/kernel/arch/parisc/lib/fixup.S b/kernel/arch/parisc/lib/fixup.S index 536ef66bb..1052b747e 100644 --- a/kernel/arch/parisc/lib/fixup.S +++ b/kernel/arch/parisc/lib/fixup.S @@ -26,6 +26,7 @@ #ifdef CONFIG_SMP .macro get_fault_ip t1 t2 + loadgp addil LT%__per_cpu_offset,%r27 LDREG RT%__per_cpu_offset(%r1),\t1 /* t2 = smp_processor_id() */ @@ -40,14 +41,19 @@ LDREG RT%exception_data(%r1),\t1 /* t1 = this_cpu_ptr(&exception_data) */ add,l \t1,\t2,\t1 + /* %r27 = t1->fault_gp - restore gp */ + LDREG EXCDATA_GP(\t1), %r27 /* t1 = t1->fault_ip */ LDREG EXCDATA_IP(\t1), \t1 .endm #else .macro get_fault_ip t1 t2 + loadgp /* t1 = this_cpu_ptr(&exception_data) */ addil LT%exception_data,%r27 LDREG RT%exception_data(%r1),\t2 + /* %r27 = t2->fault_gp - restore gp */ + LDREG EXCDATA_GP(\t2), %r27 /* t1 = t2->fault_ip */ LDREG EXCDATA_IP(\t2), \t1 .endm diff --git a/kernel/arch/parisc/mm/fault.c b/kernel/arch/parisc/mm/fault.c index a762864ec..f90644499 100644 --- a/kernel/arch/parisc/mm/fault.c +++ b/kernel/arch/parisc/mm/fault.c @@ -151,6 +151,7 @@ int fixup_exception(struct pt_regs *regs) struct exception_data *d; d = this_cpu_ptr(&exception_data); d->fault_ip = regs->iaoq[0]; + d->fault_gp = regs->gr[27]; d->fault_space = regs->isr; d->fault_addr = regs->ior; diff --git a/kernel/arch/powerpc/boot/ps3-head.S b/kernel/arch/powerpc/boot/ps3-head.S index b6fcbaf50..3dc44b05f 100644 --- a/kernel/arch/powerpc/boot/ps3-head.S +++ b/kernel/arch/powerpc/boot/ps3-head.S @@ -57,11 +57,6 @@ __system_reset_overlay: bctr 1: - /* Save the value at addr zero for a null pointer write check later. */ - - li r4, 0 - lwz r3, 0(r4) - /* Primary delays then goes to _zimage_start in wrapper. */ or 31, 31, 31 /* db16cyc */ diff --git a/kernel/arch/powerpc/boot/ps3.c b/kernel/arch/powerpc/boot/ps3.c index 4ec2d86d3..a05558a7e 100644 --- a/kernel/arch/powerpc/boot/ps3.c +++ b/kernel/arch/powerpc/boot/ps3.c @@ -119,13 +119,12 @@ void ps3_copy_vectors(void) flush_cache((void *)0x100, 512); } -void platform_init(unsigned long null_check) +void platform_init(void) { const u32 heapsize = 0x1000000 - (u32)_end; /* 16MiB */ void *chosen; unsigned long ft_addr; u64 rm_size; - unsigned long val; console_ops.write = ps3_console_write; platform_ops.exit = ps3_exit; @@ -153,11 +152,6 @@ void platform_init(unsigned long null_check) printf(" flat tree at 0x%lx\n\r", ft_addr); - val = *(unsigned long *)0; - - if (val != null_check) - printf("null check failed: %lx != %lx\n\r", val, null_check); - ((kernel_entry_t)0)(ft_addr, 0, NULL); ps3_exit(); diff --git a/kernel/arch/powerpc/include/asm/icswx.h b/kernel/arch/powerpc/include/asm/icswx.h index 9f8402b35..27e588f6c 100644 --- a/kernel/arch/powerpc/include/asm/icswx.h +++ b/kernel/arch/powerpc/include/asm/icswx.h @@ -164,6 +164,7 @@ struct coprocessor_request_block { #define ICSWX_INITIATED (0x8) #define ICSWX_BUSY (0x4) #define ICSWX_REJECTED (0x2) +#define ICSWX_XERS0 (0x1) /* undefined or set from XERSO. */ static inline int icswx(__be32 ccw, struct coprocessor_request_block *crb) { diff --git a/kernel/arch/powerpc/include/asm/kvm_host.h b/kernel/arch/powerpc/include/asm/kvm_host.h index f8673ff84..203765805 100644 --- a/kernel/arch/powerpc/include/asm/kvm_host.h +++ b/kernel/arch/powerpc/include/asm/kvm_host.h @@ -545,6 +545,7 @@ struct kvm_vcpu_arch { u64 tfiar; u32 cr_tm; + u64 xer_tm; u64 lr_tm; u64 ctr_tm; u64 amr_tm; diff --git a/kernel/arch/powerpc/include/asm/reg.h b/kernel/arch/powerpc/include/asm/reg.h index 2220f7a60..627d129d7 100644 --- a/kernel/arch/powerpc/include/asm/reg.h +++ b/kernel/arch/powerpc/include/asm/reg.h @@ -707,7 +707,8 @@ #define MMCR0_FCWAIT 0x00000002UL /* freeze counter in WAIT state */ #define MMCR0_FCHV 0x00000001UL /* freeze conditions in hypervisor mode */ #define SPRN_MMCR1 798 -#define SPRN_MMCR2 769 +#define SPRN_MMCR2 785 +#define SPRN_UMMCR2 769 #define SPRN_MMCRA 0x312 #define MMCRA_SDSYNC 0x80000000UL /* SDAR synced with SIAR */ #define MMCRA_SDAR_DCACHE_MISS 0x40000000UL @@ -744,13 +745,13 @@ #define SPRN_PMC6 792 #define SPRN_PMC7 793 #define SPRN_PMC8 794 -#define SPRN_SIAR 780 -#define SPRN_SDAR 781 #define SPRN_SIER 784 #define SIER_SIPR 0x2000000 /* Sampled MSR_PR */ #define SIER_SIHV 0x1000000 /* Sampled MSR_HV */ #define SIER_SIAR_VALID 0x0400000 /* SIAR contents valid */ #define SIER_SDAR_VALID 0x0200000 /* SDAR contents valid */ +#define SPRN_SIAR 796 +#define SPRN_SDAR 797 #define SPRN_TACR 888 #define SPRN_TCSCR 889 #define SPRN_CSIGR 890 diff --git a/kernel/arch/powerpc/include/asm/uaccess.h b/kernel/arch/powerpc/include/asm/uaccess.h index 2a8ebae09..a5ffe0207 100644 --- a/kernel/arch/powerpc/include/asm/uaccess.h +++ b/kernel/arch/powerpc/include/asm/uaccess.h @@ -323,30 +323,17 @@ extern unsigned long __copy_tofrom_user(void __user *to, static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) { - unsigned long over; - - if (access_ok(VERIFY_READ, from, n)) + if (likely(access_ok(VERIFY_READ, from, n))) return __copy_tofrom_user((__force void __user *)to, from, n); - if ((unsigned long)from < TASK_SIZE) { - over = (unsigned long)from + n - TASK_SIZE; - return __copy_tofrom_user((__force void __user *)to, from, - n - over) + over; - } + memset(to, 0, n); return n; } static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) { - unsigned long over; - if (access_ok(VERIFY_WRITE, to, n)) return __copy_tofrom_user(to, (__force void __user *)from, n); - if ((unsigned long)to < TASK_SIZE) { - over = (unsigned long)to + n - TASK_SIZE; - return __copy_tofrom_user(to, (__force void __user *)from, - n - over) + over; - } return n; } @@ -437,10 +424,6 @@ static inline unsigned long clear_user(void __user *addr, unsigned long size) might_fault(); if (likely(access_ok(VERIFY_WRITE, addr, size))) return __clear_user(addr, size); - if ((unsigned long)addr < TASK_SIZE) { - unsigned long over = (unsigned long)addr + size - TASK_SIZE; - return __clear_user(addr, size - over) + over; - } return size; } diff --git a/kernel/arch/powerpc/include/asm/word-at-a-time.h b/kernel/arch/powerpc/include/asm/word-at-a-time.h index e4396a7d0..4afe66aa1 100644 --- a/kernel/arch/powerpc/include/asm/word-at-a-time.h +++ b/kernel/arch/powerpc/include/asm/word-at-a-time.h @@ -82,7 +82,7 @@ static inline unsigned long create_zero_mask(unsigned long bits) "andc %1,%1,%2\n\t" "popcntd %0,%1" : "=r" (leading_zero_bits), "=&r" (trailing_zero_bit_mask) - : "r" (bits)); + : "b" (bits)); return leading_zero_bits; } diff --git a/kernel/arch/powerpc/include/uapi/asm/cputable.h b/kernel/arch/powerpc/include/uapi/asm/cputable.h index 43686043e..2734c005d 100644 --- a/kernel/arch/powerpc/include/uapi/asm/cputable.h +++ b/kernel/arch/powerpc/include/uapi/asm/cputable.h @@ -31,6 +31,7 @@ #define PPC_FEATURE_PSERIES_PERFMON_COMPAT \ 0x00000040 +/* Reserved - do not use 0x00000004 */ #define PPC_FEATURE_TRUE_LE 0x00000002 #define PPC_FEATURE_PPC_LE 0x00000001 diff --git a/kernel/arch/powerpc/include/uapi/asm/kvm.h b/kernel/arch/powerpc/include/uapi/asm/kvm.h index ab4d4732c..720b71a63 100644 --- a/kernel/arch/powerpc/include/uapi/asm/kvm.h +++ b/kernel/arch/powerpc/include/uapi/asm/kvm.h @@ -587,6 +587,7 @@ struct kvm_get_htab_header { #define KVM_REG_PPC_TM_VSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U32 | 0x67) #define KVM_REG_PPC_TM_DSCR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x68) #define KVM_REG_PPC_TM_TAR (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x69) +#define KVM_REG_PPC_TM_XER (KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x6a) /* PPC64 eXternal Interrupt Controller Specification */ #define KVM_DEV_XICS_GRP_SOURCES 1 /* 64-bit source attributes */ diff --git a/kernel/arch/powerpc/kernel/asm-offsets.c b/kernel/arch/powerpc/kernel/asm-offsets.c index d6d0c59ef..bd0408153 100644 --- a/kernel/arch/powerpc/kernel/asm-offsets.c +++ b/kernel/arch/powerpc/kernel/asm-offsets.c @@ -585,6 +585,7 @@ int main(void) DEFINE(VCPU_VRS_TM, offsetof(struct kvm_vcpu, arch.vr_tm.vr)); DEFINE(VCPU_VRSAVE_TM, offsetof(struct kvm_vcpu, arch.vrsave_tm)); DEFINE(VCPU_CR_TM, offsetof(struct kvm_vcpu, arch.cr_tm)); + DEFINE(VCPU_XER_TM, offsetof(struct kvm_vcpu, arch.xer_tm)); DEFINE(VCPU_LR_TM, offsetof(struct kvm_vcpu, arch.lr_tm)); DEFINE(VCPU_CTR_TM, offsetof(struct kvm_vcpu, arch.ctr_tm)); DEFINE(VCPU_AMR_TM, offsetof(struct kvm_vcpu, arch.amr_tm)); diff --git a/kernel/arch/powerpc/kernel/eeh.c b/kernel/arch/powerpc/kernel/eeh.c index 40e4d4a27..98949b0df 100644 --- a/kernel/arch/powerpc/kernel/eeh.c +++ b/kernel/arch/powerpc/kernel/eeh.c @@ -677,7 +677,7 @@ int eeh_pci_enable(struct eeh_pe *pe, int function) /* Check if the request is finished successfully */ if (active_flag) { rc = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); - if (rc <= 0) + if (rc < 0) return rc; if (rc & active_flag) @@ -1072,7 +1072,7 @@ void eeh_add_device_early(struct pci_dn *pdn) struct pci_controller *phb; struct eeh_dev *edev = pdn_to_eeh_dev(pdn); - if (!edev || !eeh_enabled()) + if (!edev) return; if (!eeh_has_flag(EEH_PROBE_MODE_DEVTREE)) diff --git a/kernel/arch/powerpc/kernel/eeh_driver.c b/kernel/arch/powerpc/kernel/eeh_driver.c index 52c1e273f..c314db8b7 100644 --- a/kernel/arch/powerpc/kernel/eeh_driver.c +++ b/kernel/arch/powerpc/kernel/eeh_driver.c @@ -166,6 +166,16 @@ static void *eeh_dev_save_state(void *data, void *userdata) if (!edev) return NULL; + /* + * We cannot access the config space on some adapters. + * Otherwise, it will cause fenced PHB. We don't save + * the content in their config space and will restore + * from the initial config space saved when the EEH + * device is created. + */ + if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED)) + return NULL; + pdev = eeh_dev_to_pci_dev(edev); if (!pdev) return NULL; @@ -305,6 +315,19 @@ static void *eeh_dev_restore_state(void *data, void *userdata) if (!edev) return NULL; + /* + * The content in the config space isn't saved because + * the blocked config space on some adapters. We have + * to restore the initial saved config space when the + * EEH device is created. + */ + if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED)) { + if (list_is_last(&edev->list, &edev->pe->edevs)) + eeh_pe_restore_bars(edev->pe); + + return NULL; + } + pdev = eeh_dev_to_pci_dev(edev); if (!pdev) return NULL; @@ -462,7 +485,7 @@ static void *eeh_pe_detach_dev(void *data, void *userdata) static void *__eeh_clear_pe_frozen_state(void *data, void *flag) { struct eeh_pe *pe = (struct eeh_pe *)data; - bool *clear_sw_state = flag; + bool clear_sw_state = *(bool *)flag; int i, rc = 1; for (i = 0; rc && i < 3; i++) @@ -504,9 +527,6 @@ int eeh_pe_reset_and_recover(struct eeh_pe *pe) /* Save states */ eeh_pe_dev_traverse(pe, eeh_dev_save_state, NULL); - /* Report error */ - eeh_pe_dev_traverse(pe, eeh_report_error, &result); - /* Issue reset */ ret = eeh_reset_pe(pe); if (ret) { @@ -592,8 +612,10 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus) /* Clear frozen state */ rc = eeh_clear_pe_frozen_state(pe, false); - if (rc) + if (rc) { + pci_unlock_rescan_remove(); return rc; + } /* Give the system 5 seconds to finish running the user-space * hotplug shutdown scripts, e.g. ifdown for ethernet. Yes, @@ -889,6 +911,14 @@ static void eeh_handle_special_event(void) /* Notify all devices to be down */ eeh_pe_state_clear(pe, EEH_PE_PRI_BUS); bus = eeh_pe_bus_get(phb_pe); + if (!bus) { + pr_err("%s: Cannot find PCI bus for " + "PHB#%d-PE#%x\n", + __func__, + pe->phb->global_number, + pe->addr); + break; + } eeh_pe_dev_traverse(pe, eeh_report_failure, NULL); pcibios_remove_pci_devices(bus); diff --git a/kernel/arch/powerpc/kernel/entry_64.S b/kernel/arch/powerpc/kernel/entry_64.S index 5bb3148e5..1aae3fdb0 100644 --- a/kernel/arch/powerpc/kernel/entry_64.S +++ b/kernel/arch/powerpc/kernel/entry_64.S @@ -334,13 +334,13 @@ syscall_exit_work: tabort_syscall: /* Firstly we need to enable TM in the kernel */ mfmsr r10 - li r13, 1 - rldimi r10, r13, MSR_TM_LG, 63-MSR_TM_LG + li r9, 1 + rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG mtmsrd r10, 0 /* tabort, this dooms the transaction, nothing else */ - li r13, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT) - TABORT(R13) + li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT) + TABORT(R9) /* * Return directly to userspace. We have corrupted user register state, @@ -348,8 +348,8 @@ tabort_syscall: * resume after the tbegin of the aborted transaction with the * checkpointed register state. */ - li r13, MSR_RI - andc r10, r10, r13 + li r9, MSR_RI + andc r10, r10, r9 mtmsrd r10, 1 mtspr SPRN_SRR0, r11 mtspr SPRN_SRR1, r12 diff --git a/kernel/arch/powerpc/kernel/exceptions-64s.S b/kernel/arch/powerpc/kernel/exceptions-64s.S index 0a0399c2a..b81ccc5fb 100644 --- a/kernel/arch/powerpc/kernel/exceptions-64s.S +++ b/kernel/arch/powerpc/kernel/exceptions-64s.S @@ -962,11 +962,6 @@ hv_facility_unavailable_relon_trampoline: #endif STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist) - /* Other future vectors */ - .align 7 - .globl __end_interrupts -__end_interrupts: - .align 7 system_call_entry: b system_call_common @@ -1253,6 +1248,17 @@ __end_handlers: STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable) STD_RELON_EXCEPTION_HV_OOL(0xf80, hv_facility_unavailable) + /* + * The __end_interrupts marker must be past the out-of-line (OOL) + * handlers, so that they are copied to real address 0x100 when running + * a relocatable kernel. This ensures they can be reached from the short + * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch + * directly, without using LOAD_HANDLER(). + */ + .align 7 + .globl __end_interrupts +__end_interrupts: + #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) /* * Data area reserved for FWNMI option. diff --git a/kernel/arch/powerpc/kernel/ibmebus.c b/kernel/arch/powerpc/kernel/ibmebus.c index ac86c53e2..e524a775f 100644 --- a/kernel/arch/powerpc/kernel/ibmebus.c +++ b/kernel/arch/powerpc/kernel/ibmebus.c @@ -180,6 +180,7 @@ static int ibmebus_create_device(struct device_node *dn) static int ibmebus_create_devices(const struct of_device_id *matches) { struct device_node *root, *child; + struct device *dev; int ret = 0; root = of_find_node_by_path("/"); @@ -188,9 +189,12 @@ static int ibmebus_create_devices(const struct of_device_id *matches) if (!of_match_node(matches, child)) continue; - if (bus_find_device(&ibmebus_bus_type, NULL, child, - ibmebus_match_node)) + dev = bus_find_device(&ibmebus_bus_type, NULL, child, + ibmebus_match_node); + if (dev) { + put_device(dev); continue; + } ret = ibmebus_create_device(child); if (ret) { @@ -262,6 +266,7 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus, const char *buf, size_t count) { struct device_node *dn = NULL; + struct device *dev; char *path; ssize_t rc = 0; @@ -269,8 +274,10 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus, if (!path) return -ENOMEM; - if (bus_find_device(&ibmebus_bus_type, NULL, path, - ibmebus_match_path)) { + dev = bus_find_device(&ibmebus_bus_type, NULL, path, + ibmebus_match_path); + if (dev) { + put_device(dev); printk(KERN_WARNING "%s: %s has already been probed\n", __func__, path); rc = -EEXIST; @@ -307,6 +314,7 @@ static ssize_t ibmebus_store_remove(struct bus_type *bus, if ((dev = bus_find_device(&ibmebus_bus_type, NULL, path, ibmebus_match_path))) { of_device_unregister(to_platform_device(dev)); + put_device(dev); kfree(path); return count; diff --git a/kernel/arch/powerpc/kernel/idle_power7.S b/kernel/arch/powerpc/kernel/idle_power7.S index 112ccf497..73f638789 100644 --- a/kernel/arch/powerpc/kernel/idle_power7.S +++ b/kernel/arch/powerpc/kernel/idle_power7.S @@ -44,7 +44,7 @@ std r0,0(r1); \ ptesync; \ ld r0,0(r1); \ -1: cmp cr0,r0,r0; \ +1: cmpd cr0,r0,r0; \ bne 1b; \ IDLE_INST; \ b . diff --git a/kernel/arch/powerpc/kernel/misc_32.S b/kernel/arch/powerpc/kernel/misc_32.S index 8b261416c..9cb0c2f6e 100644 --- a/kernel/arch/powerpc/kernel/misc_32.S +++ b/kernel/arch/powerpc/kernel/misc_32.S @@ -315,7 +315,7 @@ _GLOBAL(flush_instruction_cache) lis r3, KERNELBASE@h iccci 0,r3 #endif -#elif CONFIG_FSL_BOOKE +#elif defined(CONFIG_FSL_BOOKE) BEGIN_FTR_SECTION mfspr r3,SPRN_L1CSR0 ori r3,r3,L1CSR0_CFI|L1CSR0_CLFC diff --git a/kernel/arch/powerpc/kernel/nvram_64.c b/kernel/arch/powerpc/kernel/nvram_64.c index 32e26526f..1eb698f65 100644 --- a/kernel/arch/powerpc/kernel/nvram_64.c +++ b/kernel/arch/powerpc/kernel/nvram_64.c @@ -969,7 +969,7 @@ int __init nvram_remove_partition(const char *name, int sig, /* Make partition a free partition */ part->header.signature = NVRAM_SIG_FREE; - strncpy(part->header.name, "wwwwwwwwwwww", 12); + memset(part->header.name, 'w', 12); part->header.checksum = nvram_checksum(&part->header); rc = nvram_write_header(part); if (rc <= 0) { @@ -987,8 +987,8 @@ int __init nvram_remove_partition(const char *name, int sig, } if (prev) { prev->header.length += part->header.length; - prev->header.checksum = nvram_checksum(&part->header); - rc = nvram_write_header(part); + prev->header.checksum = nvram_checksum(&prev->header); + rc = nvram_write_header(prev); if (rc <= 0) { printk(KERN_ERR "nvram_remove_partition: nvram_write failed (%d)\n", rc); return rc; diff --git a/kernel/arch/powerpc/kernel/process.c b/kernel/arch/powerpc/kernel/process.c index ef2ad2d68..cf788d7d7 100644 --- a/kernel/arch/powerpc/kernel/process.c +++ b/kernel/arch/powerpc/kernel/process.c @@ -551,24 +551,6 @@ static void tm_reclaim_thread(struct thread_struct *thr, msr_diff &= MSR_FP | MSR_VEC | MSR_VSX | MSR_FE0 | MSR_FE1; } - /* - * Use the current MSR TM suspended bit to track if we have - * checkpointed state outstanding. - * On signal delivery, we'd normally reclaim the checkpointed - * state to obtain stack pointer (see:get_tm_stackpointer()). - * This will then directly return to userspace without going - * through __switch_to(). However, if the stack frame is bad, - * we need to exit this thread which calls __switch_to() which - * will again attempt to reclaim the already saved tm state. - * Hence we need to check that we've not already reclaimed - * this state. - * We do this using the current MSR, rather tracking it in - * some specific thread_struct bit, as it has the additional - * benifit of checking for a potential TM bad thing exception. - */ - if (!MSR_TM_SUSPENDED(mfmsr())) - return; - /* * Use the current MSR TM suspended bit to track if we have * checkpointed state outstanding. @@ -1257,6 +1239,16 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) current->thread.regs = regs - 1; } +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + /* + * Clear any transactional state, we're exec()ing. The cause is + * not important as there will never be a recheckpoint so it's not + * user visible. + */ + if (MSR_TM_SUSPENDED(mfmsr())) + tm_reclaim_current(0); +#endif + memset(regs->gpr, 0, sizeof(regs->gpr)); regs->ctr = 0; regs->link = 0; diff --git a/kernel/arch/powerpc/kernel/prom.c b/kernel/arch/powerpc/kernel/prom.c index 7030b0359..a15fe1d4e 100644 --- a/kernel/arch/powerpc/kernel/prom.c +++ b/kernel/arch/powerpc/kernel/prom.c @@ -148,23 +148,25 @@ static struct ibm_pa_feature { unsigned long cpu_features; /* CPU_FTR_xxx bit */ unsigned long mmu_features; /* MMU_FTR_xxx bit */ unsigned int cpu_user_ftrs; /* PPC_FEATURE_xxx bit */ + unsigned int cpu_user_ftrs2; /* PPC_FEATURE2_xxx bit */ unsigned char pabyte; /* byte number in ibm,pa-features */ unsigned char pabit; /* bit number (big-endian) */ unsigned char invert; /* if 1, pa bit set => clear feature */ } ibm_pa_features[] __initdata = { - {0, 0, PPC_FEATURE_HAS_MMU, 0, 0, 0}, - {0, 0, PPC_FEATURE_HAS_FPU, 0, 1, 0}, - {CPU_FTR_CTRL, 0, 0, 0, 3, 0}, - {CPU_FTR_NOEXECUTE, 0, 0, 0, 6, 0}, - {CPU_FTR_NODSISRALIGN, 0, 0, 1, 1, 1}, - {0, MMU_FTR_CI_LARGE_PAGE, 0, 1, 2, 0}, - {CPU_FTR_REAL_LE, PPC_FEATURE_TRUE_LE, 5, 0, 0}, + {0, 0, PPC_FEATURE_HAS_MMU, 0, 0, 0, 0}, + {0, 0, PPC_FEATURE_HAS_FPU, 0, 0, 1, 0}, + {CPU_FTR_CTRL, 0, 0, 0, 0, 3, 0}, + {CPU_FTR_NOEXECUTE, 0, 0, 0, 0, 6, 0}, + {CPU_FTR_NODSISRALIGN, 0, 0, 0, 1, 1, 1}, + {0, MMU_FTR_CI_LARGE_PAGE, 0, 0, 1, 2, 0}, + {CPU_FTR_REAL_LE, 0, PPC_FEATURE_TRUE_LE, 0, 5, 0, 0}, /* - * If the kernel doesn't support TM (ie. CONFIG_PPC_TRANSACTIONAL_MEM=n), - * we don't want to turn on CPU_FTR_TM here, so we use CPU_FTR_TM_COMP - * which is 0 if the kernel doesn't support TM. + * If the kernel doesn't support TM (ie CONFIG_PPC_TRANSACTIONAL_MEM=n), + * we don't want to turn on TM here, so we use the *_COMP versions + * which are 0 if the kernel doesn't support TM. */ - {CPU_FTR_TM_COMP, 0, 0, 22, 0, 0}, + {CPU_FTR_TM_COMP, 0, 0, + PPC_FEATURE2_HTM_COMP|PPC_FEATURE2_HTM_NOSC_COMP, 22, 0, 0}, }; static void __init scan_features(unsigned long node, const unsigned char *ftrs, @@ -195,10 +197,12 @@ static void __init scan_features(unsigned long node, const unsigned char *ftrs, if (bit ^ fp->invert) { cur_cpu_spec->cpu_features |= fp->cpu_features; cur_cpu_spec->cpu_user_features |= fp->cpu_user_ftrs; + cur_cpu_spec->cpu_user_features2 |= fp->cpu_user_ftrs2; cur_cpu_spec->mmu_features |= fp->mmu_features; } else { cur_cpu_spec->cpu_features &= ~fp->cpu_features; cur_cpu_spec->cpu_user_features &= ~fp->cpu_user_ftrs; + cur_cpu_spec->cpu_user_features2 &= ~fp->cpu_user_ftrs2; cur_cpu_spec->mmu_features &= ~fp->mmu_features; } } diff --git a/kernel/arch/powerpc/kernel/prom_init.c b/kernel/arch/powerpc/kernel/prom_init.c index 92dea8df6..313953364 100644 --- a/kernel/arch/powerpc/kernel/prom_init.c +++ b/kernel/arch/powerpc/kernel/prom_init.c @@ -655,6 +655,7 @@ unsigned char ibm_architecture_vec[] = { W(0xffff0000), W(0x003e0000), /* POWER6 */ W(0xffff0000), W(0x003f0000), /* POWER7 */ W(0xffff0000), W(0x004b0000), /* POWER8E */ + W(0xffff0000), W(0x004c0000), /* POWER8NVL */ W(0xffff0000), W(0x004d0000), /* POWER8 */ W(0xffffffff), W(0x0f000004), /* all 2.07-compliant */ W(0xffffffff), W(0x0f000003), /* all 2.06-compliant */ @@ -693,7 +694,7 @@ unsigned char ibm_architecture_vec[] = { OV4_MIN_ENT_CAP, /* minimum VP entitled capacity */ /* option vector 5: PAPR/OF options */ - VECTOR_LENGTH(18), /* length */ + VECTOR_LENGTH(21), /* length */ 0, /* don't ignore, don't halt */ OV5_FEAT(OV5_LPAR) | OV5_FEAT(OV5_SPLPAR) | OV5_FEAT(OV5_LARGE_PAGES) | OV5_FEAT(OV5_DRCONF_MEMORY) | OV5_FEAT(OV5_DONATE_DEDICATE_CPU) | @@ -717,15 +718,18 @@ unsigned char ibm_architecture_vec[] = { * must match by the macro below. Update the definition if * the structure layout changes. */ -#define IBM_ARCH_VEC_NRCORES_OFFSET 125 +#define IBM_ARCH_VEC_NRCORES_OFFSET 133 W(NR_CPUS), /* number of cores supported */ 0, 0, 0, 0, OV5_FEAT(OV5_PFO_HW_RNG) | OV5_FEAT(OV5_PFO_HW_ENCR) | - OV5_FEAT(OV5_PFO_HW_842), - OV5_FEAT(OV5_SUB_PROCESSORS), + OV5_FEAT(OV5_PFO_HW_842), /* Byte 17 */ + 0, /* Byte 18 */ + 0, /* Byte 19 */ + 0, /* Byte 20 */ + OV5_FEAT(OV5_SUB_PROCESSORS), /* Byte 21 */ /* option vector 6: IBM PAPR hints */ VECTOR_LENGTH(3), /* length */ @@ -2660,6 +2664,9 @@ static void __init prom_find_boot_cpu(void) cpu_pkg = call_prom("instance-to-package", 1, 1, prom_cpu); + if (!PHANDLE_VALID(cpu_pkg)) + return; + prom_getprop(cpu_pkg, "reg", &rval, sizeof(rval)); prom.cpu = be32_to_cpu(rval); diff --git a/kernel/arch/powerpc/kernel/ptrace.c b/kernel/arch/powerpc/kernel/ptrace.c index 737c0d0b5..b38fd081b 100644 --- a/kernel/arch/powerpc/kernel/ptrace.c +++ b/kernel/arch/powerpc/kernel/ptrace.c @@ -376,7 +376,7 @@ static int fpr_get(struct task_struct *target, const struct user_regset *regset, #else BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) != - offsetof(struct thread_fp_state, fpr[32][0])); + offsetof(struct thread_fp_state, fpr[32])); return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.fp_state, 0, -1); @@ -404,7 +404,7 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, return 0; #else BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) != - offsetof(struct thread_fp_state, fpr[32][0])); + offsetof(struct thread_fp_state, fpr[32])); return user_regset_copyin(&pos, &count, &kbuf, &ubuf, &target->thread.fp_state, 0, -1); diff --git a/kernel/arch/powerpc/kernel/tm.S b/kernel/arch/powerpc/kernel/tm.S index bf8f34a58..b7019b559 100644 --- a/kernel/arch/powerpc/kernel/tm.S +++ b/kernel/arch/powerpc/kernel/tm.S @@ -110,17 +110,11 @@ _GLOBAL(tm_reclaim) std r3, STK_PARAM(R3)(r1) SAVE_NVGPRS(r1) - /* We need to setup MSR for VSX register save instructions. Here we - * also clear the MSR RI since when we do the treclaim, we won't have a - * valid kernel pointer for a while. We clear RI here as it avoids - * adding another mtmsr closer to the treclaim. This makes the region - * maked as non-recoverable wider than it needs to be but it saves on - * inserting another mtmsrd later. - */ + /* We need to setup MSR for VSX register save instructions. */ mfmsr r14 mr r15, r14 ori r15, r15, MSR_FP - li r16, MSR_RI + li r16, 0 ori r16, r16, MSR_EE /* IRQs hard off */ andc r15, r15, r16 oris r15, r15, MSR_VEC@h @@ -176,7 +170,17 @@ dont_backup_fp: 1: tdeqi r6, 0 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0 - /* The moment we treclaim, ALL of our GPRs will switch + /* Clear MSR RI since we are about to change r1, EE is already off. */ + li r4, 0 + mtmsrd r4, 1 + + /* + * BE CAREFUL HERE: + * At this point we can't take an SLB miss since we have MSR_RI + * off. Load only to/from the stack/paca which are in SLB bolted regions + * until we turn MSR RI back on. + * + * The moment we treclaim, ALL of our GPRs will switch * to user register state. (FPRs, CCR etc. also!) * Use an sprg and a tm_scratch in the PACA to shuffle. */ @@ -197,6 +201,11 @@ dont_backup_fp: /* Store the PPR in r11 and reset to decent value */ std r11, GPR11(r1) /* Temporary stash */ + + /* Reset MSR RI so we can take SLB faults again */ + li r11, MSR_RI + mtmsrd r11, 1 + mfspr r11, SPRN_PPR HMT_MEDIUM @@ -397,11 +406,6 @@ restore_gprs: ld r5, THREAD_TM_DSCR(r3) ld r6, THREAD_TM_PPR(r3) - /* Clear the MSR RI since we are about to change R1. EE is already off - */ - li r4, 0 - mtmsrd r4, 1 - REST_GPR(0, r7) /* GPR0 */ REST_2GPRS(2, r7) /* GPR2-3 */ REST_GPR(4, r7) /* GPR4 */ @@ -439,10 +443,33 @@ restore_gprs: ld r6, _CCR(r7) mtcr r6 - REST_GPR(1, r7) /* GPR1 */ - REST_GPR(5, r7) /* GPR5-7 */ REST_GPR(6, r7) - ld r7, GPR7(r7) + + /* + * Store r1 and r5 on the stack so that we can access them + * after we clear MSR RI. + */ + + REST_GPR(5, r7) + std r5, -8(r1) + ld r5, GPR1(r7) + std r5, -16(r1) + + REST_GPR(7, r7) + + /* Clear MSR RI since we are about to change r1. EE is already off */ + li r5, 0 + mtmsrd r5, 1 + + /* + * BE CAREFUL HERE: + * At this point we can't take an SLB miss since we have MSR_RI + * off. Load only to/from the stack/paca which are in SLB bolted regions + * until we turn MSR RI back on. + */ + + ld r5, -8(r1) + ld r1, -16(r1) /* Commit register state as checkpointed state: */ TRECHKPT diff --git a/kernel/arch/powerpc/kernel/vdso64/datapage.S b/kernel/arch/powerpc/kernel/vdso64/datapage.S index 2f01c4a0d..7612eeb31 100644 --- a/kernel/arch/powerpc/kernel/vdso64/datapage.S +++ b/kernel/arch/powerpc/kernel/vdso64/datapage.S @@ -59,7 +59,7 @@ V_FUNCTION_BEGIN(__kernel_get_syscall_map) bl V_LOCAL_FUNC(__get_datapage) mtlr r12 addi r3,r3,CFG_SYSCALL_MAP64 - cmpli cr0,r4,0 + cmpldi cr0,r4,0 crclr cr0*4+so beqlr li r0,__NR_syscalls diff --git a/kernel/arch/powerpc/kernel/vdso64/gettimeofday.S b/kernel/arch/powerpc/kernel/vdso64/gettimeofday.S index a76b4af37..382021324 100644 --- a/kernel/arch/powerpc/kernel/vdso64/gettimeofday.S +++ b/kernel/arch/powerpc/kernel/vdso64/gettimeofday.S @@ -145,7 +145,7 @@ V_FUNCTION_BEGIN(__kernel_clock_getres) bne cr0,99f li r3,0 - cmpli cr0,r4,0 + cmpldi cr0,r4,0 crclr cr0*4+so beqlr lis r5,CLOCK_REALTIME_RES@h diff --git a/kernel/arch/powerpc/kvm/book3s_emulate.c b/kernel/arch/powerpc/kvm/book3s_emulate.c index 2afdb9c09..729f8faa9 100644 --- a/kernel/arch/powerpc/kvm/book3s_emulate.c +++ b/kernel/arch/powerpc/kvm/book3s_emulate.c @@ -498,6 +498,7 @@ int kvmppc_core_emulate_mtspr_pr(struct kvm_vcpu *vcpu, int sprn, ulong spr_val) case SPRN_MMCR0: case SPRN_MMCR1: case SPRN_MMCR2: + case SPRN_UMMCR2: #endif break; unprivileged: @@ -640,6 +641,7 @@ int kvmppc_core_emulate_mfspr_pr(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val case SPRN_MMCR0: case SPRN_MMCR1: case SPRN_MMCR2: + case SPRN_UMMCR2: case SPRN_TIR: #endif *spr_val = 0; diff --git a/kernel/arch/powerpc/kvm/book3s_hv.c b/kernel/arch/powerpc/kvm/book3s_hv.c index df34a6432..2ac7a8b10 100644 --- a/kernel/arch/powerpc/kvm/book3s_hv.c +++ b/kernel/arch/powerpc/kvm/book3s_hv.c @@ -1186,6 +1186,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: *val = get_reg_val(id, vcpu->arch.cr_tm); break; + case KVM_REG_PPC_TM_XER: + *val = get_reg_val(id, vcpu->arch.xer_tm); + break; case KVM_REG_PPC_TM_LR: *val = get_reg_val(id, vcpu->arch.lr_tm); break; @@ -1393,6 +1396,9 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_TM_CR: vcpu->arch.cr_tm = set_reg_val(id, *val); break; + case KVM_REG_PPC_TM_XER: + vcpu->arch.xer_tm = set_reg_val(id, *val); + break; case KVM_REG_PPC_TM_LR: vcpu->arch.lr_tm = set_reg_val(id, *val); break; diff --git a/kernel/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/kernel/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 91700518b..d509ff5c8 100644 --- a/kernel/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/kernel/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -653,6 +653,8 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, HPTE_V_ABSENT); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); + /* Don't lose R/C bit updates done by hardware */ + r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C); hpte[1] = cpu_to_be64(r); } } diff --git a/kernel/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/kernel/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 463af88c9..1a743f87b 100644 --- a/kernel/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/kernel/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -655,112 +655,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM BEGIN_FTR_SECTION - b skip_tm -END_FTR_SECTION_IFCLR(CPU_FTR_TM) - - /* Turn on TM/FP/VSX/VMX so we can restore them. */ - mfmsr r5 - li r6, MSR_TM >> 32 - sldi r6, r6, 32 - or r5, r5, r6 - ori r5, r5, MSR_FP - oris r5, r5, (MSR_VEC | MSR_VSX)@h - mtmsrd r5 - - /* - * The user may change these outside of a transaction, so they must - * always be context switched. - */ - ld r5, VCPU_TFHAR(r4) - ld r6, VCPU_TFIAR(r4) - ld r7, VCPU_TEXASR(r4) - mtspr SPRN_TFHAR, r5 - mtspr SPRN_TFIAR, r6 - mtspr SPRN_TEXASR, r7 - - ld r5, VCPU_MSR(r4) - rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 - beq skip_tm /* TM not active in guest */ - - /* Make sure the failure summary is set, otherwise we'll program check - * when we trechkpt. It's possible that this might have been not set - * on a kvmppc_set_one_reg() call but we shouldn't let this crash the - * host. - */ - oris r7, r7, (TEXASR_FS)@h - mtspr SPRN_TEXASR, r7 - - /* - * We need to load up the checkpointed state for the guest. - * We need to do this early as it will blow away any GPRs, VSRs and - * some SPRs. - */ - - mr r31, r4 - addi r3, r31, VCPU_FPRS_TM - bl load_fp_state - addi r3, r31, VCPU_VRS_TM - bl load_vr_state - mr r4, r31 - lwz r7, VCPU_VRSAVE_TM(r4) - mtspr SPRN_VRSAVE, r7 - - ld r5, VCPU_LR_TM(r4) - lwz r6, VCPU_CR_TM(r4) - ld r7, VCPU_CTR_TM(r4) - ld r8, VCPU_AMR_TM(r4) - ld r9, VCPU_TAR_TM(r4) - mtlr r5 - mtcr r6 - mtctr r7 - mtspr SPRN_AMR, r8 - mtspr SPRN_TAR, r9 - - /* - * Load up PPR and DSCR values but don't put them in the actual SPRs - * till the last moment to avoid running with userspace PPR and DSCR for - * too long. - */ - ld r29, VCPU_DSCR_TM(r4) - ld r30, VCPU_PPR_TM(r4) - - std r2, PACATMSCRATCH(r13) /* Save TOC */ - - /* Clear the MSR RI since r1, r13 are all going to be foobar. */ - li r5, 0 - mtmsrd r5, 1 - - /* Load GPRs r0-r28 */ - reg = 0 - .rept 29 - ld reg, VCPU_GPRS_TM(reg)(r31) - reg = reg + 1 - .endr - - mtspr SPRN_DSCR, r29 - mtspr SPRN_PPR, r30 - - /* Load final GPRs */ - ld 29, VCPU_GPRS_TM(29)(r31) - ld 30, VCPU_GPRS_TM(30)(r31) - ld 31, VCPU_GPRS_TM(31)(r31) - - /* TM checkpointed state is now setup. All GPRs are now volatile. */ - TRECHKPT - - /* Now let's get back the state we need. */ - HMT_MEDIUM - GET_PACA(r13) - ld r29, HSTATE_DSCR(r13) - mtspr SPRN_DSCR, r29 - ld r4, HSTATE_KVM_VCPU(r13) - ld r1, HSTATE_HOST_R1(r13) - ld r2, PACATMSCRATCH(r13) - - /* Set the MSR RI since we have our registers back. */ - li r5, MSR_RI - mtmsrd r5, 1 -skip_tm: + bl kvmppc_restore_tm +END_FTR_SECTION_IFSET(CPU_FTR_TM) #endif /* Load guest PMU registers */ @@ -841,12 +737,6 @@ BEGIN_FTR_SECTION /* Skip next section on POWER7 */ b 8f END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) - /* Turn on TM so we can access TFHAR/TFIAR/TEXASR */ - mfmsr r8 - li r0, 1 - rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG - mtmsrd r8 - /* Load up POWER8-specific registers */ ld r5, VCPU_IAMR(r4) lwz r6, VCPU_PSPB(r4) @@ -1436,106 +1326,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM BEGIN_FTR_SECTION - b 2f -END_FTR_SECTION_IFCLR(CPU_FTR_TM) - /* Turn on TM. */ - mfmsr r8 - li r0, 1 - rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG - mtmsrd r8 - - ld r5, VCPU_MSR(r9) - rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 - beq 1f /* TM not active in guest. */ - - li r3, TM_CAUSE_KVM_RESCHED - - /* Clear the MSR RI since r1, r13 are all going to be foobar. */ - li r5, 0 - mtmsrd r5, 1 - - /* All GPRs are volatile at this point. */ - TRECLAIM(R3) - - /* Temporarily store r13 and r9 so we have some regs to play with */ - SET_SCRATCH0(r13) - GET_PACA(r13) - std r9, PACATMSCRATCH(r13) - ld r9, HSTATE_KVM_VCPU(r13) - - /* Get a few more GPRs free. */ - std r29, VCPU_GPRS_TM(29)(r9) - std r30, VCPU_GPRS_TM(30)(r9) - std r31, VCPU_GPRS_TM(31)(r9) - - /* Save away PPR and DSCR soon so don't run with user values. */ - mfspr r31, SPRN_PPR - HMT_MEDIUM - mfspr r30, SPRN_DSCR - ld r29, HSTATE_DSCR(r13) - mtspr SPRN_DSCR, r29 - - /* Save all but r9, r13 & r29-r31 */ - reg = 0 - .rept 29 - .if (reg != 9) && (reg != 13) - std reg, VCPU_GPRS_TM(reg)(r9) - .endif - reg = reg + 1 - .endr - /* ... now save r13 */ - GET_SCRATCH0(r4) - std r4, VCPU_GPRS_TM(13)(r9) - /* ... and save r9 */ - ld r4, PACATMSCRATCH(r13) - std r4, VCPU_GPRS_TM(9)(r9) - - /* Reload stack pointer and TOC. */ - ld r1, HSTATE_HOST_R1(r13) - ld r2, PACATOC(r13) - - /* Set MSR RI now we have r1 and r13 back. */ - li r5, MSR_RI - mtmsrd r5, 1 - - /* Save away checkpinted SPRs. */ - std r31, VCPU_PPR_TM(r9) - std r30, VCPU_DSCR_TM(r9) - mflr r5 - mfcr r6 - mfctr r7 - mfspr r8, SPRN_AMR - mfspr r10, SPRN_TAR - std r5, VCPU_LR_TM(r9) - stw r6, VCPU_CR_TM(r9) - std r7, VCPU_CTR_TM(r9) - std r8, VCPU_AMR_TM(r9) - std r10, VCPU_TAR_TM(r9) - - /* Restore r12 as trap number. */ - lwz r12, VCPU_TRAP(r9) - - /* Save FP/VSX. */ - addi r3, r9, VCPU_FPRS_TM - bl store_fp_state - addi r3, r9, VCPU_VRS_TM - bl store_vr_state - mfspr r6, SPRN_VRSAVE - stw r6, VCPU_VRSAVE_TM(r9) -1: - /* - * We need to save these SPRs after the treclaim so that the software - * error code is recorded correctly in the TEXASR. Also the user may - * change these outside of a transaction, so they must always be - * context switched. - */ - mfspr r5, SPRN_TFHAR - mfspr r6, SPRN_TFIAR - mfspr r7, SPRN_TEXASR - std r5, VCPU_TFHAR(r9) - std r6, VCPU_TFIAR(r9) - std r7, VCPU_TEXASR(r9) -2: + bl kvmppc_save_tm +END_FTR_SECTION_IFSET(CPU_FTR_TM) #endif /* Increment yield count if they have a VPA */ @@ -2245,6 +2037,13 @@ _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ /* save FP state */ bl kvmppc_save_fp +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +BEGIN_FTR_SECTION + ld r9, HSTATE_KVM_VCPU(r13) + bl kvmppc_save_tm +END_FTR_SECTION_IFSET(CPU_FTR_TM) +#endif + /* * Set DEC to the smaller of DEC and HDEC, so that we wake * no later than the end of our timeslice (HDEC interrupts @@ -2321,6 +2120,12 @@ kvm_end_cede: bl kvmhv_accumulate_time #endif +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +BEGIN_FTR_SECTION + bl kvmppc_restore_tm +END_FTR_SECTION_IFSET(CPU_FTR_TM) +#endif + /* load up FP state */ bl kvmppc_load_fp @@ -2629,6 +2434,243 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) mr r4,r31 blr +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +/* + * Save transactional state and TM-related registers. + * Called with r9 pointing to the vcpu struct. + * This can modify all checkpointed registers, but + * restores r1, r2 and r9 (vcpu pointer) before exit. + */ +kvmppc_save_tm: + mflr r0 + std r0, PPC_LR_STKOFF(r1) + + /* Turn on TM. */ + mfmsr r8 + li r0, 1 + rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG + mtmsrd r8 + + ld r5, VCPU_MSR(r9) + rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 + beq 1f /* TM not active in guest. */ + + std r1, HSTATE_HOST_R1(r13) + li r3, TM_CAUSE_KVM_RESCHED + + /* Clear the MSR RI since r1, r13 are all going to be foobar. */ + li r5, 0 + mtmsrd r5, 1 + + /* All GPRs are volatile at this point. */ + TRECLAIM(R3) + + /* Temporarily store r13 and r9 so we have some regs to play with */ + SET_SCRATCH0(r13) + GET_PACA(r13) + std r9, PACATMSCRATCH(r13) + ld r9, HSTATE_KVM_VCPU(r13) + + /* Get a few more GPRs free. */ + std r29, VCPU_GPRS_TM(29)(r9) + std r30, VCPU_GPRS_TM(30)(r9) + std r31, VCPU_GPRS_TM(31)(r9) + + /* Save away PPR and DSCR soon so don't run with user values. */ + mfspr r31, SPRN_PPR + HMT_MEDIUM + mfspr r30, SPRN_DSCR + ld r29, HSTATE_DSCR(r13) + mtspr SPRN_DSCR, r29 + + /* Save all but r9, r13 & r29-r31 */ + reg = 0 + .rept 29 + .if (reg != 9) && (reg != 13) + std reg, VCPU_GPRS_TM(reg)(r9) + .endif + reg = reg + 1 + .endr + /* ... now save r13 */ + GET_SCRATCH0(r4) + std r4, VCPU_GPRS_TM(13)(r9) + /* ... and save r9 */ + ld r4, PACATMSCRATCH(r13) + std r4, VCPU_GPRS_TM(9)(r9) + + /* Reload stack pointer and TOC. */ + ld r1, HSTATE_HOST_R1(r13) + ld r2, PACATOC(r13) + + /* Set MSR RI now we have r1 and r13 back. */ + li r5, MSR_RI + mtmsrd r5, 1 + + /* Save away checkpinted SPRs. */ + std r31, VCPU_PPR_TM(r9) + std r30, VCPU_DSCR_TM(r9) + mflr r5 + mfcr r6 + mfctr r7 + mfspr r8, SPRN_AMR + mfspr r10, SPRN_TAR + mfxer r11 + std r5, VCPU_LR_TM(r9) + stw r6, VCPU_CR_TM(r9) + std r7, VCPU_CTR_TM(r9) + std r8, VCPU_AMR_TM(r9) + std r10, VCPU_TAR_TM(r9) + std r11, VCPU_XER_TM(r9) + + /* Restore r12 as trap number. */ + lwz r12, VCPU_TRAP(r9) + + /* Save FP/VSX. */ + addi r3, r9, VCPU_FPRS_TM + bl store_fp_state + addi r3, r9, VCPU_VRS_TM + bl store_vr_state + mfspr r6, SPRN_VRSAVE + stw r6, VCPU_VRSAVE_TM(r9) +1: + /* + * We need to save these SPRs after the treclaim so that the software + * error code is recorded correctly in the TEXASR. Also the user may + * change these outside of a transaction, so they must always be + * context switched. + */ + mfspr r5, SPRN_TFHAR + mfspr r6, SPRN_TFIAR + mfspr r7, SPRN_TEXASR + std r5, VCPU_TFHAR(r9) + std r6, VCPU_TFIAR(r9) + std r7, VCPU_TEXASR(r9) + + ld r0, PPC_LR_STKOFF(r1) + mtlr r0 + blr + +/* + * Restore transactional state and TM-related registers. + * Called with r4 pointing to the vcpu struct. + * This potentially modifies all checkpointed registers. + * It restores r1, r2, r4 from the PACA. + */ +kvmppc_restore_tm: + mflr r0 + std r0, PPC_LR_STKOFF(r1) + + /* Turn on TM/FP/VSX/VMX so we can restore them. */ + mfmsr r5 + li r6, MSR_TM >> 32 + sldi r6, r6, 32 + or r5, r5, r6 + ori r5, r5, MSR_FP + oris r5, r5, (MSR_VEC | MSR_VSX)@h + mtmsrd r5 + + /* + * The user may change these outside of a transaction, so they must + * always be context switched. + */ + ld r5, VCPU_TFHAR(r4) + ld r6, VCPU_TFIAR(r4) + ld r7, VCPU_TEXASR(r4) + mtspr SPRN_TFHAR, r5 + mtspr SPRN_TFIAR, r6 + mtspr SPRN_TEXASR, r7 + + ld r5, VCPU_MSR(r4) + rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 + beqlr /* TM not active in guest */ + std r1, HSTATE_HOST_R1(r13) + + /* Make sure the failure summary is set, otherwise we'll program check + * when we trechkpt. It's possible that this might have been not set + * on a kvmppc_set_one_reg() call but we shouldn't let this crash the + * host. + */ + oris r7, r7, (TEXASR_FS)@h + mtspr SPRN_TEXASR, r7 + + /* + * We need to load up the checkpointed state for the guest. + * We need to do this early as it will blow away any GPRs, VSRs and + * some SPRs. + */ + + mr r31, r4 + addi r3, r31, VCPU_FPRS_TM + bl load_fp_state + addi r3, r31, VCPU_VRS_TM + bl load_vr_state + mr r4, r31 + lwz r7, VCPU_VRSAVE_TM(r4) + mtspr SPRN_VRSAVE, r7 + + ld r5, VCPU_LR_TM(r4) + lwz r6, VCPU_CR_TM(r4) + ld r7, VCPU_CTR_TM(r4) + ld r8, VCPU_AMR_TM(r4) + ld r9, VCPU_TAR_TM(r4) + ld r10, VCPU_XER_TM(r4) + mtlr r5 + mtcr r6 + mtctr r7 + mtspr SPRN_AMR, r8 + mtspr SPRN_TAR, r9 + mtxer r10 + + /* + * Load up PPR and DSCR values but don't put them in the actual SPRs + * till the last moment to avoid running with userspace PPR and DSCR for + * too long. + */ + ld r29, VCPU_DSCR_TM(r4) + ld r30, VCPU_PPR_TM(r4) + + std r2, PACATMSCRATCH(r13) /* Save TOC */ + + /* Clear the MSR RI since r1, r13 are all going to be foobar. */ + li r5, 0 + mtmsrd r5, 1 + + /* Load GPRs r0-r28 */ + reg = 0 + .rept 29 + ld reg, VCPU_GPRS_TM(reg)(r31) + reg = reg + 1 + .endr + + mtspr SPRN_DSCR, r29 + mtspr SPRN_PPR, r30 + + /* Load final GPRs */ + ld 29, VCPU_GPRS_TM(29)(r31) + ld 30, VCPU_GPRS_TM(30)(r31) + ld 31, VCPU_GPRS_TM(31)(r31) + + /* TM checkpointed state is now setup. All GPRs are now volatile. */ + TRECHKPT + + /* Now let's get back the state we need. */ + HMT_MEDIUM + GET_PACA(r13) + ld r29, HSTATE_DSCR(r13) + mtspr SPRN_DSCR, r29 + ld r4, HSTATE_KVM_VCPU(r13) + ld r1, HSTATE_HOST_R1(r13) + ld r2, PACATMSCRATCH(r13) + + /* Set the MSR RI since we have our registers back. */ + li r5, MSR_RI + mtmsrd r5, 1 + + ld r0, PPC_LR_STKOFF(r1) + mtlr r0 + blr +#endif + /* * We come here if we get any exception or interrupt while we are * executing host real mode code while in guest MMU context. diff --git a/kernel/arch/powerpc/kvm/booke.c b/kernel/arch/powerpc/kvm/booke.c index fd5875179..6d63cd67b 100644 --- a/kernel/arch/powerpc/kvm/booke.c +++ b/kernel/arch/powerpc/kvm/booke.c @@ -2033,7 +2033,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, if (type == KVMPPC_DEBUG_NONE) continue; - if (type & !(KVMPPC_DEBUG_WATCH_READ | + if (type & ~(KVMPPC_DEBUG_WATCH_READ | KVMPPC_DEBUG_WATCH_WRITE | KVMPPC_DEBUG_BREAKPOINT)) return -EINVAL; diff --git a/kernel/arch/powerpc/lib/copyuser_64.S b/kernel/arch/powerpc/lib/copyuser_64.S index f09899e35..7b22624f3 100644 --- a/kernel/arch/powerpc/lib/copyuser_64.S +++ b/kernel/arch/powerpc/lib/copyuser_64.S @@ -359,6 +359,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) addi r3,r3,8 171: 177: +179: addi r3,r3,8 370: 372: @@ -373,7 +374,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) 173: 174: 175: -179: 181: 184: 186: diff --git a/kernel/arch/powerpc/mm/copro_fault.c b/kernel/arch/powerpc/mm/copro_fault.c index 6527882ce..ddfd2740a 100644 --- a/kernel/arch/powerpc/mm/copro_fault.c +++ b/kernel/arch/powerpc/mm/copro_fault.c @@ -106,6 +106,8 @@ int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb) switch (REGION_ID(ea)) { case USER_REGION_ID: pr_devel("%s: 0x%llx -- USER_REGION_ID\n", __func__, ea); + if (mm == NULL) + return 1; psize = get_slice_psize(mm, ea); ssize = user_segment_size(ea); vsid = get_vsid(mm->context.id, ea, ssize); diff --git a/kernel/arch/powerpc/mm/hugetlbpage.c b/kernel/arch/powerpc/mm/hugetlbpage.c index 9833fee49..807f15947 100644 --- a/kernel/arch/powerpc/mm/hugetlbpage.c +++ b/kernel/arch/powerpc/mm/hugetlbpage.c @@ -486,13 +486,13 @@ static void hugepd_free(struct mmu_gather *tlb, void *hugepte) { struct hugepd_freelist **batchp; - batchp = this_cpu_ptr(&hugepd_freelist_cur); + batchp = &get_cpu_var(hugepd_freelist_cur); if (atomic_read(&tlb->mm->mm_users) < 2 || cpumask_equal(mm_cpumask(tlb->mm), cpumask_of(smp_processor_id()))) { kmem_cache_free(hugepte_cache, hugepte); - put_cpu_var(hugepd_freelist_cur); + put_cpu_var(hugepd_freelist_cur); return; } diff --git a/kernel/arch/powerpc/mm/slb_low.S b/kernel/arch/powerpc/mm/slb_low.S index 736d18b3c..4c48b4876 100644 --- a/kernel/arch/powerpc/mm/slb_low.S +++ b/kernel/arch/powerpc/mm/slb_low.S @@ -113,7 +113,12 @@ BEGIN_FTR_SECTION END_MMU_FTR_SECTION_IFCLR(MMU_FTR_1T_SEGMENT) b slb_finish_load_1T -0: +0: /* + * For userspace addresses, make sure this is region 0. + */ + cmpdi r9, 0 + bne 8f + /* when using slices, we extract the psize off the slice bitmaps * and then we need to get the sllp encoding off the mmu_psize_defs * array. diff --git a/kernel/arch/powerpc/platforms/powernv/eeh-powernv.c b/kernel/arch/powerpc/platforms/powernv/eeh-powernv.c index 2ba602591..92736851c 100644 --- a/kernel/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/kernel/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -956,6 +956,11 @@ static int pnv_eeh_reset(struct eeh_pe *pe, int option) } bus = eeh_pe_bus_get(pe); + if (!bus) { + pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n", + __func__, pe->phb->global_number, pe->addr); + return -EIO; + } if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) ret = pnv_eeh_root_reset(hose, option); @@ -1163,7 +1168,7 @@ static void pnv_eeh_get_and_dump_hub_diag(struct pci_controller *hose) return; } - switch (data->type) { + switch (be16_to_cpu(data->type)) { case OPAL_P7IOC_DIAG_TYPE_RGC: pr_info("P7IOC diag-data for RGC\n\n"); pnv_eeh_dump_hub_diag_common(data); @@ -1395,7 +1400,7 @@ static int pnv_eeh_next_error(struct eeh_pe **pe) /* Try best to clear it */ opal_pci_eeh_freeze_clear(phb->opal_id, - frozen_pe_no, + be64_to_cpu(frozen_pe_no), OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); ret = EEH_NEXT_ERR_NONE; } else if ((*pe)->state & EEH_PE_ISOLATED || diff --git a/kernel/arch/powerpc/platforms/powernv/opal-dump.c b/kernel/arch/powerpc/platforms/powernv/opal-dump.c index 2ee96431f..4c827826c 100644 --- a/kernel/arch/powerpc/platforms/powernv/opal-dump.c +++ b/kernel/arch/powerpc/platforms/powernv/opal-dump.c @@ -370,6 +370,7 @@ static irqreturn_t process_dump(int irq, void *data) uint32_t dump_id, dump_size, dump_type; struct dump_obj *dump; char name[22]; + struct kobject *kobj; rc = dump_read_info(&dump_id, &dump_size, &dump_type); if (rc != OPAL_SUCCESS) @@ -381,8 +382,12 @@ static irqreturn_t process_dump(int irq, void *data) * that gracefully and not create two conflicting * entries. */ - if (kset_find_obj(dump_kset, name)) + kobj = kset_find_obj(dump_kset, name); + if (kobj) { + /* Drop reference added by kset_find_obj() */ + kobject_put(kobj); return 0; + } dump = create_dump_obj(dump_id, dump_size, dump_type); if (!dump) diff --git a/kernel/arch/powerpc/platforms/powernv/opal-elog.c b/kernel/arch/powerpc/platforms/powernv/opal-elog.c index 37f959bf3..f2344cbd2 100644 --- a/kernel/arch/powerpc/platforms/powernv/opal-elog.c +++ b/kernel/arch/powerpc/platforms/powernv/opal-elog.c @@ -247,6 +247,7 @@ static irqreturn_t elog_event(int irq, void *data) uint64_t elog_type; int rc; char name[2+16+1]; + struct kobject *kobj; rc = opal_get_elog_size(&id, &size, &type); if (rc != OPAL_SUCCESS) { @@ -269,8 +270,12 @@ static irqreturn_t elog_event(int irq, void *data) * that gracefully and not create two conflicting * entries. */ - if (kset_find_obj(elog_kset, name)) + kobj = kset_find_obj(elog_kset, name); + if (kobj) { + /* Drop reference added by kset_find_obj() */ + kobject_put(kobj); return IRQ_HANDLED; + } create_elog_obj(log_id, elog_size, elog_type); diff --git a/kernel/arch/powerpc/platforms/powernv/pci.c b/kernel/arch/powerpc/platforms/powernv/pci.c index ad8c3f4a5..dd5e0f3b1 100644 --- a/kernel/arch/powerpc/platforms/powernv/pci.c +++ b/kernel/arch/powerpc/platforms/powernv/pci.c @@ -197,8 +197,8 @@ static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose, be64_to_cpu(data->dma1ErrorLog1)); for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) { - if ((data->pestA[i] >> 63) == 0 && - (data->pestB[i] >> 63) == 0) + if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 && + (be64_to_cpu(data->pestB[i]) >> 63) == 0) continue; pr_info("PE[%3d] A/B: %016llx %016llx\n", diff --git a/kernel/arch/powerpc/platforms/pseries/eeh_pseries.c b/kernel/arch/powerpc/platforms/pseries/eeh_pseries.c index ac3ffd97e..405baaf96 100644 --- a/kernel/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/kernel/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -615,29 +615,50 @@ static int pseries_eeh_configure_bridge(struct eeh_pe *pe) { int config_addr; int ret; + /* Waiting 0.2s maximum before skipping configuration */ + int max_wait = 200; /* Figure out the PE address */ config_addr = pe->config_addr; if (pe->addr) config_addr = pe->addr; - /* Use new configure-pe function, if supported */ - if (ibm_configure_pe != RTAS_UNKNOWN_SERVICE) { - ret = rtas_call(ibm_configure_pe, 3, 1, NULL, - config_addr, BUID_HI(pe->phb->buid), - BUID_LO(pe->phb->buid)); - } else if (ibm_configure_bridge != RTAS_UNKNOWN_SERVICE) { - ret = rtas_call(ibm_configure_bridge, 3, 1, NULL, - config_addr, BUID_HI(pe->phb->buid), - BUID_LO(pe->phb->buid)); - } else { - return -EFAULT; - } + while (max_wait > 0) { + /* Use new configure-pe function, if supported */ + if (ibm_configure_pe != RTAS_UNKNOWN_SERVICE) { + ret = rtas_call(ibm_configure_pe, 3, 1, NULL, + config_addr, BUID_HI(pe->phb->buid), + BUID_LO(pe->phb->buid)); + } else if (ibm_configure_bridge != RTAS_UNKNOWN_SERVICE) { + ret = rtas_call(ibm_configure_bridge, 3, 1, NULL, + config_addr, BUID_HI(pe->phb->buid), + BUID_LO(pe->phb->buid)); + } else { + return -EFAULT; + } - if (ret) - pr_warn("%s: Unable to configure bridge PHB#%d-PE#%x (%d)\n", - __func__, pe->phb->global_number, pe->addr, ret); + if (!ret) + return ret; + + /* + * If RTAS returns a delay value that's above 100ms, cut it + * down to 100ms in case firmware made a mistake. For more + * on how these delay values work see rtas_busy_delay_time + */ + if (ret > RTAS_EXTENDED_DELAY_MIN+2 && + ret <= RTAS_EXTENDED_DELAY_MAX) + ret = RTAS_EXTENDED_DELAY_MIN+2; + + max_wait -= rtas_busy_delay_time(ret); + + if (max_wait < 0) + break; + + rtas_busy_delay(ret); + } + pr_warn("%s: Unable to configure bridge PHB#%d-PE#%x (%d)\n", + __func__, pe->phb->global_number, pe->addr, ret); return ret; } diff --git a/kernel/arch/powerpc/platforms/pseries/iommu.c b/kernel/arch/powerpc/platforms/pseries/iommu.c index bd98ce2be..3e8865b18 100644 --- a/kernel/arch/powerpc/platforms/pseries/iommu.c +++ b/kernel/arch/powerpc/platforms/pseries/iommu.c @@ -912,7 +912,8 @@ machine_arch_initcall(pseries, find_existing_ddw_windows); static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail, struct ddw_query_response *query) { - struct eeh_dev *edev; + struct device_node *dn; + struct pci_dn *pdn; u32 cfg_addr; u64 buid; int ret; @@ -923,11 +924,10 @@ static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail, * Retrieve them from the pci device, not the node with the * dma-window property */ - edev = pci_dev_to_eeh_dev(dev); - cfg_addr = edev->config_addr; - if (edev->pe_config_addr) - cfg_addr = edev->pe_config_addr; - buid = edev->phb->buid; + dn = pci_device_to_OF_node(dev); + pdn = PCI_DN(dn); + buid = pdn->phb->buid; + cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query, cfg_addr, BUID_HI(buid), BUID_LO(buid)); @@ -941,7 +941,8 @@ static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail, struct ddw_create_response *create, int page_shift, int window_shift) { - struct eeh_dev *edev; + struct device_node *dn; + struct pci_dn *pdn; u32 cfg_addr; u64 buid; int ret; @@ -952,11 +953,10 @@ static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail, * Retrieve them from the pci device, not the node with the * dma-window property */ - edev = pci_dev_to_eeh_dev(dev); - cfg_addr = edev->config_addr; - if (edev->pe_config_addr) - cfg_addr = edev->pe_config_addr; - buid = edev->phb->buid; + dn = pci_device_to_OF_node(dev); + pdn = PCI_DN(dn); + buid = pdn->phb->buid; + cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8)); do { /* extra outputs are LIOBN and dma-addr (hi, lo) */ diff --git a/kernel/arch/powerpc/platforms/pseries/lpar.c b/kernel/arch/powerpc/platforms/pseries/lpar.c index b7a67e3d2..3ae432824 100644 --- a/kernel/arch/powerpc/platforms/pseries/lpar.c +++ b/kernel/arch/powerpc/platforms/pseries/lpar.c @@ -406,7 +406,7 @@ static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot, unsigned long *vpn, int count, int psize, int ssize) { - unsigned long param[8]; + unsigned long param[PLPAR_HCALL9_BUFSIZE]; int i = 0, pix = 0, rc; unsigned long flags = 0; int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); @@ -523,7 +523,7 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local) unsigned long flags = 0; struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); - unsigned long param[9]; + unsigned long param[PLPAR_HCALL9_BUFSIZE]; unsigned long hash, index, shift, hidx, slot; real_pte_t pte; int psize, ssize; diff --git a/kernel/arch/s390/crypto/prng.c b/kernel/arch/s390/crypto/prng.c index b8045b97f..683a966b5 100644 --- a/kernel/arch/s390/crypto/prng.c +++ b/kernel/arch/s390/crypto/prng.c @@ -565,8 +565,10 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf, prng_data->prngws.byte_counter += n; prng_data->prngws.reseed_counter += n; - if (copy_to_user(ubuf, prng_data->buf, chunk)) - return -EFAULT; + if (copy_to_user(ubuf, prng_data->buf, chunk)) { + ret = -EFAULT; + break; + } nbytes -= chunk; ret += chunk; @@ -669,11 +671,13 @@ static const struct file_operations prng_tdes_fops = { static struct miscdevice prng_sha512_dev = { .name = "prandom", .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, .fops = &prng_sha512_fops, }; static struct miscdevice prng_tdes_dev = { .name = "prandom", .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, .fops = &prng_tdes_fops, }; diff --git a/kernel/arch/s390/hypfs/hypfs_diag.c b/kernel/arch/s390/hypfs/hypfs_diag.c index 045035796..b63b9a42a 100644 --- a/kernel/arch/s390/hypfs/hypfs_diag.c +++ b/kernel/arch/s390/hypfs/hypfs_diag.c @@ -525,11 +525,11 @@ static int diag224(void *ptr) static int diag224_get_name_table(void) { /* memory must be below 2GB */ - diag224_cpu_names = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); + diag224_cpu_names = (char *) __get_free_page(GFP_KERNEL | GFP_DMA); if (!diag224_cpu_names) return -ENOMEM; if (diag224(diag224_cpu_names)) { - kfree(diag224_cpu_names); + free_page((unsigned long) diag224_cpu_names); return -EOPNOTSUPP; } EBCASC(diag224_cpu_names + 16, (*diag224_cpu_names + 1) * 16); @@ -538,7 +538,7 @@ static int diag224_get_name_table(void) static void diag224_delete_name_table(void) { - kfree(diag224_cpu_names); + free_page((unsigned long) diag224_cpu_names); } static int diag224_idx2name(int index, char *name) diff --git a/kernel/arch/s390/include/asm/fpu/api.h b/kernel/arch/s390/include/asm/fpu/api.h index 5e04f3cbd..8ae236b0f 100644 --- a/kernel/arch/s390/include/asm/fpu/api.h +++ b/kernel/arch/s390/include/asm/fpu/api.h @@ -22,7 +22,7 @@ static inline int test_fp_ctl(u32 fpc) " la %0,0\n" "1:\n" EX_TABLE(0b,1b) - : "=d" (rc), "=d" (orig_fpc) + : "=d" (rc), "=&d" (orig_fpc) : "d" (fpc), "0" (-EINVAL)); return rc; } diff --git a/kernel/arch/s390/include/asm/mmu.h b/kernel/arch/s390/include/asm/mmu.h index d29ad9545..081b2ad99 100644 --- a/kernel/arch/s390/include/asm/mmu.h +++ b/kernel/arch/s390/include/asm/mmu.h @@ -11,7 +11,7 @@ typedef struct { spinlock_t list_lock; struct list_head pgtable_list; struct list_head gmap_list; - unsigned long asce_bits; + unsigned long asce; unsigned long asce_limit; unsigned long vdso_base; /* The mmu context allocates 4K page tables. */ diff --git a/kernel/arch/s390/include/asm/mmu_context.h b/kernel/arch/s390/include/asm/mmu_context.h index e485817f7..22877c944 100644 --- a/kernel/arch/s390/include/asm/mmu_context.h +++ b/kernel/arch/s390/include/asm/mmu_context.h @@ -26,12 +26,28 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.has_pgste = 0; mm->context.use_skey = 0; #endif - if (mm->context.asce_limit == 0) { + switch (mm->context.asce_limit) { + case 1UL << 42: + /* + * forked 3-level task, fall through to set new asce with new + * mm->pgd + */ + case 0: /* context created by exec, set asce limit to 4TB */ - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | _ASCE_TYPE_REGION3; mm->context.asce_limit = STACK_TOP_MAX; - } else if (mm->context.asce_limit == (1UL << 31)) { + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION3; + break; + case 1UL << 53: + /* forked 4-level task, set new asce with new mm->pgd */ + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + break; + case 1UL << 31: + /* forked 2-level compat task, set new asce with new mm->pgd */ + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; + /* pgd_alloc() did not increase mm->nr_pmds */ mm_inc_nr_pmds(mm); } crst_table_init((unsigned long *) mm->pgd, pgd_entry_type(mm)); @@ -42,7 +58,7 @@ static inline int init_new_context(struct task_struct *tsk, static inline void set_user_asce(struct mm_struct *mm) { - S390_lowcore.user_asce = mm->context.asce_bits | __pa(mm->pgd); + S390_lowcore.user_asce = mm->context.asce; if (current->thread.mm_segment.ar4) __ctl_load(S390_lowcore.user_asce, 7, 7); set_cpu_flag(CIF_ASCE); @@ -71,7 +87,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, { int cpu = smp_processor_id(); - S390_lowcore.user_asce = next->context.asce_bits | __pa(next->pgd); + S390_lowcore.user_asce = next->context.asce; if (prev == next) return; if (MACHINE_HAS_TLB_LC) diff --git a/kernel/arch/s390/include/asm/pci.h b/kernel/arch/s390/include/asm/pci.h index c873e682b..6dafabb6a 100644 --- a/kernel/arch/s390/include/asm/pci.h +++ b/kernel/arch/s390/include/asm/pci.h @@ -45,7 +45,8 @@ struct zpci_fmb { u64 rpcit_ops; u64 dma_rbytes; u64 dma_wbytes; -} __packed __aligned(16); + u64 pad[2]; +} __packed __aligned(128); enum zpci_state { ZPCI_FN_STATE_RESERVED, diff --git a/kernel/arch/s390/include/asm/pci_dma.h b/kernel/arch/s390/include/asm/pci_dma.h index 1aac41e83..92df3eb8d 100644 --- a/kernel/arch/s390/include/asm/pci_dma.h +++ b/kernel/arch/s390/include/asm/pci_dma.h @@ -23,6 +23,8 @@ enum zpci_ioat_dtype { #define ZPCI_IOTA_FS_2G 2 #define ZPCI_KEY (PAGE_DEFAULT_KEY << 5) +#define ZPCI_TABLE_SIZE_RT (1UL << 42) + #define ZPCI_IOTA_STO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_ST) #define ZPCI_IOTA_RTTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RT) #define ZPCI_IOTA_RSTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RS) diff --git a/kernel/arch/s390/include/asm/pgalloc.h b/kernel/arch/s390/include/asm/pgalloc.h index d7cc79fb6..5991cdcb5 100644 --- a/kernel/arch/s390/include/asm/pgalloc.h +++ b/kernel/arch/s390/include/asm/pgalloc.h @@ -56,8 +56,8 @@ static inline unsigned long pgd_entry_type(struct mm_struct *mm) return _REGION2_ENTRY_EMPTY; } -int crst_table_upgrade(struct mm_struct *, unsigned long limit); -void crst_table_downgrade(struct mm_struct *, unsigned long limit); +int crst_table_upgrade(struct mm_struct *); +void crst_table_downgrade(struct mm_struct *); static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) { diff --git a/kernel/arch/s390/include/asm/processor.h b/kernel/arch/s390/include/asm/processor.h index b16c3d0a1..c1ea67db8 100644 --- a/kernel/arch/s390/include/asm/processor.h +++ b/kernel/arch/s390/include/asm/processor.h @@ -163,7 +163,7 @@ extern __vector128 init_task_fpu_regs[__NUM_VXRS]; regs->psw.mask = PSW_USER_BITS | PSW_MASK_BA; \ regs->psw.addr = new_psw | PSW_ADDR_AMODE; \ regs->gprs[15] = new_stackp; \ - crst_table_downgrade(current->mm, 1UL << 31); \ + crst_table_downgrade(current->mm); \ execve_tail(); \ } while (0) diff --git a/kernel/arch/s390/include/asm/tlbflush.h b/kernel/arch/s390/include/asm/tlbflush.h index ca148f7c3..0a2031618 100644 --- a/kernel/arch/s390/include/asm/tlbflush.h +++ b/kernel/arch/s390/include/asm/tlbflush.h @@ -81,7 +81,8 @@ static inline void __tlb_flush_full(struct mm_struct *mm) } /* - * Flush TLB entries for a specific ASCE on all CPUs. + * Flush TLB entries for a specific ASCE on all CPUs. Should never be used + * when more than one asce (e.g. gmap) ran on this mm. */ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) { @@ -110,8 +111,7 @@ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) static inline void __tlb_flush_kernel(void) { if (MACHINE_HAS_IDTE) - __tlb_flush_idte((unsigned long) init_mm.pgd | - init_mm.context.asce_bits); + __tlb_flush_idte(init_mm.context.asce); else __tlb_flush_global(); } @@ -133,8 +133,7 @@ static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) static inline void __tlb_flush_kernel(void) { if (MACHINE_HAS_TLB_LC) - __tlb_flush_idte_local((unsigned long) init_mm.pgd | - init_mm.context.asce_bits); + __tlb_flush_idte_local(init_mm.context.asce); else __tlb_flush_local(); } @@ -148,8 +147,7 @@ static inline void __tlb_flush_mm(struct mm_struct * mm) * only ran on the local cpu. */ if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) - __tlb_flush_asce(mm, (unsigned long) mm->pgd | - mm->context.asce_bits); + __tlb_flush_asce(mm, mm->context.asce); else __tlb_flush_full(mm); } diff --git a/kernel/arch/s390/include/asm/uaccess.h b/kernel/arch/s390/include/asm/uaccess.h index 9dd4cc47d..5c7381c5a 100644 --- a/kernel/arch/s390/include/asm/uaccess.h +++ b/kernel/arch/s390/include/asm/uaccess.h @@ -215,28 +215,28 @@ int __put_user_bad(void) __attribute__((noreturn)); __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: { \ - unsigned char __x; \ + unsigned char __x = 0; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 2: { \ - unsigned short __x; \ + unsigned short __x = 0; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 4: { \ - unsigned int __x; \ + unsigned int __x = 0; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 8: { \ - unsigned long long __x; \ + unsigned long long __x = 0; \ __gu_err = __get_user_fn(&__x, ptr, \ sizeof(*(ptr))); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ diff --git a/kernel/arch/s390/kernel/entry.S b/kernel/arch/s390/kernel/entry.S index 857b6526d..424e6809a 100644 --- a/kernel/arch/s390/kernel/entry.S +++ b/kernel/arch/s390/kernel/entry.S @@ -1197,114 +1197,12 @@ cleanup_critical: .quad .Lpsw_idle_lpsw .Lcleanup_save_fpu_regs: - TSTMSK __LC_CPU_FLAGS,_CIF_FPU - bor %r14 - clg %r9,BASED(.Lcleanup_save_fpu_regs_done) - jhe 5f - clg %r9,BASED(.Lcleanup_save_fpu_regs_fp) - jhe 4f - clg %r9,BASED(.Lcleanup_save_fpu_regs_vx_high) - jhe 3f - clg %r9,BASED(.Lcleanup_save_fpu_regs_vx_low) - jhe 2f - clg %r9,BASED(.Lcleanup_save_fpu_fpc_end) - jhe 1f - lg %r2,__LC_CURRENT - aghi %r2,__TASK_thread -0: # Store floating-point controls - stfpc __THREAD_FPU_fpc(%r2) -1: # Load register save area and check if VX is active - lg %r3,__THREAD_FPU_regs(%r2) - TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX - jz 4f # no VX -> store FP regs -2: # Store vector registers (V0-V15) - VSTM %v0,%v15,0,%r3 # vstm 0,15,0(3) -3: # Store vector registers (V16-V31) - VSTM %v16,%v31,256,%r3 # vstm 16,31,256(3) - j 5f # -> done, set CIF_FPU flag -4: # Store floating-point registers - std 0,0(%r3) - std 1,8(%r3) - std 2,16(%r3) - std 3,24(%r3) - std 4,32(%r3) - std 5,40(%r3) - std 6,48(%r3) - std 7,56(%r3) - std 8,64(%r3) - std 9,72(%r3) - std 10,80(%r3) - std 11,88(%r3) - std 12,96(%r3) - std 13,104(%r3) - std 14,112(%r3) - std 15,120(%r3) -5: # Set CIF_FPU flag - oi __LC_CPU_FLAGS+7,_CIF_FPU - lg %r9,48(%r11) # return from save_fpu_regs + larl %r9,save_fpu_regs br %r14 -.Lcleanup_save_fpu_fpc_end: - .quad .Lsave_fpu_regs_fpc_end -.Lcleanup_save_fpu_regs_vx_low: - .quad .Lsave_fpu_regs_vx_low -.Lcleanup_save_fpu_regs_vx_high: - .quad .Lsave_fpu_regs_vx_high -.Lcleanup_save_fpu_regs_fp: - .quad .Lsave_fpu_regs_fp -.Lcleanup_save_fpu_regs_done: - .quad .Lsave_fpu_regs_done .Lcleanup_load_fpu_regs: - TSTMSK __LC_CPU_FLAGS,_CIF_FPU - bnor %r14 - clg %r9,BASED(.Lcleanup_load_fpu_regs_done) - jhe 1f - clg %r9,BASED(.Lcleanup_load_fpu_regs_fp) - jhe 2f - clg %r9,BASED(.Lcleanup_load_fpu_regs_vx_high) - jhe 3f - clg %r9,BASED(.Lcleanup_load_fpu_regs_vx) - jhe 4f - lg %r4,__LC_CURRENT - aghi %r4,__TASK_thread - lfpc __THREAD_FPU_fpc(%r4) - TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX - lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area - jz 2f # -> no VX, load FP regs -4: # Load V0 ..V15 registers - VLM %v0,%v15,0,%r4 -3: # Load V16..V31 registers - VLM %v16,%v31,256,%r4 - j 1f -2: # Load floating-point registers - ld 0,0(%r4) - ld 1,8(%r4) - ld 2,16(%r4) - ld 3,24(%r4) - ld 4,32(%r4) - ld 5,40(%r4) - ld 6,48(%r4) - ld 7,56(%r4) - ld 8,64(%r4) - ld 9,72(%r4) - ld 10,80(%r4) - ld 11,88(%r4) - ld 12,96(%r4) - ld 13,104(%r4) - ld 14,112(%r4) - ld 15,120(%r4) -1: # Clear CIF_FPU bit - ni __LC_CPU_FLAGS+7,255-_CIF_FPU - lg %r9,48(%r11) # return from load_fpu_regs + larl %r9,load_fpu_regs br %r14 -.Lcleanup_load_fpu_regs_vx: - .quad .Lload_fpu_regs_vx -.Lcleanup_load_fpu_regs_vx_high: - .quad .Lload_fpu_regs_vx_high -.Lcleanup_load_fpu_regs_fp: - .quad .Lload_fpu_regs_fp -.Lcleanup_load_fpu_regs_done: - .quad .Lload_fpu_regs_done /* * Integer constants diff --git a/kernel/arch/s390/kernel/head64.S b/kernel/arch/s390/kernel/head64.S index 58b719fa8..1ad2407c7 100644 --- a/kernel/arch/s390/kernel/head64.S +++ b/kernel/arch/s390/kernel/head64.S @@ -16,7 +16,7 @@ __HEAD ENTRY(startup_continue) - tm __LC_STFL_FAC_LIST+6,0x80 # LPP available ? + tm __LC_STFL_FAC_LIST+5,0x80 # LPP available ? jz 0f xc __LC_LPP+1(7,0),__LC_LPP+1 # clear lpp and current_pid mvi __LC_LPP,0x80 # and set LPP_MAGIC diff --git a/kernel/arch/s390/kernel/ipl.c b/kernel/arch/s390/kernel/ipl.c index b1f0a90f9..42570d8fb 100644 --- a/kernel/arch/s390/kernel/ipl.c +++ b/kernel/arch/s390/kernel/ipl.c @@ -2070,13 +2070,6 @@ void s390_reset_system(void (*fn_pre)(void), S390_lowcore.program_new_psw.addr = PSW_ADDR_AMODE | (unsigned long) s390_base_pgm_handler; - /* - * Clear subchannel ID and number to signal new kernel that no CCW or - * SCSI IPL has been done (for kexec and kdump) - */ - S390_lowcore.subchannel_id = 0; - S390_lowcore.subchannel_nr = 0; - /* Store status at absolute zero */ store_status(); diff --git a/kernel/arch/s390/kernel/ptrace.c b/kernel/arch/s390/kernel/ptrace.c index 01c37b36c..02bd587b6 100644 --- a/kernel/arch/s390/kernel/ptrace.c +++ b/kernel/arch/s390/kernel/ptrace.c @@ -963,6 +963,11 @@ static int s390_fpregs_set(struct task_struct *target, if (target == current) save_fpu_regs(); + if (MACHINE_HAS_VX) + convert_vx_to_fp(fprs, target->thread.fpu.vxrs); + else + memcpy(&fprs, target->thread.fpu.fprs, sizeof(fprs)); + /* If setting FPC, must validate it first. */ if (count > 0 && pos < offsetof(s390_fp_regs, fprs)) { u32 ufpc[2] = { target->thread.fpu.fpc, 0 }; @@ -1067,6 +1072,9 @@ static int s390_vxrs_low_set(struct task_struct *target, if (target == current) save_fpu_regs(); + for (i = 0; i < __NUM_VXRS_LOW; i++) + vxrs[i] = *((__u64 *)(target->thread.fpu.vxrs + i) + 1); + rc = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vxrs, 0, -1); if (rc == 0) for (i = 0; i < __NUM_VXRS_LOW; i++) diff --git a/kernel/arch/s390/kernel/setup.c b/kernel/arch/s390/kernel/setup.c index c837bcacf..1f581eb61 100644 --- a/kernel/arch/s390/kernel/setup.c +++ b/kernel/arch/s390/kernel/setup.c @@ -329,6 +329,7 @@ static void __init setup_lowcore(void) + PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs); lc->current_task = (unsigned long) init_thread_union.thread_info.task; lc->thread_info = (unsigned long) &init_thread_union; + lc->lpp = LPP_MAGIC; lc->machine_flags = S390_lowcore.machine_flags; lc->stfl_fac_list = S390_lowcore.stfl_fac_list; memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, diff --git a/kernel/arch/s390/mm/init.c b/kernel/arch/s390/mm/init.c index c722400c7..feff9caf8 100644 --- a/kernel/arch/s390/mm/init.c +++ b/kernel/arch/s390/mm/init.c @@ -89,7 +89,8 @@ void __init paging_init(void) asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; pgd_type = _REGION3_ENTRY_EMPTY; } - S390_lowcore.kernel_asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; + init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; + S390_lowcore.kernel_asce = init_mm.context.asce; clear_table((unsigned long *) init_mm.pgd, pgd_type, sizeof(unsigned long)*2048); vmem_map_init(); diff --git a/kernel/arch/s390/mm/mmap.c b/kernel/arch/s390/mm/mmap.c index ea01477b4..f2b6b1d9c 100644 --- a/kernel/arch/s390/mm/mmap.c +++ b/kernel/arch/s390/mm/mmap.c @@ -174,7 +174,7 @@ int s390_mmap_check(unsigned long addr, unsigned long len, unsigned long flags) if (!(flags & MAP_FIXED)) addr = 0; if ((addr + len) >= TASK_SIZE) - return crst_table_upgrade(current->mm, 1UL << 53); + return crst_table_upgrade(current->mm); return 0; } @@ -191,7 +191,7 @@ s390_get_unmapped_area(struct file *filp, unsigned long addr, return area; if (area == -ENOMEM && !is_compat_task() && TASK_SIZE < (1UL << 53)) { /* Upgrade the page table to 4 levels and retry. */ - rc = crst_table_upgrade(mm, 1UL << 53); + rc = crst_table_upgrade(mm); if (rc) return (unsigned long) rc; area = arch_get_unmapped_area(filp, addr, len, pgoff, flags); @@ -213,7 +213,7 @@ s390_get_unmapped_area_topdown(struct file *filp, const unsigned long addr, return area; if (area == -ENOMEM && !is_compat_task() && TASK_SIZE < (1UL << 53)) { /* Upgrade the page table to 4 levels and retry. */ - rc = crst_table_upgrade(mm, 1UL << 53); + rc = crst_table_upgrade(mm); if (rc) return (unsigned long) rc; area = arch_get_unmapped_area_topdown(filp, addr, len, diff --git a/kernel/arch/s390/mm/pgtable.c b/kernel/arch/s390/mm/pgtable.c index 54ef3bc01..8345ae1f1 100644 --- a/kernel/arch/s390/mm/pgtable.c +++ b/kernel/arch/s390/mm/pgtable.c @@ -49,81 +49,52 @@ static void __crst_table_upgrade(void *arg) __tlb_flush_local(); } -int crst_table_upgrade(struct mm_struct *mm, unsigned long limit) +int crst_table_upgrade(struct mm_struct *mm) { unsigned long *table, *pgd; - unsigned long entry; - int flush; - BUG_ON(limit > (1UL << 53)); - flush = 0; -repeat: + /* upgrade should only happen from 3 to 4 levels */ + BUG_ON(mm->context.asce_limit != (1UL << 42)); + table = crst_table_alloc(mm); if (!table) return -ENOMEM; + spin_lock_bh(&mm->page_table_lock); - if (mm->context.asce_limit < limit) { - pgd = (unsigned long *) mm->pgd; - if (mm->context.asce_limit <= (1UL << 31)) { - entry = _REGION3_ENTRY_EMPTY; - mm->context.asce_limit = 1UL << 42; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION3; - } else { - entry = _REGION2_ENTRY_EMPTY; - mm->context.asce_limit = 1UL << 53; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION2; - } - crst_table_init(table, entry); - pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); - mm->pgd = (pgd_t *) table; - mm->task_size = mm->context.asce_limit; - table = NULL; - flush = 1; - } + pgd = (unsigned long *) mm->pgd; + crst_table_init(table, _REGION2_ENTRY_EMPTY); + pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd); + mm->pgd = (pgd_t *) table; + mm->context.asce_limit = 1UL << 53; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_REGION2; + mm->task_size = mm->context.asce_limit; spin_unlock_bh(&mm->page_table_lock); - if (table) - crst_table_free(mm, table); - if (mm->context.asce_limit < limit) - goto repeat; - if (flush) - on_each_cpu(__crst_table_upgrade, mm, 0); + + on_each_cpu(__crst_table_upgrade, mm, 0); return 0; } -void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) +void crst_table_downgrade(struct mm_struct *mm) { pgd_t *pgd; + /* downgrade should only happen from 3 to 2 levels (compat only) */ + BUG_ON(mm->context.asce_limit != (1UL << 42)); + if (current->active_mm == mm) { clear_user_asce(); __tlb_flush_mm(mm); } - while (mm->context.asce_limit > limit) { - pgd = mm->pgd; - switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) { - case _REGION_ENTRY_TYPE_R2: - mm->context.asce_limit = 1UL << 42; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_REGION3; - break; - case _REGION_ENTRY_TYPE_R3: - mm->context.asce_limit = 1UL << 31; - mm->context.asce_bits = _ASCE_TABLE_LENGTH | - _ASCE_USER_BITS | - _ASCE_TYPE_SEGMENT; - break; - default: - BUG(); - } - mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); - mm->task_size = mm->context.asce_limit; - crst_table_free(mm, (unsigned long *) pgd); - } + + pgd = mm->pgd; + mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); + mm->context.asce_limit = 1UL << 31; + mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | + _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; + mm->task_size = mm->context.asce_limit; + crst_table_free(mm, (unsigned long *) pgd); + if (current->active_mm == mm) set_user_asce(mm); } @@ -195,7 +166,7 @@ EXPORT_SYMBOL_GPL(gmap_alloc); static void gmap_flush_tlb(struct gmap *gmap) { if (MACHINE_HAS_IDTE) - __tlb_flush_asce(gmap->mm, gmap->asce); + __tlb_flush_idte(gmap->asce); else __tlb_flush_global(); } @@ -234,7 +205,7 @@ void gmap_free(struct gmap *gmap) /* Flush tlb. */ if (MACHINE_HAS_IDTE) - __tlb_flush_asce(gmap->mm, gmap->asce); + __tlb_flush_idte(gmap->asce); else __tlb_flush_global(); diff --git a/kernel/arch/s390/net/bpf_jit.h b/kernel/arch/s390/net/bpf_jit.h index f010c93a8..fda605dbc 100644 --- a/kernel/arch/s390/net/bpf_jit.h +++ b/kernel/arch/s390/net/bpf_jit.h @@ -37,7 +37,7 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; * | | | * +---------------+ | * | 8 byte skbp | | - * R15+170 -> +---------------+ | + * R15+176 -> +---------------+ | * | 8 byte hlen | | * R15+168 -> +---------------+ | * | 4 byte align | | @@ -58,7 +58,7 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; #define STK_OFF (STK_SPACE - STK_160_UNUSED) #define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */ #define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */ -#define STK_OFF_SKBP 170 /* Offset of SKB pointer on stack */ +#define STK_OFF_SKBP 176 /* Offset of SKB pointer on stack */ #define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */ #define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */ diff --git a/kernel/arch/s390/net/bpf_jit_comp.c b/kernel/arch/s390/net/bpf_jit_comp.c index 9a0c4c22e..0e2919dd8 100644 --- a/kernel/arch/s390/net/bpf_jit_comp.c +++ b/kernel/arch/s390/net/bpf_jit_comp.c @@ -45,7 +45,7 @@ struct bpf_jit { int labels[1]; /* Labels for local jumps */ }; -#define BPF_SIZE_MAX 0x7ffff /* Max size for program (20 bit signed displ) */ +#define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */ #define SEEN_SKB 1 /* skb access */ #define SEEN_MEM 2 /* use mem[] for temporary storage */ @@ -446,7 +446,7 @@ static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic) emit_load_skb_data_hlen(jit); if (jit->seen & SEEN_SKB_CHANGE) /* stg %b1,ST_OFF_SKBP(%r0,%r15) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, + EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_1, REG_0, REG_15, STK_OFF_SKBP); /* Clear A (%b0) and X (%b7) registers for converted BPF programs */ if (is_classic) { diff --git a/kernel/arch/s390/pci/pci.c b/kernel/arch/s390/pci/pci.c index 7ef12a3ac..f2f6720a3 100644 --- a/kernel/arch/s390/pci/pci.c +++ b/kernel/arch/s390/pci/pci.c @@ -701,8 +701,7 @@ static int zpci_restore(struct device *dev) goto out; zpci_map_resources(pdev); - zpci_register_ioat(zdev, 0, zdev->start_dma + PAGE_OFFSET, - zdev->start_dma + zdev->iommu_size - 1, + zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, (u64) zdev->dma_table); out: @@ -871,8 +870,11 @@ static inline int barsize(u8 size) static int zpci_mem_init(void) { + BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || + __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); + zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), - 16, 0, NULL); + __alignof__(struct zpci_fmb), 0, NULL); if (!zdev_fmb_cache) goto error_zdev; diff --git a/kernel/arch/s390/pci/pci_dma.c b/kernel/arch/s390/pci/pci_dma.c index d348f2c09..3a40f718b 100644 --- a/kernel/arch/s390/pci/pci_dma.c +++ b/kernel/arch/s390/pci/pci_dma.c @@ -458,7 +458,19 @@ int zpci_dma_init_device(struct zpci_dev *zdev) goto out_clean; } - zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET; + /* + * Restrict the iommu bitmap size to the minimum of the following: + * - main memory size + * - 3-level pagetable address limit minus start_dma offset + * - DMA address range allowed by the hardware (clp query pci fn) + * + * Also set zdev->end_dma to the actual end address of the usable + * range, instead of the theoretical maximum as reported by hardware. + */ + zdev->iommu_size = min3((u64) high_memory, + ZPCI_TABLE_SIZE_RT - zdev->start_dma, + zdev->end_dma - zdev->start_dma + 1); + zdev->end_dma = zdev->start_dma + zdev->iommu_size - 1; zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT; zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8); if (!zdev->iommu_bitmap) { @@ -466,10 +478,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev) goto out_reg; } - rc = zpci_register_ioat(zdev, - 0, - zdev->start_dma + PAGE_OFFSET, - zdev->start_dma + zdev->iommu_size - 1, + rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, (u64) zdev->dma_table); if (rc) goto out_reg; diff --git a/kernel/arch/score/include/asm/uaccess.h b/kernel/arch/score/include/asm/uaccess.h index 20a359122..01aec8ccd 100644 --- a/kernel/arch/score/include/asm/uaccess.h +++ b/kernel/arch/score/include/asm/uaccess.h @@ -163,7 +163,7 @@ do { \ __get_user_asm(val, "lw", ptr); \ break; \ case 8: \ - if ((copy_from_user((void *)&val, ptr, 8)) == 0) \ + if (__copy_from_user((void *)&val, ptr, 8) == 0) \ __gu_err = 0; \ else \ __gu_err = -EFAULT; \ @@ -188,6 +188,8 @@ do { \ \ if (likely(access_ok(VERIFY_READ, __gu_ptr, size))) \ __get_user_common((x), size, __gu_ptr); \ + else \ + (x) = 0; \ \ __gu_err; \ }) @@ -201,6 +203,7 @@ do { \ "2:\n" \ ".section .fixup,\"ax\"\n" \ "3:li %0, %4\n" \ + "li %1, 0\n" \ "j 2b\n" \ ".previous\n" \ ".section __ex_table,\"a\"\n" \ @@ -298,35 +301,34 @@ extern int __copy_tofrom_user(void *to, const void *from, unsigned long len); static inline unsigned long copy_from_user(void *to, const void *from, unsigned long len) { - unsigned long over; + unsigned long res = len; - if (access_ok(VERIFY_READ, from, len)) - return __copy_tofrom_user(to, from, len); + if (likely(access_ok(VERIFY_READ, from, len))) + res = __copy_tofrom_user(to, from, len); - if ((unsigned long)from < TASK_SIZE) { - over = (unsigned long)from + len - TASK_SIZE; - return __copy_tofrom_user(to, from, len - over) + over; - } - return len; + if (unlikely(res)) + memset(to + (len - res), 0, res); + + return res; } static inline unsigned long copy_to_user(void *to, const void *from, unsigned long len) { - unsigned long over; - - if (access_ok(VERIFY_WRITE, to, len)) - return __copy_tofrom_user(to, from, len); + if (likely(access_ok(VERIFY_WRITE, to, len))) + len = __copy_tofrom_user(to, from, len); - if ((unsigned long)to < TASK_SIZE) { - over = (unsigned long)to + len - TASK_SIZE; - return __copy_tofrom_user(to, from, len - over) + over; - } return len; } -#define __copy_from_user(to, from, len) \ - __copy_tofrom_user((to), (from), (len)) +static inline unsigned long +__copy_from_user(void *to, const void *from, unsigned long len) +{ + unsigned long left = __copy_tofrom_user(to, from, len); + if (unlikely(left)) + memset(to + (len - left), 0, left); + return left; +} #define __copy_to_user(to, from, len) \ __copy_tofrom_user((to), (from), (len)) @@ -340,17 +342,17 @@ __copy_to_user_inatomic(void *to, const void *from, unsigned long len) static inline unsigned long __copy_from_user_inatomic(void *to, const void *from, unsigned long len) { - return __copy_from_user(to, from, len); + return __copy_tofrom_user(to, from, len); } -#define __copy_in_user(to, from, len) __copy_from_user(to, from, len) +#define __copy_in_user(to, from, len) __copy_tofrom_user(to, from, len) static inline unsigned long copy_in_user(void *to, const void *from, unsigned long len) { if (access_ok(VERIFY_READ, from, len) && access_ok(VERFITY_WRITE, to, len)) - return copy_from_user(to, from, len); + return __copy_tofrom_user(to, from, len); } /* diff --git a/kernel/arch/sh/include/asm/uaccess.h b/kernel/arch/sh/include/asm/uaccess.h index a49635c51..92ade79ac 100644 --- a/kernel/arch/sh/include/asm/uaccess.h +++ b/kernel/arch/sh/include/asm/uaccess.h @@ -151,7 +151,10 @@ copy_from_user(void *to, const void __user *from, unsigned long n) __kernel_size_t __copy_size = (__kernel_size_t) n; if (__copy_size && __access_ok(__copy_from, __copy_size)) - return __copy_user(to, from, __copy_size); + __copy_size = __copy_user(to, from, __copy_size); + + if (unlikely(__copy_size)) + memset(to + (n - __copy_size), 0, __copy_size); return __copy_size; } diff --git a/kernel/arch/sh/include/asm/uaccess_64.h b/kernel/arch/sh/include/asm/uaccess_64.h index c01376c76..ca5073dd4 100644 --- a/kernel/arch/sh/include/asm/uaccess_64.h +++ b/kernel/arch/sh/include/asm/uaccess_64.h @@ -24,6 +24,7 @@ #define __get_user_size(x,ptr,size,retval) \ do { \ retval = 0; \ + x = 0; \ switch (size) { \ case 1: \ retval = __get_user_asm_b((void *)&x, \ diff --git a/kernel/arch/sh/mm/kmap.c b/kernel/arch/sh/mm/kmap.c index ec29e14ec..bf25d7c79 100644 --- a/kernel/arch/sh/mm/kmap.c +++ b/kernel/arch/sh/mm/kmap.c @@ -36,6 +36,7 @@ void *kmap_coherent(struct page *page, unsigned long addr) BUG_ON(!test_bit(PG_dcache_clean, &page->flags)); + preempt_disable(); pagefault_disable(); idx = FIX_CMAP_END - @@ -64,4 +65,5 @@ void kunmap_coherent(void *kvaddr) } pagefault_enable(); + preempt_enable(); } diff --git a/kernel/arch/sparc/include/asm/head_64.h b/kernel/arch/sparc/include/asm/head_64.h index 10e9dabc4..f0700cfee 100644 --- a/kernel/arch/sparc/include/asm/head_64.h +++ b/kernel/arch/sparc/include/asm/head_64.h @@ -15,6 +15,10 @@ #define PTREGS_OFF (STACK_BIAS + STACKFRAME_SZ) +#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) +#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) +#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) + #define __CHEETAH_ID 0x003e0014 #define __JALAPENO_ID 0x003e0016 #define __SERRANO_ID 0x003e0022 diff --git a/kernel/arch/sparc/include/asm/mmu_64.h b/kernel/arch/sparc/include/asm/mmu_64.h index 70067ce18..f7de0dbc3 100644 --- a/kernel/arch/sparc/include/asm/mmu_64.h +++ b/kernel/arch/sparc/include/asm/mmu_64.h @@ -92,7 +92,8 @@ struct tsb_config { typedef struct { spinlock_t lock; unsigned long sparc64_ctx_val; - unsigned long huge_pte_count; + unsigned long hugetlb_pte_count; + unsigned long thp_pte_count; struct tsb_config tsb_block[MM_NUM_TSBS]; struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; } mm_context_t; diff --git a/kernel/arch/sparc/include/asm/pgtable_64.h b/kernel/arch/sparc/include/asm/pgtable_64.h index 131d36fcd..408b715c9 100644 --- a/kernel/arch/sparc/include/asm/pgtable_64.h +++ b/kernel/arch/sparc/include/asm/pgtable_64.h @@ -375,7 +375,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) #define pgprot_noncached pgprot_noncached #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline pte_t pte_mkhuge(pte_t pte) +static inline unsigned long __pte_huge_mask(void) { unsigned long mask; @@ -390,8 +390,19 @@ static inline pte_t pte_mkhuge(pte_t pte) : "=r" (mask) : "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V)); - return __pte(pte_val(pte) | mask); + return mask; +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | __pte_huge_mask()); +} + +static inline bool is_hugetlb_pte(pte_t pte) +{ + return !!(pte_val(pte) & __pte_huge_mask()); } + #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline pmd_t pmd_mkhuge(pmd_t pmd) { @@ -403,6 +414,11 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) return __pmd(pte_val(pte)); } #endif +#else +static inline bool is_hugetlb_pte(pte_t pte) +{ + return false; +} #endif static inline pte_t pte_mkdirty(pte_t pte) @@ -865,6 +881,19 @@ static inline unsigned long pud_pfn(pud_t pud) void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t orig, int fullmm); +static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm) +{ + /* It is more efficient to let flush_tlb_kernel_range() + * handle init_mm tlb flushes. + * + * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U + * and SUN4V pte layout, so this inline test is fine. + */ + if (likely(mm != &init_mm) && pte_accessible(mm, orig)) + tlb_batch_add(mm, vaddr, ptep, orig, fullmm); +} + #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr, @@ -881,15 +910,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t orig = *ptep; *ptep = pte; - - /* It is more efficient to let flush_tlb_kernel_range() - * handle init_mm tlb flushes. - * - * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U - * and SUN4V pte layout, so this inline test is fine. - */ - if (likely(mm != &init_mm) && pte_accessible(mm, orig)) - tlb_batch_add(mm, addr, ptep, orig, fullmm); + maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm); } #define set_pte_at(mm,addr,ptep,pte) \ diff --git a/kernel/arch/sparc/include/asm/tlbflush_64.h b/kernel/arch/sparc/include/asm/tlbflush_64.h index dea1cfa21..a8e192e90 100644 --- a/kernel/arch/sparc/include/asm/tlbflush_64.h +++ b/kernel/arch/sparc/include/asm/tlbflush_64.h @@ -8,6 +8,7 @@ #define TLB_BATCH_NR 192 struct tlb_batch { + bool huge; struct mm_struct *mm; unsigned long tlb_nr; unsigned long active; @@ -16,7 +17,7 @@ struct tlb_batch { void flush_tsb_kernel_range(unsigned long start, unsigned long end); void flush_tsb_user(struct tlb_batch *tb); -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge); /* TLB flush operations. */ diff --git a/kernel/arch/sparc/include/asm/ttable.h b/kernel/arch/sparc/include/asm/ttable.h index 71b5a6752..781b9f1db 100644 --- a/kernel/arch/sparc/include/asm/ttable.h +++ b/kernel/arch/sparc/include/asm/ttable.h @@ -589,8 +589,8 @@ user_rtt_fill_64bit: \ restored; \ nop; nop; nop; nop; nop; nop; \ nop; nop; nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; @@ -652,8 +652,8 @@ user_rtt_fill_32bit: \ restored; \ nop; nop; nop; nop; nop; \ nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; diff --git a/kernel/arch/sparc/include/asm/uaccess_32.h b/kernel/arch/sparc/include/asm/uaccess_32.h index 64ee103dc..dfb542c7c 100644 --- a/kernel/arch/sparc/include/asm/uaccess_32.h +++ b/kernel/arch/sparc/include/asm/uaccess_32.h @@ -328,8 +328,10 @@ static inline unsigned long copy_from_user(void *to, const void __user *from, un { if (n && __access_ok((unsigned long) from, n)) return __copy_user((__force void __user *) to, from, n); - else + else { + memset(to, 0, n); return n; + } } static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) diff --git a/kernel/arch/sparc/include/asm/uaccess_64.h b/kernel/arch/sparc/include/asm/uaccess_64.h index ea6e9a20f..f42851248 100644 --- a/kernel/arch/sparc/include/asm/uaccess_64.h +++ b/kernel/arch/sparc/include/asm/uaccess_64.h @@ -98,7 +98,6 @@ struct exception_table_entry { unsigned int insn, fixup; }; -void __ret_efault(void); void __retl_efault(void); /* Uh, these should become the main single-value transfer routines.. @@ -179,20 +178,6 @@ int __put_user_bad(void); __gu_ret; \ }) -#define __get_user_nocheck_ret(data, addr, size, type, retval) ({ \ - register unsigned long __gu_val __asm__ ("l1"); \ - switch (size) { \ - case 1: __get_user_asm_ret(__gu_val, ub, addr, retval); break; \ - case 2: __get_user_asm_ret(__gu_val, uh, addr, retval); break; \ - case 4: __get_user_asm_ret(__gu_val, uw, addr, retval); break; \ - case 8: __get_user_asm_ret(__gu_val, x, addr, retval); break; \ - default: \ - if (__get_user_bad()) \ - return retval; \ - } \ - data = (__force type) __gu_val; \ -}) - #define __get_user_asm(x, size, addr, ret) \ __asm__ __volatile__( \ "/* Get user asm, inline. */\n" \ @@ -214,80 +199,35 @@ __asm__ __volatile__( \ : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ "i" (-EFAULT)) -#define __get_user_asm_ret(x, size, addr, retval) \ -if (__builtin_constant_p(retval) && retval == -EFAULT) \ - __asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b,__ret_efault\n\n\t" \ - ".previous\n\t" \ - : "=r" (x) : "r" (__m(addr))); \ -else \ - __asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ - "1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ - "3:\n\t" \ - "ret\n\t" \ - " restore %%g0, %2, %%o0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\n\t" \ - ".previous\n\t" \ - : "=r" (x) : "r" (__m(addr)), "i" (retval)) - int __get_user_bad(void); unsigned long __must_check ___copy_from_user(void *to, const void __user *from, unsigned long size); -unsigned long copy_from_user_fixup(void *to, const void __user *from, - unsigned long size); static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long size) { - unsigned long ret = ___copy_from_user(to, from, size); - - if (unlikely(ret)) - ret = copy_from_user_fixup(to, from, size); - - return ret; + return ___copy_from_user(to, from, size); } #define __copy_from_user copy_from_user unsigned long __must_check ___copy_to_user(void __user *to, const void *from, unsigned long size); -unsigned long copy_to_user_fixup(void __user *to, const void *from, - unsigned long size); static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long size) { - unsigned long ret = ___copy_to_user(to, from, size); - - if (unlikely(ret)) - ret = copy_to_user_fixup(to, from, size); - return ret; + return ___copy_to_user(to, from, size); } #define __copy_to_user copy_to_user unsigned long __must_check ___copy_in_user(void __user *to, const void __user *from, unsigned long size); -unsigned long copy_in_user_fixup(void __user *to, void __user *from, - unsigned long size); static inline unsigned long __must_check copy_in_user(void __user *to, void __user *from, unsigned long size) { - unsigned long ret = ___copy_in_user(to, from, size); - - if (unlikely(ret)) - ret = copy_in_user_fixup(to, from, size); - return ret; + return ___copy_in_user(to, from, size); } #define __copy_in_user copy_in_user diff --git a/kernel/arch/sparc/kernel/Makefile b/kernel/arch/sparc/kernel/Makefile index 7cf9c6ea3..fdb13327f 100644 --- a/kernel/arch/sparc/kernel/Makefile +++ b/kernel/arch/sparc/kernel/Makefile @@ -21,6 +21,7 @@ CFLAGS_REMOVE_perf_event.o := -pg CFLAGS_REMOVE_pcr.o := -pg endif +obj-$(CONFIG_SPARC64) += urtt_fill.o obj-$(CONFIG_SPARC32) += entry.o wof.o wuf.o obj-$(CONFIG_SPARC32) += etrap_32.o obj-$(CONFIG_SPARC32) += rtrap_32.o diff --git a/kernel/arch/sparc/kernel/cherrs.S b/kernel/arch/sparc/kernel/cherrs.S index 4ee1ad420..655628def 100644 --- a/kernel/arch/sparc/kernel/cherrs.S +++ b/kernel/arch/sparc/kernel/cherrs.S @@ -214,8 +214,7 @@ do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ subcc %g1, %g2, %g1 ! Next cacheline bge,pt %icc, 1b nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop + ba,a,pt %xcc, dcpe_icpe_tl1_common do_dcpe_tl1_fatal: sethi %hi(1f), %g7 @@ -224,8 +223,7 @@ do_dcpe_tl1_fatal: mov 0x2, %o0 call cheetah_plus_parity_error add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_dcpe_tl1,.-do_dcpe_tl1 .globl do_icpe_tl1 @@ -259,8 +257,7 @@ do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ subcc %g1, %g2, %g1 bge,pt %icc, 1b nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop + ba,a,pt %xcc, dcpe_icpe_tl1_common do_icpe_tl1_fatal: sethi %hi(1f), %g7 @@ -269,8 +266,7 @@ do_icpe_tl1_fatal: mov 0x3, %o0 call cheetah_plus_parity_error add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_icpe_tl1,.-do_icpe_tl1 .type dcpe_icpe_tl1_common,#function @@ -456,7 +452,7 @@ __cheetah_log_error: cmp %g2, 0x63 be c_cee nop - ba,pt %xcc, c_deferred + ba,a,pt %xcc, c_deferred .size __cheetah_log_error,.-__cheetah_log_error /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc diff --git a/kernel/arch/sparc/kernel/dtlb_prot.S b/kernel/arch/sparc/kernel/dtlb_prot.S index d668ca149..4087a62f9 100644 --- a/kernel/arch/sparc/kernel/dtlb_prot.S +++ b/kernel/arch/sparc/kernel/dtlb_prot.S @@ -25,13 +25,13 @@ /* PROT ** ICACHE line 2: More real fault processing */ ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5 + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault nop nop - nop - nop /* PROT ** ICACHE line 3: Unused... */ nop diff --git a/kernel/arch/sparc/kernel/entry.S b/kernel/arch/sparc/kernel/entry.S index 33c02b15f..a83707c83 100644 --- a/kernel/arch/sparc/kernel/entry.S +++ b/kernel/arch/sparc/kernel/entry.S @@ -948,7 +948,24 @@ linux_syscall_trace: cmp %o0, 0 bne 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ld [%sp + STACKFRAME_SZ + PT_G1], %g1 + sethi %hi(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I0], %i0 + or %l7, %lo(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I1], %i1 + ld [%sp + STACKFRAME_SZ + PT_I2], %i2 + ld [%sp + STACKFRAME_SZ + PT_I3], %i3 + ld [%sp + STACKFRAME_SZ + PT_I4], %i4 + ld [%sp + STACKFRAME_SZ + PT_I5], %i5 + cmp %g1, NR_syscalls + bgeu 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 mov %i0, %o0 + ld [%l7 + %l4], %l7 mov %i1, %o1 mov %i2, %o2 mov %i3, %o3 diff --git a/kernel/arch/sparc/kernel/fpu_traps.S b/kernel/arch/sparc/kernel/fpu_traps.S index a6864826a..336d2750f 100644 --- a/kernel/arch/sparc/kernel/fpu_traps.S +++ b/kernel/arch/sparc/kernel/fpu_traps.S @@ -100,8 +100,8 @@ do_fpdis: fmuld %f0, %f2, %f26 faddd %f0, %f2, %f28 fmuld %f0, %f2, %f30 - b,pt %xcc, fpdis_exit - nop + ba,a,pt %xcc, fpdis_exit + 2: andcc %g5, FPRS_DU, %g0 bne,pt %icc, 3f fzero %f32 @@ -144,8 +144,8 @@ do_fpdis: fmuld %f32, %f34, %f58 faddd %f32, %f34, %f60 fmuld %f32, %f34, %f62 - ba,pt %xcc, fpdis_exit - nop + ba,a,pt %xcc, fpdis_exit + 3: mov SECONDARY_CONTEXT, %g3 add %g6, TI_FPREGS, %g1 @@ -197,8 +197,7 @@ fpdis_exit2: fp_other_bounce: call do_fpother add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size fp_other_bounce,.-fp_other_bounce .align 32 diff --git a/kernel/arch/sparc/kernel/head_64.S b/kernel/arch/sparc/kernel/head_64.S index f2d30cab5..7eeeb1d5a 100644 --- a/kernel/arch/sparc/kernel/head_64.S +++ b/kernel/arch/sparc/kernel/head_64.S @@ -461,9 +461,8 @@ sun4v_chip_type: subcc %g3, 1, %g3 bne,pt %xcc, 41b add %g1, 1, %g1 - mov SUN4V_CHIP_SPARC64X, %g4 ba,pt %xcc, 5f - nop + mov SUN4V_CHIP_SPARC64X, %g4 49: mov SUN4V_CHIP_UNKNOWN, %g4 @@ -548,8 +547,7 @@ sun4u_init: stxa %g0, [%g7] ASI_DMMU membar #Sync - ba,pt %xcc, sun4u_continue - nop + ba,a,pt %xcc, sun4u_continue sun4v_init: /* Set ctx 0 */ @@ -560,14 +558,12 @@ sun4v_init: mov SECONDARY_CONTEXT, %g7 stxa %g0, [%g7] ASI_MMU membar #Sync - ba,pt %xcc, niagara_tlb_fixup - nop + ba,a,pt %xcc, niagara_tlb_fixup sun4u_continue: BRANCH_IF_ANY_CHEETAH(g1, g7, cheetah_tlb_fixup) - ba,pt %xcc, spitfire_tlb_fixup - nop + ba,a,pt %xcc, spitfire_tlb_fixup niagara_tlb_fixup: mov 3, %g2 /* Set TLB type to hypervisor. */ @@ -639,8 +635,7 @@ niagara_patch: call hypervisor_patch_cachetlbops nop - ba,pt %xcc, tlb_fixup_done - nop + ba,a,pt %xcc, tlb_fixup_done cheetah_tlb_fixup: mov 2, %g2 /* Set TLB type to cheetah+. */ @@ -659,8 +654,7 @@ cheetah_tlb_fixup: call cheetah_patch_cachetlbops nop - ba,pt %xcc, tlb_fixup_done - nop + ba,a,pt %xcc, tlb_fixup_done spitfire_tlb_fixup: /* Set TLB type to spitfire. */ @@ -782,8 +776,7 @@ setup_trap_table: call %o1 add %sp, (2047 + 128), %o0 - ba,pt %xcc, 2f - nop + ba,a,pt %xcc, 2f 1: sethi %hi(sparc64_ttable_tl0), %o0 set prom_set_trap_table_name, %g2 @@ -822,8 +815,7 @@ setup_trap_table: BRANCH_IF_ANY_CHEETAH(o2, o3, 1f) - ba,pt %xcc, 2f - nop + ba,a,pt %xcc, 2f /* Disable STICK_INT interrupts. */ 1: @@ -930,47 +922,11 @@ prom_tba: .xword 0 tlb_type: .word 0 /* Must NOT end up in BSS */ .section ".fixup",#alloc,#execinstr - .globl __ret_efault, __retl_efault, __ret_one, __retl_one -ENTRY(__ret_efault) - ret - restore %g0, -EFAULT, %o0 -ENDPROC(__ret_efault) - ENTRY(__retl_efault) retl mov -EFAULT, %o0 ENDPROC(__retl_efault) -ENTRY(__retl_one) - retl - mov 1, %o0 -ENDPROC(__retl_one) - -ENTRY(__retl_one_fp) - VISExitHalf - retl - mov 1, %o0 -ENDPROC(__retl_one_fp) - -ENTRY(__ret_one_asi) - wr %g0, ASI_AIUS, %asi - ret - restore %g0, 1, %o0 -ENDPROC(__ret_one_asi) - -ENTRY(__retl_one_asi) - wr %g0, ASI_AIUS, %asi - retl - mov 1, %o0 -ENDPROC(__retl_one_asi) - -ENTRY(__retl_one_asi_fp) - wr %g0, ASI_AIUS, %asi - VISExitHalf - retl - mov 1, %o0 -ENDPROC(__retl_one_asi_fp) - ENTRY(__retl_o1) retl mov %o1, %o0 diff --git a/kernel/arch/sparc/kernel/jump_label.c b/kernel/arch/sparc/kernel/jump_label.c index 59bbeff55..07933b9e9 100644 --- a/kernel/arch/sparc/kernel/jump_label.c +++ b/kernel/arch/sparc/kernel/jump_label.c @@ -13,19 +13,30 @@ void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { - u32 val; u32 *insn = (u32 *) (unsigned long) entry->code; + u32 val; if (type == JUMP_LABEL_JMP) { s32 off = (s32)entry->target - (s32)entry->code; + bool use_v9_branch = false; + + BUG_ON(off & 3); #ifdef CONFIG_SPARC64 - /* ba,pt %xcc, . + (off << 2) */ - val = 0x10680000 | ((u32) off >> 2); -#else - /* ba . + (off << 2) */ - val = 0x10800000 | ((u32) off >> 2); + if (off <= 0xfffff && off >= -0x100000) + use_v9_branch = true; #endif + if (use_v9_branch) { + /* WDISP19 - target is . + immed << 2 */ + /* ba,pt %xcc, . + off */ + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); + } else { + /* WDISP22 - target is . + immed << 2 */ + BUG_ON(off > 0x7fffff); + BUG_ON(off < -0x800000); + /* ba . + off */ + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); + } } else { val = 0x01000000; } diff --git a/kernel/arch/sparc/kernel/ktlb.S b/kernel/arch/sparc/kernel/ktlb.S index ef0d8e9e1..f22bec0db 100644 --- a/kernel/arch/sparc/kernel/ktlb.S +++ b/kernel/arch/sparc/kernel/ktlb.S @@ -20,6 +20,10 @@ kvmap_itlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_IMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_itlb_miss branches here with the missing virtual * address already loaded into %g4 */ @@ -128,6 +132,10 @@ kvmap_dtlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_DMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_dtlb_miss branches here with the missing virtual * address already loaded into %g4 */ @@ -251,6 +259,10 @@ kvmap_dtlb_longpath: nop .previous + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g5 here. + */ + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline diff --git a/kernel/arch/sparc/kernel/misctrap.S b/kernel/arch/sparc/kernel/misctrap.S index 753b4f031..34b493390 100644 --- a/kernel/arch/sparc/kernel/misctrap.S +++ b/kernel/arch/sparc/kernel/misctrap.S @@ -18,8 +18,7 @@ __do_privact: 109: or %g7, %lo(109b), %g7 call do_privact add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __do_privact,.-__do_privact .type do_mna,#function @@ -46,8 +45,7 @@ do_mna: mov %l5, %o2 call mem_address_unaligned add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_mna,.-do_mna .type do_lddfmna,#function @@ -65,8 +63,7 @@ do_lddfmna: mov %l5, %o2 call handle_lddfmna add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_lddfmna,.-do_lddfmna .type do_stdfmna,#function @@ -84,8 +81,7 @@ do_stdfmna: mov %l5, %o2 call handle_stdfmna add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_stdfmna,.-do_stdfmna .type breakpoint_trap,#function diff --git a/kernel/arch/sparc/kernel/pci.c b/kernel/arch/sparc/kernel/pci.c index badf0951d..9f9614df9 100644 --- a/kernel/arch/sparc/kernel/pci.c +++ b/kernel/arch/sparc/kernel/pci.c @@ -994,6 +994,23 @@ void pcibios_set_master(struct pci_dev *dev) /* No special bus mastering setup handling */ } +#ifdef CONFIG_PCI_IOV +int pcibios_add_device(struct pci_dev *dev) +{ + struct pci_dev *pdev; + + /* Add sriov arch specific initialization here. + * Copy dev_archdata from PF to VF + */ + if (dev->is_virtfn) { + pdev = dev->physfn; + memcpy(&dev->dev.archdata, &pdev->dev.archdata, + sizeof(struct dev_archdata)); + } + return 0; +} +#endif /* CONFIG_PCI_IOV */ + static int __init pcibios_init(void) { pci_dfl_cache_line_size = 64 >> 2; diff --git a/kernel/arch/sparc/kernel/rtrap_64.S b/kernel/arch/sparc/kernel/rtrap_64.S index d08bdaffd..216948ca4 100644 --- a/kernel/arch/sparc/kernel/rtrap_64.S +++ b/kernel/arch/sparc/kernel/rtrap_64.S @@ -14,10 +14,6 @@ #include #include -#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) -#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) -#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) - #ifdef CONFIG_CONTEXT_TRACKING # define SCHEDULE_USER schedule_user #else @@ -242,52 +238,17 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 wrpr %g1, %cwp ba,a,pt %xcc, user_rtt_fill_64bit -user_rtt_fill_fixup: - rdpr %cwp, %g1 - add %g1, 1, %g1 - wrpr %g1, 0x0, %cwp - - rdpr %wstate, %g2 - sll %g2, 3, %g2 - wrpr %g2, 0x0, %wstate - - /* We know %canrestore and %otherwin are both zero. */ - - sethi %hi(sparc64_kern_pri_context), %g2 - ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 - mov PRIMARY_CONTEXT, %g1 - -661: stxa %g2, [%g1] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g1] ASI_MMU - .previous - - sethi %hi(KERNBASE), %g1 - flush %g1 +user_rtt_fill_fixup_dax: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 1, %g3 - or %g4, FAULT_CODE_WINFIXUP, %g4 - stb %g4, [%g6 + TI_FAULT_CODE] - stx %g5, [%g6 + TI_FAULT_ADDR] +user_rtt_fill_fixup_mna: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 2, %g3 - mov %g6, %l1 - wrpr %g0, 0x0, %tl - -661: nop - .section .sun4v_1insn_patch, "ax" - .word 661b - SET_GL(0) - .previous - - wrpr %g0, RTRAP_PSTATE, %pstate - - mov %l1, %g6 - ldx [%g6 + TI_TASK], %g4 - LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) - call do_sparc64_fault - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop +user_rtt_fill_fixup: + ba,pt %xcc, user_rtt_fill_fixup_common + clr %g3 user_rtt_pre_restore: add %g1, 1, %g1 diff --git a/kernel/arch/sparc/kernel/signal32.c b/kernel/arch/sparc/kernel/signal32.c index 4eed773a7..77655f0f0 100644 --- a/kernel/arch/sparc/kernel/signal32.c +++ b/kernel/arch/sparc/kernel/signal32.c @@ -138,12 +138,24 @@ int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from) return 0; } +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || + ((unsigned long)fp) > 0x100000000ULL - fplen) + return true; + return false; +} + void do_sigreturn32(struct pt_regs *regs) { struct signal_frame32 __user *sf; compat_uptr_t fpu_save; compat_uptr_t rwin_save; - unsigned int psr; + unsigned int psr, ufp; unsigned pc, npc; sigset_t set; compat_sigset_t seta; @@ -158,11 +170,16 @@ void do_sigreturn32(struct pt_regs *regs) sf = (struct signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) goto segv; - if (get_user(pc, &sf->info.si_regs.pc) || + if (__get_user(pc, &sf->info.si_regs.pc) || __get_user(npc, &sf->info.si_regs.npc)) goto segv; @@ -227,7 +244,7 @@ segv: asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) { struct rt_signal_frame32 __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; compat_uptr_t fpu_save; compat_uptr_t rwin_save; sigset_t set; @@ -242,11 +259,16 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) sf = (struct rt_signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) goto segv; - if (get_user(pc, &sf->regs.pc) || + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->regs.pc) || __get_user(npc, &sf->regs.npc)) goto segv; @@ -307,14 +329,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || ((unsigned long)fp) > 0x100000000ULL - fplen) - return 1; - return 0; -} - static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp; diff --git a/kernel/arch/sparc/kernel/signal_32.c b/kernel/arch/sparc/kernel/signal_32.c index 52aa5e4ce..9c0c8fd0b 100644 --- a/kernel/arch/sparc/kernel/signal_32.c +++ b/kernel/arch/sparc/kernel/signal_32.c @@ -60,10 +60,22 @@ struct rt_signal_frame { #define SF_ALIGNEDSZ (((sizeof(struct signal_frame) + 7) & (~7))) #define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static inline bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || !__access_ok((unsigned long)fp, fplen)) + return true; + + return false; +} + asmlinkage void do_sigreturn(struct pt_regs *regs) { + unsigned long up_psr, pc, npc, ufp; struct signal_frame __user *sf; - unsigned long up_psr, pc, npc; sigset_t set; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; @@ -77,10 +89,13 @@ asmlinkage void do_sigreturn(struct pt_regs *regs) sf = (struct signal_frame __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf))) + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv_and_exit; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) goto segv_and_exit; - if (((unsigned long) sf) & 3) + if (ufp & 0x7) goto segv_and_exit; err = __get_user(pc, &sf->info.si_regs.pc); @@ -127,7 +142,7 @@ segv_and_exit: asmlinkage void do_rt_sigreturn(struct pt_regs *regs) { struct rt_signal_frame __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; @@ -135,8 +150,13 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) synchronize_user_stack(); sf = (struct rt_signal_frame __user *) regs->u_regs[UREG_FP]; - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 0x03)) + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) goto segv; err = __get_user(pc, &sf->regs.pc); @@ -178,15 +198,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static inline int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || !__access_ok((unsigned long)fp, fplen)) - return 1; - - return 0; -} - static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP]; diff --git a/kernel/arch/sparc/kernel/signal_64.c b/kernel/arch/sparc/kernel/signal_64.c index d88beff47..5ee930c48 100644 --- a/kernel/arch/sparc/kernel/signal_64.c +++ b/kernel/arch/sparc/kernel/signal_64.c @@ -52,7 +52,7 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) unsigned char fenab; int err; - flush_user_windows(); + synchronize_user_stack(); if (get_thread_wsaved() || (((unsigned long)ucp) & (sizeof(unsigned long)-1)) || (!__access_ok(ucp, sizeof(*ucp)))) @@ -234,6 +234,17 @@ do_sigsegv: goto out; } +/* Checks if the fp is valid. We always build rt signal frames which + * are 16-byte aligned, therefore we can always enforce that the + * restore frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp) +{ + if (((unsigned long) fp) & 15) + return true; + return false; +} + struct rt_signal_frame { struct sparc_stackf ss; siginfo_t info; @@ -246,8 +257,8 @@ struct rt_signal_frame { void do_rt_sigreturn(struct pt_regs *regs) { + unsigned long tpc, tnpc, tstate, ufp; struct rt_signal_frame __user *sf; - unsigned long tpc, tnpc, tstate; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; @@ -261,10 +272,16 @@ void do_rt_sigreturn(struct pt_regs *regs) (regs->u_regs [UREG_FP] + STACK_BIAS); /* 1. Make sure we are not getting garbage from the user */ - if (((unsigned long) sf) & 3) + if (invalid_frame_pointer(sf)) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) goto segv; - err = get_user(tpc, &sf->regs.tpc); + if ((ufp + STACK_BIAS) & 0x7) + goto segv; + + err = __get_user(tpc, &sf->regs.tpc); err |= __get_user(tnpc, &sf->regs.tnpc); if (test_thread_flag(TIF_32BIT)) { tpc &= 0xffffffff; @@ -308,14 +325,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp) -{ - if (((unsigned long) fp) & 15) - return 1; - return 0; -} - static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP] + STACK_BIAS; diff --git a/kernel/arch/sparc/kernel/sigutil_32.c b/kernel/arch/sparc/kernel/sigutil_32.c index 0f6eebe71..e5fe8cef9 100644 --- a/kernel/arch/sparc/kernel/sigutil_32.c +++ b/kernel/arch/sparc/kernel/sigutil_32.c @@ -48,6 +48,10 @@ int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) { int err; + + if (((unsigned long) fpu) & 3) + return -EFAULT; + #ifdef CONFIG_SMP if (test_tsk_thread_flag(current, TIF_USEDFPU)) regs->psr &= ~PSR_EF; @@ -97,7 +101,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 3) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/kernel/arch/sparc/kernel/sigutil_64.c b/kernel/arch/sparc/kernel/sigutil_64.c index 387834a9c..36aadcbea 100644 --- a/kernel/arch/sparc/kernel/sigutil_64.c +++ b/kernel/arch/sparc/kernel/sigutil_64.c @@ -37,7 +37,10 @@ int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) unsigned long fprs; int err; - err = __get_user(fprs, &fpu->si_fprs); + if (((unsigned long) fpu) & 7) + return -EFAULT; + + err = get_user(fprs, &fpu->si_fprs); fprs_write(0); regs->tstate &= ~TSTATE_PEF; if (fprs & FPRS_DL) @@ -72,7 +75,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 7) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/kernel/arch/sparc/kernel/sparc_ksyms_64.c b/kernel/arch/sparc/kernel/sparc_ksyms_64.c index a92d5d2c4..51b25325a 100644 --- a/kernel/arch/sparc/kernel/sparc_ksyms_64.c +++ b/kernel/arch/sparc/kernel/sparc_ksyms_64.c @@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user); EXPORT_SYMBOL_GPL(real_hard_smp_processor_id); /* from head_64.S */ -EXPORT_SYMBOL(__ret_efault); EXPORT_SYMBOL(tlb_type); EXPORT_SYMBOL(sun4v_chip_type); EXPORT_SYMBOL(prom_root_node); diff --git a/kernel/arch/sparc/kernel/spiterrs.S b/kernel/arch/sparc/kernel/spiterrs.S index c357e40ff..4a73009f6 100644 --- a/kernel/arch/sparc/kernel/spiterrs.S +++ b/kernel/arch/sparc/kernel/spiterrs.S @@ -85,8 +85,7 @@ __spitfire_cee_trap_continue: ba,pt %xcc, etraptl1 rd %pc, %g7 - ba,pt %xcc, 2f - nop + ba,a,pt %xcc, 2f 1: ba,pt %xcc, etrap_irq rd %pc, %g7 @@ -100,8 +99,7 @@ __spitfire_cee_trap_continue: mov %l5, %o2 call spitfire_access_error add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_access_error,.-__spitfire_access_error /* This is the trap handler entry point for ECC correctable @@ -179,8 +177,7 @@ __spitfire_data_access_exception_tl1: mov %l5, %o2 call spitfire_data_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_data_access_exception_tl1,.-__spitfire_data_access_exception_tl1 .type __spitfire_data_access_exception,#function @@ -200,8 +197,7 @@ __spitfire_data_access_exception: mov %l5, %o2 call spitfire_data_access_exception add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_data_access_exception,.-__spitfire_data_access_exception .type __spitfire_insn_access_exception_tl1,#function @@ -220,8 +216,7 @@ __spitfire_insn_access_exception_tl1: mov %l5, %o2 call spitfire_insn_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_insn_access_exception_tl1,.-__spitfire_insn_access_exception_tl1 .type __spitfire_insn_access_exception,#function @@ -240,6 +235,5 @@ __spitfire_insn_access_exception: mov %l5, %o2 call spitfire_insn_access_exception add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_insn_access_exception,.-__spitfire_insn_access_exception diff --git a/kernel/arch/sparc/kernel/syscalls.S b/kernel/arch/sparc/kernel/syscalls.S index bb0008927..c4a1b5c40 100644 --- a/kernel/arch/sparc/kernel/syscalls.S +++ b/kernel/arch/sparc/kernel/syscalls.S @@ -158,7 +158,25 @@ linux_syscall_trace32: add %sp, PTREGS_OFF, %o0 brnz,pn %o0, 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 srl %i0, 0, %o0 + lduw [%l7 + %l4], %l7 srl %i4, 0, %o4 srl %i1, 0, %o1 srl %i2, 0, %o2 @@ -170,7 +188,25 @@ linux_syscall_trace: add %sp, PTREGS_OFF, %o0 brnz,pn %o0, 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 mov %i0, %o0 + lduw [%l7 + %l4], %l7 mov %i1, %o1 mov %i2, %o2 mov %i3, %o3 diff --git a/kernel/arch/sparc/kernel/tsb.S b/kernel/arch/sparc/kernel/tsb.S index be98685c1..d568c8207 100644 --- a/kernel/arch/sparc/kernel/tsb.S +++ b/kernel/arch/sparc/kernel/tsb.S @@ -29,13 +29,17 @@ */ tsb_miss_dtlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_DMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 tsb_miss_itlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_IMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 /* At this point we have: * %g1 -- PAGE_SIZE TSB entry address @@ -284,6 +288,10 @@ tsb_do_dtlb_fault: nop .previous + /* Clear context ID bits. */ + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline diff --git a/kernel/arch/sparc/kernel/urtt_fill.S b/kernel/arch/sparc/kernel/urtt_fill.S new file mode 100644 index 000000000..5604a2b05 --- /dev/null +++ b/kernel/arch/sparc/kernel/urtt_fill.S @@ -0,0 +1,98 @@ +#include +#include +#include +#include +#include + + .text + .align 8 + .globl user_rtt_fill_fixup_common +user_rtt_fill_fixup_common: + rdpr %cwp, %g1 + add %g1, 1, %g1 + wrpr %g1, 0x0, %cwp + + rdpr %wstate, %g2 + sll %g2, 3, %g2 + wrpr %g2, 0x0, %wstate + + /* We know %canrestore and %otherwin are both zero. */ + + sethi %hi(sparc64_kern_pri_context), %g2 + ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + sethi %hi(KERNBASE), %g1 + flush %g1 + + mov %g4, %l4 + mov %g5, %l5 + brnz,pn %g3, 1f + mov %g3, %l3 + + or %g4, FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] +1: + mov %g6, %l1 + wrpr %g0, 0x0, %tl + +661: nop + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(0) + .previous + + wrpr %g0, RTRAP_PSTATE, %pstate + + mov %l1, %g6 + ldx [%g6 + TI_TASK], %g4 + LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) + + brnz,pn %l3, 1f + nop + + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +1: cmp %g3, 2 + bne,pn %xcc, 2f + nop + + sethi %hi(tlb_type), %g1 + lduw [%g1 + %lo(tlb_type)], %g1 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + mov %l4, %o2 + call sun4v_do_mna + mov %l5, %o1 + ba,a,pt %xcc, rtrap +1: mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + nop + ba,a,pt %xcc, rtrap + +2: sethi %hi(tlb_type), %g1 + mov %l4, %o1 + lduw [%g1 + %lo(tlb_type)], %g1 + mov %l5, %o2 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + call sun4v_data_access_exception + nop + ba,a,pt %xcc, rtrap + +1: call spitfire_data_access_exception + nop + ba,a,pt %xcc, rtrap diff --git a/kernel/arch/sparc/kernel/utrap.S b/kernel/arch/sparc/kernel/utrap.S index b7f0f3f3a..c731e8023 100644 --- a/kernel/arch/sparc/kernel/utrap.S +++ b/kernel/arch/sparc/kernel/utrap.S @@ -11,8 +11,7 @@ utrap_trap: /* %g3=handler,%g4=level */ mov %l4, %o1 call bad_trap add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap invoke_utrap: sllx %g3, 3, %g3 diff --git a/kernel/arch/sparc/kernel/vmlinux.lds.S b/kernel/arch/sparc/kernel/vmlinux.lds.S index f1a2f688b..4a41d412d 100644 --- a/kernel/arch/sparc/kernel/vmlinux.lds.S +++ b/kernel/arch/sparc/kernel/vmlinux.lds.S @@ -33,6 +33,10 @@ ENTRY(_start) jiffies = jiffies_64; #endif +#ifdef CONFIG_SPARC64 +ASSERT((swapper_tsb == 0x0000000000408000), "Error: sparc64 early assembler too large") +#endif + SECTIONS { #ifdef CONFIG_SPARC64 diff --git a/kernel/arch/sparc/kernel/winfixup.S b/kernel/arch/sparc/kernel/winfixup.S index 1e67ce958..855019a85 100644 --- a/kernel/arch/sparc/kernel/winfixup.S +++ b/kernel/arch/sparc/kernel/winfixup.S @@ -32,8 +32,7 @@ fill_fixup: rd %pc, %g7 call do_sparc64_fault add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap /* Be very careful about usage of the trap globals here. * You cannot touch %g5 as that has the fault information. diff --git a/kernel/arch/sparc/lib/GENcopy_from_user.S b/kernel/arch/sparc/lib/GENcopy_from_user.S index b7d0bd6b1..69a439fa2 100644 --- a/kernel/arch/sparc/lib/GENcopy_from_user.S +++ b/kernel/arch/sparc/lib/GENcopy_from_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/GENcopy_to_user.S b/kernel/arch/sparc/lib/GENcopy_to_user.S index 780550e1a..9947427ce 100644 --- a/kernel/arch/sparc/lib/GENcopy_to_user.S +++ b/kernel/arch/sparc/lib/GENcopy_to_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/GENmemcpy.S b/kernel/arch/sparc/lib/GENmemcpy.S index 89358ee94..059ea24ad 100644 --- a/kernel/arch/sparc/lib/GENmemcpy.S +++ b/kernel/arch/sparc/lib/GENmemcpy.S @@ -4,21 +4,18 @@ */ #ifdef __KERNEL__ +#include #define GLOBAL_SPARE %g7 #else #define GLOBAL_SPARE %g5 #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -45,6 +42,29 @@ .register %g3,#scratch .text + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(GEN_retl_o4_1) + add %o4, %o2, %o4 + retl + add %o4, 1, %o0 +ENDPROC(GEN_retl_o4_1) +ENTRY(GEN_retl_g1_8) + add %g1, %o2, %g1 + retl + add %g1, 8, %o0 +ENDPROC(GEN_retl_g1_8) +ENTRY(GEN_retl_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(GEN_retl_o2_4) +ENTRY(GEN_retl_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(GEN_retl_o2_1) +#endif + .align 64 .globl FUNC_NAME @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %g1 sub %o2, %g1, %o2 1: subcc %g1, 0x8, %g1 - EX_LD(LOAD(ldx, %o1, %g2)) - EX_ST(STORE(stx, %g2, %o0)) + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) add %o1, 0x8, %o1 bne,pt %XCC, 1b add %o0, 0x8, %o0 @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/kernel/arch/sparc/lib/Makefile b/kernel/arch/sparc/lib/Makefile index 3269b0234..4f2384a42 100644 --- a/kernel/arch/sparc/lib/Makefile +++ b/kernel/arch/sparc/lib/Makefile @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o obj-$(CONFIG_SPARC64) += iomap.o diff --git a/kernel/arch/sparc/lib/NG2copy_from_user.S b/kernel/arch/sparc/lib/NG2copy_from_user.S index d5242b8c4..b79a6998d 100644 --- a/kernel/arch/sparc/lib/NG2copy_from_user.S +++ b/kernel/arch/sparc/lib/NG2copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NG2copy_to_user.S b/kernel/arch/sparc/lib/NG2copy_to_user.S index 4e962d993..dcec55f25 100644 --- a/kernel/arch/sparc/lib/NG2copy_to_user.S +++ b/kernel/arch/sparc/lib/NG2copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NG2memcpy.S b/kernel/arch/sparc/lib/NG2memcpy.S index d5f585df2..c629dbd12 100644 --- a/kernel/arch/sparc/lib/NG2memcpy.S +++ b/kernel/arch/sparc/lib/NG2memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include #include #include #define GLOBAL_SPARE %g7 @@ -32,21 +33,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -140,45 +137,110 @@ fsrc2 %x6, %f12; \ fsrc2 %x7, %f14; #define FREG_LOAD_1(base, x0) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)) + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) #define FREG_LOAD_2(base, x0, x1) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); #define FREG_LOAD_3(base, x0, x1, x2) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); #define FREG_LOAD_4(base, x0, x1, x2, x3) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \ - EX_LD_FP(LOAD(ldd, base + 0x30, %x6)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi +ENTRY(NG2_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG2_retl_o2) +ENTRY(NG2_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG2_retl_o2_plus_1) +ENTRY(NG2_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG2_retl_o2_plus_4) +ENTRY(NG2_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(NG2_retl_o2_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_1) + add %o4, 1, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_1) +ENTRY(NG2_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_16) +ENTRY(NG2_retl_o2_plus_g1_fp) + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) + add %g1, 64, %g1 + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_1) +ENTRY(NG2_retl_o2_and_7_plus_o4) + and %o2, 7, %o2 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4) +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) + and %o2, 7, %o2 + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) +#endif + .align 64 .globl FUNC_NAME @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 ! bytes to align dst sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop /* fall through for 0 < low bits < 8 */ 110: sub %o4, 64, %g2 - EX_LD_FP(LOAD_BLK(%g2, %f0)) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 120: sub %o4, 56, %g2 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 130: sub %o4, 48, %g2 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_6(f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 140: sub %o4, 40, %g2 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_5(f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 150: sub %o4, 32, %g2 FREG_LOAD_4(%g2, f0, f2, f4, f6) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_4(f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 160: sub %o4, 24, %g2 FREG_LOAD_3(%g2, f0, f2, f4) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_3(f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 170: sub %o4, 16, %g2 FREG_LOAD_2(%g2, f0, f2) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_2(f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 180: sub %o4, 8, %g2 FREG_LOAD_1(%g2, f0) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_1(f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop 190: -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) subcc %g1, 64, %g1 - EX_LD_FP(LOAD_BLK(%o4, %f0)) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) add %o4, 64, %o4 bne,pt %xcc, 1b LOAD(prefetch, %o4 + 64, #one_read) @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, %o4 and %o2, 0xf, %o2 1: subcc %o4, 0x10, %o4 - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x08, %o1 - EX_LD(LOAD(ldx, %o1, %g1)) + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) sub %o1, 0x08, %o1 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, GLOBAL_SPARE andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) sub GLOBAL_SPARE, %g1, GLOBAL_SPARE andn %o2, 0x7, %o4 sllx %g2, %g1, %g2 1: add %o1, 0x8, %o1 - EX_LD(LOAD(ldx, %o1, %g3)) + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) subcc %o4, 0x8, %o4 srlx %g3, GLOBAL_SPARE, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/kernel/arch/sparc/lib/NG4copy_from_user.S b/kernel/arch/sparc/lib/NG4copy_from_user.S index 2e8ee7ad0..16a286c1a 100644 --- a/kernel/arch/sparc/lib/NG4copy_from_user.S +++ b/kernel/arch/sparc/lib/NG4copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x, y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NG4copy_to_user.S b/kernel/arch/sparc/lib/NG4copy_to_user.S index be0bf4590..6b0276ffc 100644 --- a/kernel/arch/sparc/lib/NG4copy_to_user.S +++ b/kernel/arch/sparc/lib/NG4copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NG4memcpy.S b/kernel/arch/sparc/lib/NG4memcpy.S index 8e13ee1f4..75bb93b14 100644 --- a/kernel/arch/sparc/lib/NG4memcpy.S +++ b/kernel/arch/sparc/lib/NG4memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include #include #include #define GLOBAL_SPARE %g7 @@ -46,22 +47,19 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x +#define EX_ST_FP(x,y) x #endif -#ifndef EX_RETVAL -#define EX_RETVAL(x) x -#endif #ifndef LOAD #define LOAD(type,addr,dest) type [addr], dest @@ -94,6 +92,158 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi + +ENTRY(NG4_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG4_retl_o2) +ENTRY(NG4_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG4_retl_o2_plus_1) +ENTRY(NG4_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG4_retl_o2_plus_4) +ENTRY(NG4_retl_o2_plus_o5) + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5) +ENTRY(NG4_retl_o2_plus_o5_plus_4) + add %o5, 4, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_4) +ENTRY(NG4_retl_o2_plus_o5_plus_8) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_8) +ENTRY(NG4_retl_o2_plus_o5_plus_16) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_16) +ENTRY(NG4_retl_o2_plus_o5_plus_24) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_24) +ENTRY(NG4_retl_o2_plus_o5_plus_32) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_32) +ENTRY(NG4_retl_o2_plus_g1) + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1) +ENTRY(NG4_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1_plus_1) +ENTRY(NG4_retl_o2_plus_g1_plus_8) + add %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1_plus_8) +ENTRY(NG4_retl_o2_plus_o4) + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4) +ENTRY(NG4_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_8) +ENTRY(NG4_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_16) +ENTRY(NG4_retl_o2_plus_o4_plus_24) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_24) +ENTRY(NG4_retl_o2_plus_o4_plus_32) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_32) +ENTRY(NG4_retl_o2_plus_o4_plus_40) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_40) +ENTRY(NG4_retl_o2_plus_o4_plus_48) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_48) +ENTRY(NG4_retl_o2_plus_o4_plus_56) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_56) +ENTRY(NG4_retl_o2_plus_o4_plus_64) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_64) +ENTRY(NG4_retl_o2_plus_o4_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp) +#endif .align 64 .globl FUNC_NAME @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 51f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) LOAD(prefetch, %o1 + 0x080, #n_reads_strong) @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, .Llarge_aligned sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 8, %o1 subcc %g1, 8, %g1 add %o0, 8, %o0 bne,pt %icc, 1b - EX_ST(STORE(stx, %g2, %o0 - 0x08)) + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8) .Llarge_aligned: /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ andn %o2, 0x3f, %o4 sub %o2, %o4, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4) add %o1, 0x40, %o1 - EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) - EX_ST(STORE_INIT(%g1, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64) + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) - EX_ST(STORE_INIT(%g3, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48) + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) - EX_ST(STORE_INIT(%o5, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32) + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g3, %o0)) + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b LOAD(prefetch, %o1 + 0x200, #n_reads_strong) @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %o4, %o2 alignaddr %o1, %g0, %g1 add %o1, %o4, %o1 - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0)) -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2)) + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4) +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4)) - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6)) - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8)) - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10)) - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12)) - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14)) + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0)) + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64) faligndata %f2, %f4, %f18 add %g1, 0x40, %g1 faligndata %f4, %f6, %f20 @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST_FP(STORE(std, %f16, %o0 + 0x00)) - EX_ST_FP(STORE(std, %f18, %o0 + 0x08)) - EX_ST_FP(STORE(std, %f20, %o0 + 0x10)) - EX_ST_FP(STORE(std, %f22, %o0 + 0x18)) - EX_ST_FP(STORE(std, %f24, %o0 + 0x20)) - EX_ST_FP(STORE(std, %f26, %o0 + 0x28)) - EX_ST_FP(STORE(std, %f28, %o0 + 0x30)) - EX_ST_FP(STORE(std, %f30, %o0 + 0x38)) + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64) + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56) + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48) + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40) + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32) + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24) + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16) + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8) add %o0, 0x40, %o0 bne,pt %icc, 1b LOAD(prefetch, %g1 + 0x200, #n_reads_strong) @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andncc %o2, 0x20 - 1, %o5 be,pn %icc, 2f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5) add %o1, 0x20, %o1 subcc %o5, 0x20, %o5 - EX_ST(STORE(stx, %g1, %o0 + 0x00)) - EX_ST(STORE(stx, %g2, %o0 + 0x08)) - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) - EX_ST(STORE(stx, %o4, %o0 + 0x18)) + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8) bne,pt %icc, 1b add %o0, 0x20, %o0 2: andcc %o2, 0x18, %o5 be,pt %icc, 3f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) add %o1, 0x08, %o1 add %o0, 0x08, %o0 subcc %o5, 0x08, %o5 bne,pt %icc, 1b - EX_ST(STORE(stx, %g1, %o0 - 0x08)) + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8) 3: brz,pt %o2, .Lexit cmp %o2, 0x04 bl,pn %icc, .Ltiny nop - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2) add %o1, 0x04, %o1 add %o0, 0x04, %o0 subcc %o2, 0x04, %o2 bne,pn %icc, .Ltiny - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4) ba,a,pt %icc, .Lexit .Lmedium_unaligned: /* First get dest 8 byte aligned. */ @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 2f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) 2: and %o1, 0x7, %g1 brz,pn %g1, .Lmedium_noprefetch @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov 64, %g2 sub %g2, %g1, %g2 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2) sllx %o4, %g1, %o4 andn %o2, 0x08 - 1, %o5 sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5) add %o1, 0x08, %o1 subcc %o5, 0x08, %o5 srlx %g3, %g2, GLOBAL_SPARE or GLOBAL_SPARE, %o4, GLOBAL_SPARE - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b sllx %g3, %g1, %o4 @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ba,pt %icc, .Lsmall_unaligned .Ltiny: - EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x00)) - EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x01)) - EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2) ba,pt %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x02)) + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2) .Lsmall: andcc %g2, 0x3, %g0 @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x4 - 1, %o5 sub %o2, %o5, %o2 1: - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) add %o1, 0x04, %o1 subcc %o5, 0x04, %o5 add %o0, 0x04, %o0 bne,pt %icc, 1b - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4) brz,pt %o2, .Lexit nop ba,a,pt %icc, .Ltiny .Lsmall_unaligned: -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) add %o1, 1, %o1 add %o0, 1, %o0 subcc %o2, 1, %o2 bne,pt %icc, 1b - EX_ST(STORE(stb, %g1, %o0 - 0x01)) + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) ba,a,pt %icc, .Lexit .size FUNC_NAME, .-FUNC_NAME diff --git a/kernel/arch/sparc/lib/NGcopy_from_user.S b/kernel/arch/sparc/lib/NGcopy_from_user.S index 5d1e4d1ac..9cd42fcbc 100644 --- a/kernel/arch/sparc/lib/NGcopy_from_user.S +++ b/kernel/arch/sparc/lib/NGcopy_from_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NGcopy_to_user.S b/kernel/arch/sparc/lib/NGcopy_to_user.S index ff630dcb2..5c358afd4 100644 --- a/kernel/arch/sparc/lib/NGcopy_to_user.S +++ b/kernel/arch/sparc/lib/NGcopy_to_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/NGmemcpy.S b/kernel/arch/sparc/lib/NGmemcpy.S index 96a14caf6..d88c4ed50 100644 --- a/kernel/arch/sparc/lib/NGmemcpy.S +++ b/kernel/arch/sparc/lib/NGmemcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include #include #include #define GLOBAL_SPARE %g7 @@ -27,15 +28,11 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -79,6 +76,92 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi: + ret + wr %g0, ASI_AIUS, %asi + restore +ENTRY(NG_ret_i2_plus_i4_plus_1) + ba,pt %xcc, __restore_asi + add %i2, %i5, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_1) +ENTRY(NG_ret_i2_plus_g1) + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1) +ENTRY(NG_ret_i2_plus_g1_minus_8) + sub %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_8) +ENTRY(NG_ret_i2_plus_g1_minus_16) + sub %g1, 16, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_16) +ENTRY(NG_ret_i2_plus_g1_minus_24) + sub %g1, 24, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_24) +ENTRY(NG_ret_i2_plus_g1_minus_32) + sub %g1, 32, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_32) +ENTRY(NG_ret_i2_plus_g1_minus_40) + sub %g1, 40, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_40) +ENTRY(NG_ret_i2_plus_g1_minus_48) + sub %g1, 48, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_48) +ENTRY(NG_ret_i2_plus_g1_minus_56) + sub %g1, 56, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_56) +ENTRY(NG_ret_i2_plus_i4) + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4) +ENTRY(NG_ret_i2_plus_i4_minus_8) + sub %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_minus_8) +ENTRY(NG_ret_i2_plus_8) + ba,pt %xcc, __restore_asi + add %i2, 8, %i0 +ENDPROC(NG_ret_i2_plus_8) +ENTRY(NG_ret_i2_plus_4) + ba,pt %xcc, __restore_asi + add %i2, 4, %i0 +ENDPROC(NG_ret_i2_plus_4) +ENTRY(NG_ret_i2_plus_1) + ba,pt %xcc, __restore_asi + add %i2, 1, %i0 +ENDPROC(NG_ret_i2_plus_1) +ENTRY(NG_ret_i2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_plus_1) +ENTRY(NG_ret_i2) + ba,pt %xcc, __restore_asi + mov %i2, %i0 +ENDPROC(NG_ret_i2) +ENTRY(NG_ret_i2_and_7_plus_i4) + and %i2, 7, %i2 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +#endif + .align 64 .globl FUNC_NAME @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %g0, %i4, %i4 ! bytes to align dst sub %i2, %i4, %i2 1: subcc %i4, 1, %i4 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) add %i1, 1, %i1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ and %i4, 0x7, GLOBAL_SPARE sll GLOBAL_SPARE, 3, GLOBAL_SPARE mov 64, %i5 - EX_LD(LOAD_TWIN(%i1, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) sub %i5, GLOBAL_SPARE, %i5 mov 16, %o4 mov 32, %o5 @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ srlx WORD3, PRE_SHIFT, TMP; \ or WORD2, TMP, WORD2; -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g2, %o0 + 0x00)) - EX_ST(STORE_INIT(%g3, %o0 + 0x08)) + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g2, %o0 + 0x20)) - EX_ST(STORE_INIT(%g3, %o0 + 0x28)) + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 8b @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ ba,pt %XCC, 60f add %i1, %i4, %i1 -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g3, %o0 + 0x00)) - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%g2, %o0 + 0x18)) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g3, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%g2, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 9b @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ * one twin load ahead, then add 8 back into source when * we finish the loop. */ - EX_LD(LOAD_TWIN(%i1, %o4, %o5)) + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) mov 16, %o7 mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%o4, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) - EX_ST(STORE_INIT(%o5, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%o4, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o5, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o4, %o0 + 0x20)) - EX_ST(STORE_INIT(%o5, %o0 + 0x28)) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ andn %i2, 0xf, %i4 and %i2, 0xf, %i2 1: subcc %i4, 0x10, %i4 - EX_LD(LOAD(ldx, %i1, %o4)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4) add %i1, 0x08, %i1 - EX_LD(LOAD(ldx, %i1, %g1)) + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4) sub %i1, 0x08, %i1 - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4) add %i1, 0x8, %i1 - EX_ST(STORE(stx, %g1, %i1 + %i3)) + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8) bgu,pt %XCC, 1b add %i1, 0x8, %i1 73: andcc %i2, 0x8, %g0 be,pt %XCC, 1f nop sub %i2, 0x8, %i2 - EX_LD(LOAD(ldx, %i1, %o4)) - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) add %i1, 0x8, %i1 1: andcc %i2, 0x4, %g0 be,pt %XCC, 1f nop sub %i2, 0x4, %i2 - EX_LD(LOAD(lduw, %i1, %i5)) - EX_ST(STORE(stw, %i5, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) add %i1, 0x4, %i1 1: cmp %i2, 0 be,pt %XCC, 85f @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %i2, %g1, %i2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %i1, %i5)) - EX_ST(STORE(stb, %i5, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) bgu,pt %icc, 1b add %i1, 1, %i1 @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 8: mov 64, %i3 andn %i1, 0x7, %i1 - EX_LD(LOAD(ldx, %i1, %g2)) + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) sub %i3, %g1, %i3 andn %i2, 0x7, %i4 sllx %g2, %g1, %g2 1: add %i1, 0x8, %i1 - EX_LD(LOAD(ldx, %i1, %g3)) + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) subcc %i4, 0x8, %i4 srlx %g3, %i3, %i5 or %i5, %g2, %i5 - EX_ST(STORE(stx, %i5, %o0)) + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 1: subcc %i2, 4, %i2 - EX_LD(LOAD(lduw, %i1, %g1)) - EX_ST(STORE(stw, %g1, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) bgu,pt %XCC, 1b add %i1, 4, %i1 @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ .align 32 90: subcc %i2, 1, %i2 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) bgu,pt %XCC, 90b add %i1, 1, %i1 ret diff --git a/kernel/arch/sparc/lib/U1copy_from_user.S b/kernel/arch/sparc/lib/U1copy_from_user.S index ecc5692fa..bb6ff7322 100644 --- a/kernel/arch/sparc/lib/U1copy_from_user.S +++ b/kernel/arch/sparc/lib/U1copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/U1copy_to_user.S b/kernel/arch/sparc/lib/U1copy_to_user.S index 9eea392e4..ed92ce739 100644 --- a/kernel/arch/sparc/lib/U1copy_to_user.S +++ b/kernel/arch/sparc/lib/U1copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/U1memcpy.S b/kernel/arch/sparc/lib/U1memcpy.S index 3e6209ebb..f30d2ab2c 100644 --- a/kernel/arch/sparc/lib/U1memcpy.S +++ b/kernel/arch/sparc/lib/U1memcpy.S @@ -5,6 +5,7 @@ */ #ifdef __KERNEL__ +#include #include #include #define GLOBAL_SPARE g7 @@ -23,21 +24,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -78,53 +75,169 @@ faligndata %f7, %f8, %f60; \ faligndata %f8, %f9, %f62; -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ - EX_LD_FP(LOAD_BLK(%src, %fdest)); \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ - add %src, 0x40, %src; \ - subcc %len, 0x40, %len; \ - be,pn %xcc, jmptgt; \ - add %dest, 0x40, %dest; \ - -#define LOOP_CHUNK1(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) -#define LOOP_CHUNK2(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) -#define LOOP_CHUNK3(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ + add %src, 0x40, %src; \ + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ + be,pn %xcc, jmptgt; \ + add %dest, 0x40, %dest; \ + +#define LOOP_CHUNK1(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) +#define LOOP_CHUNK2(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) +#define LOOP_CHUNK3(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) #define DO_SYNC membar #Sync; #define STORE_SYNC(dest, fsrc) \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ add %dest, 0x40, %dest; \ DO_SYNC #define STORE_JUMP(dest, fsrc, target) \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ add %dest, 0x40, %dest; \ ba,pt %xcc, target; \ nop; -#define FINISH_VISCHUNK(dest, f0, f1, left) \ - subcc %left, 8, %left;\ - bl,pn %xcc, 95f; \ - faligndata %f0, %f1, %f48; \ - EX_ST_FP(STORE(std, %f48, %dest)); \ +#define FINISH_VISCHUNK(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ + faligndata %f0, %f1, %f48; \ + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ add %dest, 8, %dest; -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ - subcc %left, 8, %left; \ - bl,pn %xcc, 95f; \ +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ fsrc2 %f0, %f1; -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ +#define UNEVEN_VISCHUNK(dest, f0, f1) \ + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ ba,a,pt %xcc, 93f; .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(U1_g1_1_fp) + VISExitHalf + add %g1, 1, %g1 + add %g1, %g2, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1_fp) +ENTRY(U1_g2_0_fp) + VISExitHalf + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_0_fp) +ENTRY(U1_g2_8_fp) + VISExitHalf + add %g2, 8, %g2 + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_8_fp) +ENTRY(U1_gs_0_fp) + VISExitHalf + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_0_fp) +ENTRY(U1_gs_80_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_80_fp) +ENTRY(U1_gs_40_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_40_fp) +ENTRY(U1_g3_0_fp) + VISExitHalf + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_0_fp) +ENTRY(U1_g3_8_fp) + VISExitHalf + add %g3, 8, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_8_fp) +ENTRY(U1_o2_0_fp) + VISExitHalf + retl + mov %o2, %o0 +ENDPROC(U1_o2_0_fp) +ENTRY(U1_o2_1_fp) + VISExitHalf + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1_fp) +ENTRY(U1_gs_0) + VISExitHalf + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0) +ENTRY(U1_gs_8) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x8, %o0 +ENDPROC(U1_gs_8) +ENTRY(U1_gs_10) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x10, %o0 +ENDPROC(U1_gs_10) +ENTRY(U1_o2_0) + retl + mov %o2, %o0 +ENDPROC(U1_o2_0) +ENTRY(U1_o2_8) + retl + add %o2, 8, %o0 +ENDPROC(U1_o2_8) +ENTRY(U1_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(U1_o2_4) +ENTRY(U1_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1) +ENTRY(U1_g1_0) + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_0) +ENTRY(U1_g1_1) + add %g1, 1, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1) +ENTRY(U1_gs_0_o2_adj) + and %o2, 7, %o2 + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0_o2_adj) +ENTRY(U1_gs_8_o2_adj) + and %o2, 7, %o2 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_8_o2_adj) +#endif + .align 64 .globl FUNC_NAME @@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD_FP(LOAD(ldd, %o1, %f4)) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ add %g1, %GLOBAL_SPARE, %g1 subcc %o2, %g3, %o2 - EX_LD_FP(LOAD_BLK(%o1, %f0)) + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) add %o1, 0x40, %o1 add %g1, %g3, %g1 - EX_LD_FP(LOAD_BLK(%o1, %f16)) + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) add %o1, 0x40, %o1 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE - EX_LD_FP(LOAD_BLK(%o1, %f32)) + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) add %o1, 0x40, %o1 /* There are 8 instances of the unrolled loop, @@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f0, %f2, %f48 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) @@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 56f) 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f2, %f4, %f48 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) @@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 57f) 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f4, %f6, %f48 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) @@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 58f) 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f6, %f8, %f48 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) @@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 59f) 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f8, %f10, %f48 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) @@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 60f) 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f10, %f12, %f48 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) @@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 61f) 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f12, %f14, %f48 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) @@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 62f) 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f14, %f16, %f48 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) @@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, 63f) -40: FINISH_VISCHUNK(o0, f0, f2, g3) -41: FINISH_VISCHUNK(o0, f2, f4, g3) -42: FINISH_VISCHUNK(o0, f4, f6, g3) -43: FINISH_VISCHUNK(o0, f6, f8, g3) -44: FINISH_VISCHUNK(o0, f8, f10, g3) -45: FINISH_VISCHUNK(o0, f10, f12, g3) -46: FINISH_VISCHUNK(o0, f12, f14, g3) -47: UNEVEN_VISCHUNK(o0, f14, f0, g3) -48: FINISH_VISCHUNK(o0, f16, f18, g3) -49: FINISH_VISCHUNK(o0, f18, f20, g3) -50: FINISH_VISCHUNK(o0, f20, f22, g3) -51: FINISH_VISCHUNK(o0, f22, f24, g3) -52: FINISH_VISCHUNK(o0, f24, f26, g3) -53: FINISH_VISCHUNK(o0, f26, f28, g3) -54: FINISH_VISCHUNK(o0, f28, f30, g3) -55: UNEVEN_VISCHUNK(o0, f30, f0, g3) -56: FINISH_VISCHUNK(o0, f32, f34, g3) -57: FINISH_VISCHUNK(o0, f34, f36, g3) -58: FINISH_VISCHUNK(o0, f36, f38, g3) -59: FINISH_VISCHUNK(o0, f38, f40, g3) -60: FINISH_VISCHUNK(o0, f40, f42, g3) -61: FINISH_VISCHUNK(o0, f42, f44, g3) -62: FINISH_VISCHUNK(o0, f44, f46, g3) -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) - -93: EX_LD_FP(LOAD(ldd, %o1, %f2)) +40: FINISH_VISCHUNK(o0, f0, f2) +41: FINISH_VISCHUNK(o0, f2, f4) +42: FINISH_VISCHUNK(o0, f4, f6) +43: FINISH_VISCHUNK(o0, f6, f8) +44: FINISH_VISCHUNK(o0, f8, f10) +45: FINISH_VISCHUNK(o0, f10, f12) +46: FINISH_VISCHUNK(o0, f12, f14) +47: UNEVEN_VISCHUNK(o0, f14, f0) +48: FINISH_VISCHUNK(o0, f16, f18) +49: FINISH_VISCHUNK(o0, f18, f20) +50: FINISH_VISCHUNK(o0, f20, f22) +51: FINISH_VISCHUNK(o0, f22, f24) +52: FINISH_VISCHUNK(o0, f24, f26) +53: FINISH_VISCHUNK(o0, f26, f28) +54: FINISH_VISCHUNK(o0, f28, f30) +55: UNEVEN_VISCHUNK(o0, f30, f0) +56: FINISH_VISCHUNK(o0, f32, f34) +57: FINISH_VISCHUNK(o0, f34, f36) +58: FINISH_VISCHUNK(o0, f36, f38) +59: FINISH_VISCHUNK(o0, f38, f40) +60: FINISH_VISCHUNK(o0, f40, f42) +61: FINISH_VISCHUNK(o0, f42, f44) +62: FINISH_VISCHUNK(o0, f44, f46) +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) + +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f0, %f2, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) bl,pn %xcc, 95f add %o0, 8, %o0 - EX_LD_FP(LOAD(ldd, %o1, %f0)) + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f2, %f0, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) bge,pt %xcc, 93b add %o0, 8, %o0 95: brz,pt %o2, 2f mov %g1, %o1 -1: EX_LD_FP(LOAD(ldub, %o1, %o3)) +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) add %o1, 1, %o1 subcc %o2, 1, %o2 - EX_ST_FP(STORE(stb, %o3, %o0)) + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) bne,pt %xcc, 1b add %o0, 1, %o0 @@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 72: andn %o2, 0xf, %GLOBAL_SPARE and %o2, 0xf, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) sub %o2, 0x8, %o2 - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) sub %o2, 0x4, %o2 - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %g1, %g1 sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1, %o5)) +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) subcc %g1, 1, %g1 - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) sub %o3, %g1, %o3 andn %o2, 0x7, %GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ bne,pn %XCC, 90f sub %o0, %o1, %o3 -1: EX_LD(LOAD(lduw, %o1, %g1)) +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) subcc %o2, 4, %o2 - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov EX_RETVAL(%o4), %o0 .align 32 -90: EX_LD(LOAD(ldub, %o1, %g1)) +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) subcc %o2, 1, %o2 - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/kernel/arch/sparc/lib/U3copy_from_user.S b/kernel/arch/sparc/lib/U3copy_from_user.S index 88ad73d86..db73010a1 100644 --- a/kernel/arch/sparc/lib/U3copy_from_user.S +++ b/kernel/arch/sparc/lib/U3copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/U3copy_to_user.S b/kernel/arch/sparc/lib/U3copy_to_user.S index 845139d75..c4ee858e3 100644 --- a/kernel/arch/sparc/lib/U3copy_to_user.S +++ b/kernel/arch/sparc/lib/U3copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/kernel/arch/sparc/lib/U3memcpy.S b/kernel/arch/sparc/lib/U3memcpy.S index 491ee69e4..54f98706b 100644 --- a/kernel/arch/sparc/lib/U3memcpy.S +++ b/kernel/arch/sparc/lib/U3memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include #include #include #define GLOBAL_SPARE %g7 @@ -22,21 +23,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -77,6 +74,87 @@ */ .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf + retl + nop +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) + add %g1, 1, %g1 + add %g2, %g1, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) +ENTRY(U3_retl_o2_plus_g2_fp) + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_fp) +ENTRY(U3_retl_o2_plus_g2_plus_8_fp) + add %g2, 8, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) +ENTRY(U3_retl_o2) + retl + mov %o2, %o0 +ENDPROC(U3_retl_o2) +ENTRY(U3_retl_o2_plus_1) + retl + add %o2, 1, %o0 +ENDPROC(U3_retl_o2_plus_1) +ENTRY(U3_retl_o2_plus_4) + retl + add %o2, 4, %o0 +ENDPROC(U3_retl_o2_plus_4) +ENTRY(U3_retl_o2_plus_8) + retl + add %o2, 8, %o0 +ENDPROC(U3_retl_o2_plus_8) +ENTRY(U3_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + retl + add %o2, %g1, %o0 +ENDPROC(U3_retl_o2_plus_g1_plus_1) +ENTRY(U3_retl_o2_fp) + ba,pt %xcc, __restore_fp + mov %o2, %o0 +ENDPROC(U3_retl_o2_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) + sll %o3, 6, %o3 + add %o3, 0x80, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) + sll %o3, 6, %o3 + add %o3, 0x40, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) +ENTRY(U3_retl_o2_plus_GS_plus_0x10) + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10) +ENTRY(U3_retl_o2_plus_GS_plus_0x08) + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08) +ENTRY(U3_retl_o2_and_7_plus_GS) + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o2 +ENDPROC(U3_retl_o2_and_7_plus_GS) +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) + add GLOBAL_SPARE, 8, GLOBAL_SPARE + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o2 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) +#endif + .align 64 /* The cheetah's flexible spine, oversized liver, enlarged heart, @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD_FP(LOAD(ldd, %o1, %f4)) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f2 - EX_ST_FP(STORE(std, %f2, %o0)) + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ LOAD(prefetch, %o1 + 0x080, #one_read) LOAD(prefetch, %o1 + 0x0c0, #one_read) LOAD(prefetch, %o1 + 0x100, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) LOAD(prefetch, %o1 + 0x140, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) LOAD(prefetch, %o1 + 0x180, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) faligndata %f8, %f10, %f24 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) faligndata %f10, %f12, %f26 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE add %o1, 0x40, %o1 @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 add %o0, 0x40, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) subcc %o3, 0x01, %o3 faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f8, %f10, %f24 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f10, %f12, %f26 bg,pt %XCC, 1b @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ /* Finally we copy the last full 64-byte block. */ 2: - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f8, %f10, %f24 cmp %g1, 0 be,pt %XCC, 1f add %o0, 0x40, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) 1: faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) add %o0, 0x40, %o0 add %o1, 0x40, %o1 membar #Sync @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g2, %o2 be,a,pt %XCC, 1f - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2)) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f0, %f2, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %XCC, 2f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f2, %f0, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) bne,pn %XCC, 1b add %o0, 0x8, %o0 @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andcc %o2, 0x8, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x8, %o1 + sub %o2, 8, %o2 1: andcc %o2, 0x4, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x4, %o1 + sub %o2, 4, %o2 1: andcc %o2, 0x2, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduh, %o1, %o5)) - EX_ST(STORE(sth, %o5, %o1 + %o3)) + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x2, %o1 + sub %o2, 2, %o2 1: andcc %o2, 0x1, %g0 be,pt %icc, 85f nop - EX_LD(LOAD(ldub, %o1, %o5)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) ba,pt %xcc, 85f - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) .align 64 70: /* 16 < len <= 64 */ @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, GLOBAL_SPARE and %o2, 0xf, %o2 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE - EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) sub %o3, %g1, %o3 andn %o2, 0x7, GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/kernel/arch/sparc/lib/copy_in_user.S b/kernel/arch/sparc/lib/copy_in_user.S index 302c0e60d..4c89b486f 100644 --- a/kernel/arch/sparc/lib/copy_in_user.S +++ b/kernel/arch/sparc/lib/copy_in_user.S @@ -8,18 +8,33 @@ #define XCC xcc -#define EX(x,y) \ +#define EX(x,y,z) \ 98: x,y; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, z; \ .text; \ .align 4; +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) + .register %g2,#scratch .register %g3,#scratch .text +__retl_o4_plus_8: + add %o4, %o2, %o4 + retl + add %o4, 8, %o0 +__retl_o2_plus_4: + retl + add %o2, 4, %o0 +__retl_o2_plus_1: + retl + add %o2, 1, %o0 + .align 32 /* Don't try to get too fancy here, just nice and @@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %o4 and %o2, 0x7, %o2 1: subcc %o4, 0x8, %o4 - EX(ldxa [%o1] %asi, %o5) - EX(stxa %o5, [%o0] %asi) + EX_O4(ldxa [%o1] %asi, %o5) + EX_O4(stxa %o5, [%o0] %asi) add %o1, 0x8, %o1 bgu,pt %XCC, 1b add %o0, 0x8, %o0 @@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX(lduwa [%o1] %asi, %o5) - EX(stwa %o5, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %o5) + EX_O2_4(stwa %o5, [%o0] %asi) add %o1, 0x4, %o1 add %o0, 0x4, %o0 1: cmp %o2, 0 @@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ 82: subcc %o2, 4, %o2 - EX(lduwa [%o1] %asi, %g1) - EX(stwa %g1, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %g1) + EX_O2_4(stwa %g1, [%o0] %asi) add %o1, 4, %o1 bgu,pt %XCC, 82b add %o0, 4, %o0 @@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX(lduba [%o1] %asi, %g1) - EX(stba %g1, [%o0] %asi) + EX_O2_1(lduba [%o1] %asi, %g1) + EX_O2_1(stba %g1, [%o0] %asi) add %o1, 1, %o1 bgu,pt %XCC, 90b add %o0, 1, %o0 diff --git a/kernel/arch/sparc/lib/user_fixup.c b/kernel/arch/sparc/lib/user_fixup.c deleted file mode 100644 index ac96ae236..000000000 --- a/kernel/arch/sparc/lib/user_fixup.c +++ /dev/null @@ -1,71 +0,0 @@ -/* user_fixup.c: Fix up user copy faults. - * - * Copyright (C) 2004 David S. Miller - */ - -#include -#include -#include -#include -#include - -#include - -/* Calculating the exact fault address when using - * block loads and stores can be very complicated. - * - * Instead of trying to be clever and handling all - * of the cases, just fix things up simply here. - */ - -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long end = start + size; - - if (fault_addr < start || fault_addr >= end) { - *offset = 0; - } else { - *offset = fault_addr - start; - size = end - fault_addr; - } - return size; -} - -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) -{ - unsigned long offset; - - size = compute_size((unsigned long) from, size, &offset); - if (likely(size)) - memset(to + offset, 0, size); - - return size; -} -EXPORT_SYMBOL(copy_from_user_fixup); - -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) -{ - unsigned long offset; - - return compute_size((unsigned long) to, size, &offset); -} -EXPORT_SYMBOL(copy_to_user_fixup); - -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long start = (unsigned long) to; - unsigned long end = start + size; - - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - start = (unsigned long) from; - end = start + size; - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - return size; -} -EXPORT_SYMBOL(copy_in_user_fixup); diff --git a/kernel/arch/sparc/mm/fault_64.c b/kernel/arch/sparc/mm/fault_64.c index dbabe5713..e15f33715 100644 --- a/kernel/arch/sparc/mm/fault_64.c +++ b/kernel/arch/sparc/mm/fault_64.c @@ -479,14 +479,14 @@ good_area: up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); #endif if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss = mm->context.huge_pte_count; + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count; if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { if (mm->context.tsb_block[MM_TSB_HUGE].tsb) diff --git a/kernel/arch/sparc/mm/hugetlbpage.c b/kernel/arch/sparc/mm/hugetlbpage.c index 131eaf4ad..da1142401 100644 --- a/kernel/arch/sparc/mm/hugetlbpage.c +++ b/kernel/arch/sparc/mm/hugetlbpage.c @@ -176,17 +176,31 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { int i; + pte_t orig[2]; + unsigned long nptes; if (!pte_present(*ptep) && pte_present(entry)) - mm->context.huge_pte_count++; + mm->context.hugetlb_pte_count++; addr &= HPAGE_MASK; - for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - set_pte_at(mm, addr, ptep, entry); + + nptes = 1 << HUGETLB_PAGE_ORDER; + orig[0] = *ptep; + orig[1] = *(ptep + nptes / 2); + for (i = 0; i < nptes; i++) { + *ptep = entry; ptep++; addr += PAGE_SIZE; pte_val(entry) += PAGE_SIZE; } + + /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ + addr -= REAL_HPAGE_SIZE; + ptep -= nptes / 2; + maybe_tlb_batch_add(mm, addr, ptep, orig[1], 0); + addr -= REAL_HPAGE_SIZE; + ptep -= nptes / 2; + maybe_tlb_batch_add(mm, addr, ptep, orig[0], 0); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, @@ -194,19 +208,28 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, { pte_t entry; int i; + unsigned long nptes; entry = *ptep; if (pte_present(entry)) - mm->context.huge_pte_count--; + mm->context.hugetlb_pte_count--; addr &= HPAGE_MASK; - - for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - pte_clear(mm, addr, ptep); + nptes = 1 << HUGETLB_PAGE_ORDER; + for (i = 0; i < nptes; i++) { + *ptep = __pte(0UL); addr += PAGE_SIZE; ptep++; } + /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ + addr -= REAL_HPAGE_SIZE; + ptep -= nptes / 2; + maybe_tlb_batch_add(mm, addr, ptep, entry, 0); + addr -= REAL_HPAGE_SIZE; + ptep -= nptes / 2; + maybe_tlb_batch_add(mm, addr, ptep, entry, 0); + return entry; } diff --git a/kernel/arch/sparc/mm/init_64.c b/kernel/arch/sparc/mm/init_64.c index 3025bd57f..3d3414c14 100644 --- a/kernel/arch/sparc/mm/init_64.c +++ b/kernel/arch/sparc/mm/init_64.c @@ -324,18 +324,6 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde tsb_insert(tsb, tag, tte); } -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline bool is_hugetlb_pte(pte_t pte) -{ - if ((tlb_type == hypervisor && - (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || - (tlb_type != hypervisor && - (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) - return true; - return false; -} -#endif - void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { struct mm_struct *mm; @@ -358,7 +346,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * spin_lock_irqsave(&mm->context.lock, flags); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) + if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) && + is_hugetlb_pte(pte)) __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, address, pte_val(pte)); else @@ -811,8 +800,10 @@ struct mdesc_mblock { }; static struct mdesc_mblock *mblocks; static int num_mblocks; +static int find_numa_node_for_addr(unsigned long pa, + struct node_mem_mask *pnode_mask); -static unsigned long ra_to_pa(unsigned long addr) +static unsigned long __init ra_to_pa(unsigned long addr) { int i; @@ -828,8 +819,11 @@ static unsigned long ra_to_pa(unsigned long addr) return addr; } -static int find_node(unsigned long addr) +static int __init find_node(unsigned long addr) { + static bool search_mdesc = true; + static struct node_mem_mask last_mem_mask = { ~0UL, ~0UL }; + static int last_index; int i; addr = ra_to_pa(addr); @@ -839,13 +833,30 @@ static int find_node(unsigned long addr) if ((addr & p->mask) == p->val) return i; } - /* The following condition has been observed on LDOM guests.*/ - WARN_ONCE(1, "find_node: A physical address doesn't match a NUMA node" - " rule. Some physical memory will be owned by node 0."); - return 0; + /* The following condition has been observed on LDOM guests because + * node_masks only contains the best latency mask and value. + * LDOM guest's mdesc can contain a single latency group to + * cover multiple address range. Print warning message only if the + * address cannot be found in node_masks nor mdesc. + */ + if ((search_mdesc) && + ((addr & last_mem_mask.mask) != last_mem_mask.val)) { + /* find the available node in the mdesc */ + last_index = find_numa_node_for_addr(addr, &last_mem_mask); + numadbg("find_node: latency group for address 0x%lx is %d\n", + addr, last_index); + if ((last_index < 0) || (last_index >= num_node_masks)) { + /* WARN_ONCE() and use default group 0 */ + WARN_ONCE(1, "find_node: A physical address doesn't match a NUMA node rule. Some physical memory will be owned by node 0."); + search_mdesc = false; + last_index = 0; + } + } + + return last_index; } -static u64 memblock_nid_range(u64 start, u64 end, int *nid) +static u64 __init memblock_nid_range(u64 start, u64 end, int *nid) { *nid = find_node(start); start += PAGE_SIZE; @@ -1169,6 +1180,41 @@ int __node_distance(int from, int to) return numa_latency[from][to]; } +static int find_numa_node_for_addr(unsigned long pa, + struct node_mem_mask *pnode_mask) +{ + struct mdesc_handle *md = mdesc_grab(); + u64 node, arc; + int i = 0; + + node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); + if (node == MDESC_NODE_NULL) + goto out; + + mdesc_for_each_node_by_name(md, node, "group") { + mdesc_for_each_arc(arc, md, node, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + struct mdesc_mlgroup *m = find_mlgroup(target); + + if (!m) + continue; + if ((pa & m->mask) == m->match) { + if (pnode_mask) { + pnode_mask->mask = m->mask; + pnode_mask->val = m->match; + } + mdesc_release(md); + return i; + } + } + i++; + } + +out: + mdesc_release(md); + return -1; +} + static int find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) { int i; @@ -1267,13 +1313,6 @@ static int __init numa_parse_mdesc(void) int i, j, err, count; u64 node; - /* Some sane defaults for numa latency values */ - for (i = 0; i < MAX_NUMNODES; i++) { - for (j = 0; j < MAX_NUMNODES; j++) - numa_latency[i][j] = (i == j) ? - LOCAL_DISTANCE : REMOTE_DISTANCE; - } - node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); if (node == MDESC_NODE_NULL) { mdesc_release(md); @@ -1369,10 +1408,18 @@ static int __init numa_parse_sun4u(void) static int __init bootmem_init_numa(void) { + int i, j; int err = -1; numadbg("bootmem_init_numa()\n"); + /* Some sane defaults for numa latency values */ + for (i = 0; i < MAX_NUMNODES; i++) { + for (j = 0; j < MAX_NUMNODES; j++) + numa_latency[i][j] = (i == j) ? + LOCAL_DISTANCE : REMOTE_DISTANCE; + } + if (numa_enabled) { if (tlb_type == hypervisor) err = numa_parse_mdesc(); @@ -2832,9 +2879,10 @@ void hugetlb_setup(struct pt_regs *regs) * the Data-TLB for huge pages. */ if (tlb_type == cheetah_plus) { + bool need_context_reload = false; unsigned long ctx; - spin_lock(&ctx_alloc_lock); + spin_lock_irq(&ctx_alloc_lock); ctx = mm->context.sparc64_ctx_val; ctx &= ~CTX_PGSZ_MASK; ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; @@ -2853,9 +2901,12 @@ void hugetlb_setup(struct pt_regs *regs) * also executing in this address space. */ mm->context.sparc64_ctx_val = ctx; - on_each_cpu(context_reload, mm, 0); + need_context_reload = true; } - spin_unlock(&ctx_alloc_lock); + spin_unlock_irq(&ctx_alloc_lock); + + if (need_context_reload) + on_each_cpu(context_reload, mm, 0); } } #endif diff --git a/kernel/arch/sparc/mm/tlb.c b/kernel/arch/sparc/mm/tlb.c index 9df2190c0..3659d37b4 100644 --- a/kernel/arch/sparc/mm/tlb.c +++ b/kernel/arch/sparc/mm/tlb.c @@ -67,7 +67,7 @@ void arch_leave_lazy_mmu_mode(void) } static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, - bool exec) + bool exec, bool huge) { struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; @@ -84,13 +84,21 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, } if (!tb->active) { - flush_tsb_user_page(mm, vaddr); + flush_tsb_user_page(mm, vaddr, huge); global_flush_tlb_page(mm, vaddr); goto out; } - if (nr == 0) + if (nr == 0) { tb->mm = mm; + tb->huge = huge; + } + + if (tb->huge != huge) { + flush_tlb_pending(); + tb->huge = huge; + nr = 0; + } tb->vaddrs[nr] = vaddr; tb->tlb_nr = ++nr; @@ -104,6 +112,8 @@ out: void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t orig, int fullmm) { + bool huge = is_hugetlb_pte(orig); + if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); @@ -129,7 +139,7 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, no_cache_flush: if (!fullmm) - tlb_batch_add_one(mm, vaddr, pte_exec(orig)); + tlb_batch_add_one(mm, vaddr, pte_exec(orig), huge); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -145,7 +155,7 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, if (pte_val(*pte) & _PAGE_VALID) { bool exec = pte_exec(*pte); - tlb_batch_add_one(mm, vaddr, exec); + tlb_batch_add_one(mm, vaddr, exec, false); } pte++; vaddr += PAGE_SIZE; @@ -165,9 +175,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { if (pmd_val(pmd) & _PAGE_PMD_HUGE) - mm->context.huge_pte_count++; + mm->context.thp_pte_count++; else - mm->context.huge_pte_count--; + mm->context.thp_pte_count--; /* Do not try to allocate the TSB hash table if we * don't have one already. We have various locks held @@ -185,8 +195,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pte_t orig_pte = __pte(pmd_val(orig)); bool exec = pte_exec(orig_pte); - tlb_batch_add_one(mm, addr, exec); - tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); + tlb_batch_add_one(mm, addr, exec, true); + tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, + true); } else { tlb_batch_pmd_scan(mm, addr, orig); } diff --git a/kernel/arch/sparc/mm/tsb.c b/kernel/arch/sparc/mm/tsb.c index a06576683..9cdeca0fa 100644 --- a/kernel/arch/sparc/mm/tsb.c +++ b/kernel/arch/sparc/mm/tsb.c @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) return (tag == (vaddr >> 22)); } +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) +{ + unsigned long idx; + + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { + struct tsb *ent = &swapper_tsb[idx]; + unsigned long match = idx << 13; + + match |= (ent->tag << 22); + if (match >= start && match < end) + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +} + /* TSB flushes need only occur on the processor initiating the address * space modification, not on each cpu the address space has run on. * Only the TLB flush needs that treatment. @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) { unsigned long v; + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) + return flush_tsb_kernel_range_scan(start, end); + for (v = start; v < end; v += PAGE_SIZE) { unsigned long hash = tsb_hash(v, PAGE_SHIFT, KERNEL_TSB_NENTRIES); @@ -76,14 +93,15 @@ void flush_tsb_user(struct tlb_batch *tb) spin_lock_irqsave(&mm->context.lock, flags); - base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; - nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; - if (tlb_type == cheetah_plus || tlb_type == hypervisor) - base = __pa(base); - __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); - + if (!tb->huge) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); + } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + if (tb->huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) @@ -94,20 +112,21 @@ void flush_tsb_user(struct tlb_batch *tb) spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, bool huge) { unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); - base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; - nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; - if (tlb_type == cheetah_plus || tlb_type == hypervisor) - base = __pa(base); - __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); - + if (!huge) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); + } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + if (huge && mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) @@ -468,7 +487,7 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - unsigned long huge_pte_count; + unsigned long total_huge_pte_count; #endif unsigned int i; @@ -477,12 +496,14 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - /* We reset it to zero because the fork() page copying + /* We reset them to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. */ - huge_pte_count = mm->context.huge_pte_count; - mm->context.huge_pte_count = 0; + total_huge_pte_count = mm->context.hugetlb_pte_count + + mm->context.thp_pte_count; + mm->context.hugetlb_pte_count = 0; + mm->context.thp_pte_count = 0; #endif /* copy_mm() copies over the parent's mm_struct before calling @@ -498,8 +519,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (unlikely(huge_pte_count)) - tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); + if (unlikely(total_huge_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, total_huge_pte_count); #endif if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) diff --git a/kernel/arch/sparc/mm/ultra.S b/kernel/arch/sparc/mm/ultra.S index b4f4733ab..5d2fd6cd3 100644 --- a/kernel/arch/sparc/mm/ultra.S +++ b/kernel/arch/sparc/mm/ultra.S @@ -30,7 +30,7 @@ .text .align 32 .globl __flush_tlb_mm -__flush_tlb_mm: /* 18 insns */ +__flush_tlb_mm: /* 19 insns */ /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ ldxa [%o1] ASI_DMMU, %g2 cmp %g2, %o0 @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */ .align 32 .globl __flush_tlb_pending -__flush_tlb_pending: /* 26 insns */ +__flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ rdpr %pstate, %g7 sllx %o1, 3, %o1 @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */ .align 32 .globl __flush_tlb_kernel_range -__flush_tlb_kernel_range: /* 16 insns */ +__flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow sethi %hi(PAGE_SIZE), %o4 - sub %o1, %o0, %o3 sub %o3, %o4, %o3 or %o0, 0x20, %o0 ! Nucleus 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */ retl nop nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__spitfire_flush_tlb_kernel_range_slow: + mov 63 * 8, %o4 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_IMMU + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_DMMU + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %o4, 8, %o4 + brgez,pt %o4, 1b + nop + retl + nop __spitfire_flush_tlb_mm_slow: rdpr %pstate, %g1 @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */ retl wrpr %g7, 0x0, %pstate +__cheetah_flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, 3f + sethi %hi(PAGE_SIZE), %o4 + sub %o3, %o4, %o3 + or %o0, 0x20, %o0 ! Nucleus +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o3, 1b + sub %o3, %o4, %o3 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop +3: mov 0x80, %o4 + stxa %g0, [%o4] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%o4] ASI_IMMU_DEMAP + membar #Sync + retl + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE __cheetah_flush_dcache_page: /* 11 insns */ sethi %hi(PAGE_OFFSET), %g1 @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error: ret restore -__hypervisor_flush_tlb_mm: /* 10 insns */ +__hypervisor_flush_tlb_mm: /* 19 insns */ mov %o0, %o2 /* ARG2: mmu context */ mov 0, %o0 /* ARG0: CPU lists unimplemented */ mov 0, %o1 /* ARG1: CPU lists unimplemented */ mov HV_MMU_ALL, %o3 /* ARG3: flags */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_FAST_MMU_DEMAP_CTX, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_page: /* 11 insns */ +__hypervisor_flush_tlb_page: /* 22 insns */ /* %o0 = context, %o1 = vaddr */ mov %o0, %g2 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_pending: /* 16 insns */ +__hypervisor_flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ sllx %o1, 3, %g1 mov %o2, %g2 @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g1, 1b nop retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_kernel_range: /* 16 insns */ +__hypervisor_flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f - sethi %hi(PAGE_SIZE), %g3 - mov %o0, %g1 - sub %o1, %g1, %g2 + sub %o1, %o0, %g2 + srlx %g2, 18, %g3 + brnz,pn %g3, 4f + mov %o0, %g1 + sethi %hi(PAGE_SIZE), %g3 sub %g2, %g3, %g2 1: add %g1, %g2, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 3f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g2, 1b sub %g2, %g3, %g2 2: retl nop +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 3b + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop #ifdef DCACHE_ALIASING_POSSIBLE /* XXX Niagara and friends have an 8K cache, so no aliasing is @@ -394,43 +511,6 @@ tlb_patch_one: retl nop - .globl cheetah_patch_cachetlbops -cheetah_patch_cachetlbops: - save %sp, -128, %sp - - sethi %hi(__flush_tlb_mm), %o0 - or %o0, %lo(__flush_tlb_mm), %o0 - sethi %hi(__cheetah_flush_tlb_mm), %o1 - or %o1, %lo(__cheetah_flush_tlb_mm), %o1 - call tlb_patch_one - mov 19, %o2 - - sethi %hi(__flush_tlb_page), %o0 - or %o0, %lo(__flush_tlb_page), %o0 - sethi %hi(__cheetah_flush_tlb_page), %o1 - or %o1, %lo(__cheetah_flush_tlb_page), %o1 - call tlb_patch_one - mov 22, %o2 - - sethi %hi(__flush_tlb_pending), %o0 - or %o0, %lo(__flush_tlb_pending), %o0 - sethi %hi(__cheetah_flush_tlb_pending), %o1 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1 - call tlb_patch_one - mov 27, %o2 - -#ifdef DCACHE_ALIASING_POSSIBLE - sethi %hi(__flush_dcache_page), %o0 - or %o0, %lo(__flush_dcache_page), %o0 - sethi %hi(__cheetah_flush_dcache_page), %o1 - or %o1, %lo(__cheetah_flush_dcache_page), %o1 - call tlb_patch_one - mov 11, %o2 -#endif /* DCACHE_ALIASING_POSSIBLE */ - - ret - restore - #ifdef CONFIG_SMP /* These are all called by the slaves of a cross call, at * trap level 1, with interrupts fully disabled. @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops: */ .align 32 .globl xcall_flush_tlb_mm -xcall_flush_tlb_mm: /* 21 insns */ +xcall_flush_tlb_mm: /* 24 insns */ mov PRIMARY_CONTEXT, %g2 ldxa [%g2] ASI_DMMU, %g3 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */ nop nop nop + nop + nop + nop .globl xcall_flush_tlb_page -xcall_flush_tlb_page: /* 17 insns */ +xcall_flush_tlb_page: /* 20 insns */ /* %g5=context, %g1=vaddr */ mov PRIMARY_CONTEXT, %g4 ldxa [%g4] ASI_DMMU, %g2 @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */ retry nop nop + nop + nop + nop .globl xcall_flush_tlb_kernel_range -xcall_flush_tlb_kernel_range: /* 25 insns */ +xcall_flush_tlb_kernel_range: /* 44 insns */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 - add %g2, 1, %g2 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + add %g2, 1, %g2 sub %g3, %g2, %g3 or %g1, 0x20, %g1 ! Nucleus 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */ brnz,pt %g3, 1b sub %g3, %g2, %g3 retry - nop - nop +2: mov 63 * 8, %g1 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_IMMU + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_DMMU + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %g1, 8, %g1 + brgez,pt %g1, 1b + nop + retry nop nop nop @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4: retry +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + add %g2, 1, %g2 + sub %g3, %g2, %g3 + or %g1, 0x20, %g1 ! Nucleus +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + retry +2: mov 0x80, %g2 + stxa %g0, [%g2] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%g2] ASI_IMMU_DEMAP + membar #Sync + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error: ba,a,pt %xcc, rtrap .globl __hypervisor_xcall_flush_tlb_mm -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ mov %o0, %g2 mov %o1, %g3 @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP mov HV_FAST_MMU_DEMAP_CTX, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov %g7, %o5 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_page -__hypervisor_xcall_flush_tlb_page: /* 17 insns */ +__hypervisor_xcall_flush_tlb_page: /* 20 insns */ /* %g5=ctx, %g1=vaddr */ mov %o0, %g2 mov %o1, %g3 @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */ sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,a,pn %o0, __hypervisor_tlb_xcall_error + brnz,a,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 mov %g4, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_kernel_range -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 + srlx %g3, 18, %g7 add %g2, 1, %g2 sub %g3, %g2, %g3 mov %o0, %g2 mov %o1, %g4 - mov %o2, %g7 + brnz,pn %g7, 2f + mov %o2, %g7 1: add %g1, %g3, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 sethi %hi(PAGE_SIZE), %o2 brnz,pt %g3, 1b sub %g3, %o2, %g3 - mov %g2, %o0 +5: mov %g2, %o0 mov %g4, %o1 mov %g7, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop +2: mov %o3, %g1 + mov %o5, %g3 + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov %g1, %o3 + brz,pt %o0, 5b + mov %g3, %o5 + mov HV_FAST_MMU_DEMAP_CTX, %g6 + ba,pt %xcc, 1b + clr %g5 /* These just get rescheduled to PIL vectors. */ .globl xcall_call_function @@ -809,6 +985,58 @@ xcall_kgdb_capture: #endif /* CONFIG_SMP */ + .globl cheetah_patch_cachetlbops +cheetah_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__cheetah_flush_tlb_mm), %o1 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1 + call tlb_patch_one + mov 19, %o2 + + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__cheetah_flush_tlb_pending), %o1 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 31, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__cheetah_flush_dcache_page), %o1 + or %o1, %lo(__cheetah_flush_dcache_page), %o1 + call tlb_patch_one + mov 11, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 44, %o2 +#endif /* CONFIG_SMP */ + + ret + restore .globl hypervisor_patch_cachetlbops hypervisor_patch_cachetlbops: @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 call tlb_patch_one - mov 10, %o2 + mov 19, %o2 sethi %hi(__flush_tlb_page), %o0 or %o0, %lo(__flush_tlb_page), %o0 sethi %hi(__hypervisor_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 call tlb_patch_one - mov 11, %o2 + mov 22, %o2 sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__hypervisor_flush_tlb_pending), %o1 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 call tlb_patch_one - mov 16, %o2 + mov 27, %o2 sethi %hi(__flush_tlb_kernel_range), %o0 or %o0, %lo(__flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 16, %o2 + mov 31, %o2 #ifdef DCACHE_ALIASING_POSSIBLE sethi %hi(__flush_dcache_page), %o0 @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 call tlb_patch_one - mov 21, %o2 + mov 24, %o2 sethi %hi(xcall_flush_tlb_page), %o0 or %o0, %lo(xcall_flush_tlb_page), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 call tlb_patch_one - mov 17, %o2 + mov 20, %o2 sethi %hi(xcall_flush_tlb_kernel_range), %o0 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 25, %o2 + mov 44, %o2 #endif /* CONFIG_SMP */ ret diff --git a/kernel/arch/tile/include/asm/elf.h b/kernel/arch/tile/include/asm/elf.h index c505d77e4..e9d54a067 100644 --- a/kernel/arch/tile/include/asm/elf.h +++ b/kernel/arch/tile/include/asm/elf.h @@ -129,6 +129,7 @@ extern int dump_task_regs(struct task_struct *, elf_gregset_t *); struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int executable_stack); +/* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */ #define ARCH_DLINFO \ do { \ NEW_AUX_ENT(AT_SYSINFO_EHDR, VDSO_BASE); \ diff --git a/kernel/arch/tile/include/uapi/asm/auxvec.h b/kernel/arch/tile/include/uapi/asm/auxvec.h index c93e92709..f497123ed 100644 --- a/kernel/arch/tile/include/uapi/asm/auxvec.h +++ b/kernel/arch/tile/include/uapi/asm/auxvec.h @@ -18,4 +18,6 @@ /* The vDSO location. */ #define AT_SYSINFO_EHDR 33 +#define AT_VECTOR_SIZE_ARCH 1 /* entries in ARCH_DLINFO */ + #endif /* _ASM_TILE_AUXVEC_H */ diff --git a/kernel/arch/tile/kernel/ptrace.c b/kernel/arch/tile/kernel/ptrace.c index bdc126faf..6239aa155 100644 --- a/kernel/arch/tile/kernel/ptrace.c +++ b/kernel/arch/tile/kernel/ptrace.c @@ -111,7 +111,7 @@ static int tile_gpr_set(struct task_struct *target, const void *kbuf, const void __user *ubuf) { int ret; - struct pt_regs regs; + struct pt_regs regs = *task_pt_regs(target); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, ®s, 0, sizeof(regs)); diff --git a/kernel/arch/tile/kernel/time.c b/kernel/arch/tile/kernel/time.c index 178989e6d..ea960d660 100644 --- a/kernel/arch/tile/kernel/time.c +++ b/kernel/arch/tile/kernel/time.c @@ -218,8 +218,8 @@ void do_timer_interrupt(struct pt_regs *regs, int fault_num) */ unsigned long long sched_clock(void) { - return clocksource_cyc2ns(get_cycles(), - sched_clock_mult, SCHED_CLOCK_SHIFT); + return mult_frac(get_cycles(), + sched_clock_mult, 1ULL << SCHED_CLOCK_SHIFT); } int setup_profiling_timer(unsigned int multiplier) diff --git a/kernel/arch/um/drivers/mconsole_kern.c b/kernel/arch/um/drivers/mconsole_kern.c index 29880c9b3..e22e57298 100644 --- a/kernel/arch/um/drivers/mconsole_kern.c +++ b/kernel/arch/um/drivers/mconsole_kern.c @@ -133,7 +133,7 @@ void mconsole_proc(struct mc_request *req) ptr += strlen("proc"); ptr = skip_spaces(ptr); - file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY); + file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY, 0); if (IS_ERR(file)) { mconsole_reply(req, "Failed to open file", 1, 0); printk(KERN_ERR "open /proc/%s: %ld\n", ptr, PTR_ERR(file)); diff --git a/kernel/arch/um/include/asm/common.lds.S b/kernel/arch/um/include/asm/common.lds.S index 1dd5bd8a8..133055311 100644 --- a/kernel/arch/um/include/asm/common.lds.S +++ b/kernel/arch/um/include/asm/common.lds.S @@ -81,7 +81,7 @@ .altinstr_replacement : { *(.altinstr_replacement) } /* .exit.text is discard at runtime, not link time, to deal with references from .altinstructions and .eh_frame */ - .exit.text : { *(.exit.text) } + .exit.text : { EXIT_TEXT } .exit.data : { *(.exit.data) } .preinit_array : { diff --git a/kernel/arch/x86/Kconfig b/kernel/arch/x86/Kconfig index d8a1650a0..6ee1dd0de 100644 --- a/kernel/arch/x86/Kconfig +++ b/kernel/arch/x86/Kconfig @@ -1130,22 +1130,23 @@ config MICROCODE bool "CPU microcode loading support" default y depends on CPU_SUP_AMD || CPU_SUP_INTEL - depends on BLK_DEV_INITRD select FW_LOADER ---help--- - If you say Y here, you will be able to update the microcode on - certain Intel and AMD processors. The Intel support is for the - IA32 family, e.g. Pentium Pro, Pentium II, Pentium III, Pentium 4, - Xeon etc. The AMD support is for families 0x10 and later. You will - obviously need the actual microcode binary data itself which is not - shipped with the Linux kernel. - - This option selects the general module only, you need to select - at least one vendor specific module as well. - - To compile this driver as a module, choose M here: the module - will be called microcode. + Intel and AMD processors. The Intel support is for the IA32 family, + e.g. Pentium Pro, Pentium II, Pentium III, Pentium 4, Xeon etc. The + AMD support is for families 0x10 and later. You will obviously need + the actual microcode binary data itself which is not shipped with + the Linux kernel. + + The preferred method to load microcode from a detached initrd is described + in Documentation/x86/early-microcode.txt. For that you need to enable + CONFIG_BLK_DEV_INITRD in order for the loader to be able to scan the + initrd for microcode blobs. + + In addition, you can build-in the microcode into the kernel. For that you + need to enable FIRMWARE_IN_KERNEL and add the vendor-supplied microcode + to the CONFIG_EXTRA_FIRMWARE config option. config MICROCODE_INTEL bool "Intel microcode loading support" diff --git a/kernel/arch/x86/boot/Makefile b/kernel/arch/x86/boot/Makefile index 2ee62dba0..c0cc2a6be 100644 --- a/kernel/arch/x86/boot/Makefile +++ b/kernel/arch/x86/boot/Makefile @@ -162,6 +162,9 @@ isoimage: $(obj)/bzImage for i in lib lib64 share end ; do \ if [ -f /usr/$$i/syslinux/isolinux.bin ] ; then \ cp /usr/$$i/syslinux/isolinux.bin $(obj)/isoimage ; \ + if [ -f /usr/$$i/syslinux/ldlinux.c32 ]; then \ + cp /usr/$$i/syslinux/ldlinux.c32 $(obj)/isoimage ; \ + fi ; \ break ; \ fi ; \ if [ $$i = end ] ; then exit 1 ; fi ; \ diff --git a/kernel/arch/x86/boot/compressed/Makefile b/kernel/arch/x86/boot/compressed/Makefile index 0a291cdfa..efa6073ff 100644 --- a/kernel/arch/x86/boot/compressed/Makefile +++ b/kernel/arch/x86/boot/compressed/Makefile @@ -22,7 +22,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 -KBUILD_CFLAGS += -fno-strict-aliasing -fPIC +KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC) KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING cflags-$(CONFIG_X86_32) := -march=i386 cflags-$(CONFIG_X86_64) := -mcmodel=small @@ -35,6 +35,18 @@ KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ GCOV_PROFILE := n LDFLAGS := -m elf_$(UTS_MACHINE) +ifeq ($(CONFIG_RELOCATABLE),y) +# If kernel is relocatable, build compressed kernel as PIE. +ifeq ($(CONFIG_X86_32),y) +LDFLAGS += $(call ld-option, -pie) $(call ld-option, --no-dynamic-linker) +else +# To build 64-bit compressed kernel as PIE, we disable relocation +# overflow check to avoid relocation overflow error with a new linker +# command-line option, -z noreloc-overflow. +LDFLAGS += $(shell $(LD) --help 2>&1 | grep -q "\-z noreloc-overflow" \ + && echo "-z noreloc-overflow -pie --no-dynamic-linker") +endif +endif LDFLAGS_vmlinux := -T hostprogs-y := mkpiggy diff --git a/kernel/arch/x86/boot/compressed/head_32.S b/kernel/arch/x86/boot/compressed/head_32.S index 8ef964ddc..0256064da 100644 --- a/kernel/arch/x86/boot/compressed/head_32.S +++ b/kernel/arch/x86/boot/compressed/head_32.S @@ -31,6 +31,34 @@ #include #include +/* + * The 32-bit x86 assembler in binutils 2.26 will generate R_386_GOT32X + * relocation to get the symbol address in PIC. When the compressed x86 + * kernel isn't built as PIC, the linker optimizes R_386_GOT32X + * relocations to their fixed symbol addresses. However, when the + * compressed x86 kernel is loaded at a different address, it leads + * to the following load failure: + * + * Failed to allocate space for phdrs + * + * during the decompression stage. + * + * If the compressed x86 kernel is relocatable at run-time, it should be + * compiled with -fPIE, instead of -fPIC, if possible and should be built as + * Position Independent Executable (PIE) so that linker won't optimize + * R_386_GOT32X relocation to its fixed symbol address. Older + * linkers generate R_386_32 relocations against locally defined symbols, + * _bss, _ebss, _got and _egot, in PIE. It isn't wrong, just less + * optimal than R_386_RELATIVE. But the x86 kernel fails to properly handle + * R_386_32 relocations when relocating the kernel. To generate + * R_386_RELATIVE relocations, we mark _bss, _ebss, _got and _egot as + * hidden: + */ + .hidden _bss + .hidden _ebss + .hidden _got + .hidden _egot + __HEAD ENTRY(startup_32) #ifdef CONFIG_EFI_STUB diff --git a/kernel/arch/x86/boot/compressed/head_64.S b/kernel/arch/x86/boot/compressed/head_64.S index b0c0d16ef..86558a199 100644 --- a/kernel/arch/x86/boot/compressed/head_64.S +++ b/kernel/arch/x86/boot/compressed/head_64.S @@ -33,6 +33,14 @@ #include #include +/* + * Locally defined symbols should be marked hidden: + */ + .hidden _bss + .hidden _ebss + .hidden _got + .hidden _egot + __HEAD .code32 ENTRY(startup_32) diff --git a/kernel/arch/x86/configs/tiny.config b/kernel/arch/x86/configs/tiny.config index 4e2ecfa23..4b429df40 100644 --- a/kernel/arch/x86/configs/tiny.config +++ b/kernel/arch/x86/configs/tiny.config @@ -1 +1,3 @@ CONFIG_NOHIGHMEM=y +# CONFIG_HIGHMEM4G is not set +# CONFIG_HIGHMEM64G is not set diff --git a/kernel/arch/x86/crypto/sha-mb/sha1_mb.c b/kernel/arch/x86/crypto/sha-mb/sha1_mb.c index a841e9765..8381c09d2 100644 --- a/kernel/arch/x86/crypto/sha-mb/sha1_mb.c +++ b/kernel/arch/x86/crypto/sha-mb/sha1_mb.c @@ -453,10 +453,10 @@ static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, req = cast_mcryptd_ctx_to_req(req_ctx); if (irqs_disabled()) - rctx->complete(&req->base, ret); + req_ctx->complete(&req->base, ret); else { local_bh_disable(); - rctx->complete(&req->base, ret); + req_ctx->complete(&req->base, ret); local_bh_enable(); } } diff --git a/kernel/arch/x86/entry/common.c b/kernel/arch/x86/entry/common.c index 3ec240f39..75a301b6a 100644 --- a/kernel/arch/x86/entry/common.c +++ b/kernel/arch/x86/entry/common.c @@ -275,6 +275,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags) /* Called with IRQs disabled. */ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) { + struct thread_info *ti = pt_regs_to_thread_info(regs); u32 cached_flags; if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled())) @@ -282,12 +283,22 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) lockdep_sys_exit(); - cached_flags = - READ_ONCE(pt_regs_to_thread_info(regs)->flags); + cached_flags = READ_ONCE(ti->flags); if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS)) exit_to_usermode_loop(regs, cached_flags); +#ifdef CONFIG_COMPAT + /* + * Compat syscalls set TS_COMPAT. Make sure we clear it before + * returning to user mode. We need to clear it *after* signal + * handling, because syscall restart has a fixup for compat + * syscalls. The fixup is exercised by the ptrace_syscall_32 + * selftest. + */ + ti->status &= ~TS_COMPAT; +#endif + user_enter(); } @@ -339,14 +350,6 @@ __visible inline void syscall_return_slowpath(struct pt_regs *regs) if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS)) syscall_slow_exit_work(regs, cached_flags); -#ifdef CONFIG_COMPAT - /* - * Compat syscalls set TS_COMPAT. Make sure we clear it before - * returning to user mode. - */ - ti->status &= ~TS_COMPAT; -#endif - local_irq_disable(); prepare_exit_to_usermode(regs); } diff --git a/kernel/arch/x86/entry/entry_32.S b/kernel/arch/x86/entry/entry_32.S index 2d722ee01..3bcef8bdb 100644 --- a/kernel/arch/x86/entry/entry_32.S +++ b/kernel/arch/x86/entry/entry_32.S @@ -782,8 +782,8 @@ ftrace_graph_call: jmp ftrace_stub #endif -.globl ftrace_stub -ftrace_stub: +/* This is weak to keep gas from relaxing the jumps */ +WEAK(ftrace_stub) ret END(ftrace_caller) diff --git a/kernel/arch/x86/entry/syscalls/syscall_32.tbl b/kernel/arch/x86/entry/syscalls/syscall_32.tbl index f17705e13..e62f4401e 100644 --- a/kernel/arch/x86/entry/syscalls/syscall_32.tbl +++ b/kernel/arch/x86/entry/syscalls/syscall_32.tbl @@ -294,7 +294,7 @@ # 285 sys_setaltroot 286 i386 add_key sys_add_key 287 i386 request_key sys_request_key -288 i386 keyctl sys_keyctl +288 i386 keyctl sys_keyctl compat_sys_keyctl 289 i386 ioprio_set sys_ioprio_set 290 i386 ioprio_get sys_ioprio_get 291 i386 inotify_init sys_inotify_init diff --git a/kernel/arch/x86/include/asm/apic.h b/kernel/arch/x86/include/asm/apic.h index 9686289d2..6a03a760f 100644 --- a/kernel/arch/x86/include/asm/apic.h +++ b/kernel/arch/x86/include/asm/apic.h @@ -640,8 +640,8 @@ static inline void entering_irq(void) static inline void entering_ack_irq(void) { - ack_APIC_irq(); entering_irq(); + ack_APIC_irq(); } static inline void ipi_entering_ack_irq(void) diff --git a/kernel/arch/x86/include/asm/hugetlb.h b/kernel/arch/x86/include/asm/hugetlb.h index f8a29d2c9..e6a8613fb 100644 --- a/kernel/arch/x86/include/asm/hugetlb.h +++ b/kernel/arch/x86/include/asm/hugetlb.h @@ -4,6 +4,7 @@ #include #include +#define hugepages_supported() cpu_has_pse static inline int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, diff --git a/kernel/arch/x86/include/asm/hw_irq.h b/kernel/arch/x86/include/asm/hw_irq.h index 1e3408e88..59caa55fb 100644 --- a/kernel/arch/x86/include/asm/hw_irq.h +++ b/kernel/arch/x86/include/asm/hw_irq.h @@ -136,6 +136,7 @@ struct irq_alloc_info { struct irq_cfg { unsigned int dest_apicid; u8 vector; + u8 old_vector; }; extern struct irq_cfg *irq_cfg(unsigned int irq); diff --git a/kernel/arch/x86/include/asm/kvm_host.h b/kernel/arch/x86/include/asm/kvm_host.h index fe68e836e..2cf52f11a 100644 --- a/kernel/arch/x86/include/asm/kvm_host.h +++ b/kernel/arch/x86/include/asm/kvm_host.h @@ -41,7 +41,7 @@ #define KVM_PIO_PAGE_OFFSET 1 #define KVM_COALESCED_MMIO_PAGE_OFFSET 2 -#define KVM_HALT_POLL_NS_DEFAULT 500000 +#define KVM_HALT_POLL_NS_DEFAULT 400000 #define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS diff --git a/kernel/arch/x86/include/asm/microcode.h b/kernel/arch/x86/include/asm/microcode.h index 34e62b1dc..712b24ed3 100644 --- a/kernel/arch/x86/include/asm/microcode.h +++ b/kernel/arch/x86/include/asm/microcode.h @@ -2,6 +2,7 @@ #define _ASM_X86_MICROCODE_H #include +#include #define native_rdmsr(msr, val1, val2) \ do { \ @@ -168,4 +169,29 @@ static inline void reload_early_microcode(void) { } static inline bool get_builtin_firmware(struct cpio_data *cd, const char *name) { return false; } #endif + +static inline unsigned long get_initrd_start(void) +{ +#ifdef CONFIG_BLK_DEV_INITRD + return initrd_start; +#else + return 0; +#endif +} + +static inline unsigned long get_initrd_start_addr(void) +{ +#ifdef CONFIG_BLK_DEV_INITRD +#ifdef CONFIG_X86_32 + unsigned long *initrd_start_p = (unsigned long *)__pa_nodebug(&initrd_start); + + return (unsigned long)__pa_nodebug(*initrd_start_p); +#else + return get_initrd_start(); +#endif +#else /* CONFIG_BLK_DEV_INITRD */ + return 0; +#endif +} + #endif /* _ASM_X86_MICROCODE_H */ diff --git a/kernel/arch/x86/include/asm/mtrr.h b/kernel/arch/x86/include/asm/mtrr.h index b94f6f64e..dbff1456d 100644 --- a/kernel/arch/x86/include/asm/mtrr.h +++ b/kernel/arch/x86/include/asm/mtrr.h @@ -24,6 +24,7 @@ #define _ASM_X86_MTRR_H #include +#include /* @@ -83,9 +84,12 @@ static inline int mtrr_trim_uncached_memory(unsigned long end_pfn) static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi) { } +static inline void mtrr_bp_init(void) +{ + pat_disable("MTRRs disabled, skipping PAT initialization too."); +} #define mtrr_ap_init() do {} while (0) -#define mtrr_bp_init() do {} while (0) #define set_mtrr_aps_delayed_init() do {} while (0) #define mtrr_aps_init() do {} while (0) #define mtrr_bp_restore() do {} while (0) diff --git a/kernel/arch/x86/include/asm/pat.h b/kernel/arch/x86/include/asm/pat.h index ca6c228d5..0b1ff4c1c 100644 --- a/kernel/arch/x86/include/asm/pat.h +++ b/kernel/arch/x86/include/asm/pat.h @@ -5,8 +5,8 @@ #include bool pat_enabled(void); +void pat_disable(const char *reason); extern void pat_init(void); -void pat_init_cache_modes(u64); extern int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_pcm, enum page_cache_mode *ret_pcm); diff --git a/kernel/arch/x86/include/asm/pci_x86.h b/kernel/arch/x86/include/asm/pci_x86.h index fa1195dae..164e3f8d3 100644 --- a/kernel/arch/x86/include/asm/pci_x86.h +++ b/kernel/arch/x86/include/asm/pci_x86.h @@ -93,6 +93,8 @@ extern raw_spinlock_t pci_config_lock; extern int (*pcibios_enable_irq)(struct pci_dev *dev); extern void (*pcibios_disable_irq)(struct pci_dev *dev); +extern bool mp_should_keep_irq(struct device *dev); + struct pci_raw_ops { int (*read)(unsigned int domain, unsigned int bus, unsigned int devfn, int reg, int len, u32 *val); diff --git a/kernel/arch/x86/include/asm/perf_event.h b/kernel/arch/x86/include/asm/perf_event.h index 7bcb861a0..5a2ed3ed2 100644 --- a/kernel/arch/x86/include/asm/perf_event.h +++ b/kernel/arch/x86/include/asm/perf_event.h @@ -165,6 +165,7 @@ struct x86_pmu_capability { #define GLOBAL_STATUS_ASIF BIT_ULL(60) #define GLOBAL_STATUS_COUNTERS_FROZEN BIT_ULL(59) #define GLOBAL_STATUS_LBRS_FROZEN BIT_ULL(58) +#define GLOBAL_STATUS_TRACE_TOPAPMI BIT_ULL(55) /* * IBS cpuid feature detection diff --git a/kernel/arch/x86/include/asm/preempt.h b/kernel/arch/x86/include/asm/preempt.h index 5dbd2d0f9..6f432adc5 100644 --- a/kernel/arch/x86/include/asm/preempt.h +++ b/kernel/arch/x86/include/asm/preempt.h @@ -89,6 +89,8 @@ static __always_inline bool __preempt_count_dec_and_test(void) if (____preempt_count_dec_and_test()) return true; #ifdef CONFIG_PREEMPT_LAZY + if (current_thread_info()->preempt_lazy_count) + return false; return test_thread_flag(TIF_NEED_RESCHED_LAZY); #else return false; @@ -101,8 +103,19 @@ static __always_inline bool __preempt_count_dec_and_test(void) static __always_inline bool should_resched(int preempt_offset) { #ifdef CONFIG_PREEMPT_LAZY - return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset || - test_thread_flag(TIF_NEED_RESCHED_LAZY)); + u32 tmp; + + tmp = raw_cpu_read_4(__preempt_count); + if (tmp == preempt_offset) + return true; + + /* preempt count == 0 ? */ + tmp &= ~PREEMPT_NEED_RESCHED; + if (tmp) + return false; + if (current_thread_info()->preempt_lazy_count) + return false; + return test_thread_flag(TIF_NEED_RESCHED_LAZY); #else return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset); #endif diff --git a/kernel/arch/x86/include/asm/pvclock.h b/kernel/arch/x86/include/asm/pvclock.h index 7a6bed5c0..baad72e4c 100644 --- a/kernel/arch/x86/include/asm/pvclock.h +++ b/kernel/arch/x86/include/asm/pvclock.h @@ -76,6 +76,8 @@ unsigned __pvclock_read_cycles(const struct pvclock_vcpu_time_info *src, u8 ret_flags; version = src->version; + /* Make the latest version visible */ + smp_rmb(); offset = pvclock_get_nsec_offset(src); ret = src->system_time + offset; diff --git a/kernel/arch/x86/include/asm/tlbflush.h b/kernel/arch/x86/include/asm/tlbflush.h index 6df202940..6433e28dc 100644 --- a/kernel/arch/x86/include/asm/tlbflush.h +++ b/kernel/arch/x86/include/asm/tlbflush.h @@ -32,7 +32,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate); /* Initialize cr4 shadow for this CPU. */ static inline void cr4_init_shadow(void) { - this_cpu_write(cpu_tlbstate.cr4, __read_cr4()); + this_cpu_write(cpu_tlbstate.cr4, __read_cr4_safe()); } /* Set in this cpu's CR4. */ @@ -86,7 +86,14 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask) static inline void __native_flush_tlb(void) { + /* + * If current->mm == NULL then we borrow a mm which may change during a + * task switch and therefore we must not be preempted while we write CR3 + * back: + */ + preempt_disable(); native_write_cr3(native_read_cr3()); + preempt_enable(); } static inline void __native_flush_tlb_global_irq_disabled(void) diff --git a/kernel/arch/x86/include/asm/uaccess.h b/kernel/arch/x86/include/asm/uaccess.h index 09b1b0ab9..b8c75f3aa 100644 --- a/kernel/arch/x86/include/asm/uaccess.h +++ b/kernel/arch/x86/include/asm/uaccess.h @@ -332,7 +332,7 @@ do { \ #define __get_user_asm_u64(x, ptr, retval, errret) \ __get_user_asm(x, ptr, retval, "q", "", "=r", errret) #define __get_user_asm_ex_u64(x, ptr) \ - __get_user_asm_ex(x, ptr, "q", "", "=r") + __get_user_asm_ex(x, ptr, "q", "", "=&r") #endif #define __get_user_size(x, ptr, size, retval, errret) \ @@ -375,13 +375,13 @@ do { \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: \ - __get_user_asm_ex(x, ptr, "b", "b", "=q"); \ + __get_user_asm_ex(x, ptr, "b", "b", "=&q"); \ break; \ case 2: \ - __get_user_asm_ex(x, ptr, "w", "w", "=r"); \ + __get_user_asm_ex(x, ptr, "w", "w", "=&r"); \ break; \ case 4: \ - __get_user_asm_ex(x, ptr, "l", "k", "=r"); \ + __get_user_asm_ex(x, ptr, "l", "k", "=&r"); \ break; \ case 8: \ __get_user_asm_ex_u64(x, ptr); \ @@ -395,7 +395,7 @@ do { \ asm volatile("1: mov"itype" %1,%"rtype"0\n" \ "2:\n" \ _ASM_EXTABLE_EX(1b, 2b) \ - : ltype(x) : "m" (__m(addr))) + : ltype(x) : "m" (__m(addr)), "0" (0)) #define __put_user_nocheck(x, ptr, size) \ ({ \ diff --git a/kernel/arch/x86/include/asm/xen/hypervisor.h b/kernel/arch/x86/include/asm/xen/hypervisor.h index 8b2d4bea9..39171b364 100644 --- a/kernel/arch/x86/include/asm/xen/hypervisor.h +++ b/kernel/arch/x86/include/asm/xen/hypervisor.h @@ -62,4 +62,6 @@ void xen_arch_register_cpu(int num); void xen_arch_unregister_cpu(int num); #endif +extern void xen_set_iopl_mask(unsigned mask); + #endif /* _ASM_X86_XEN_HYPERVISOR_H */ diff --git a/kernel/arch/x86/kernel/acpi/boot.c b/kernel/arch/x86/kernel/acpi/boot.c index e75907601..a29fc4f84 100644 --- a/kernel/arch/x86/kernel/acpi/boot.c +++ b/kernel/arch/x86/kernel/acpi/boot.c @@ -87,7 +87,9 @@ static u64 acpi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; * ->ioapic_mutex * ->ioapic_lock */ +#ifdef CONFIG_X86_IO_APIC static DEFINE_MUTEX(acpi_ioapic_lock); +#endif /* -------------------------------------------------------------------------- Boot-time Configuration diff --git a/kernel/arch/x86/kernel/amd_nb.c b/kernel/arch/x86/kernel/amd_nb.c index 29fa475ec..c986d0b3b 100644 --- a/kernel/arch/x86/kernel/amd_nb.c +++ b/kernel/arch/x86/kernel/amd_nb.c @@ -71,8 +71,8 @@ int amd_cache_northbridges(void) while ((misc = next_northbridge(misc, amd_nb_misc_ids)) != NULL) i++; - if (i == 0) - return 0; + if (!i) + return -ENODEV; nb = kzalloc(i * sizeof(struct amd_northbridge), GFP_KERNEL); if (!nb) diff --git a/kernel/arch/x86/kernel/apic/apic.c b/kernel/arch/x86/kernel/apic/apic.c index 58e7cdb25..042e744ef 100644 --- a/kernel/arch/x86/kernel/apic/apic.c +++ b/kernel/arch/x86/kernel/apic/apic.c @@ -1611,6 +1611,9 @@ void __init enable_IR_x2apic(void) unsigned long flags; int ret, ir_stat; + if (skip_ioapic_setup) + return; + ir_stat = irq_remapping_prepare(); if (ir_stat < 0 && !x2apic_supported()) return; diff --git a/kernel/arch/x86/kernel/apic/io_apic.c b/kernel/arch/x86/kernel/apic/io_apic.c index 678c711e2..7bfb53808 100644 --- a/kernel/arch/x86/kernel/apic/io_apic.c +++ b/kernel/arch/x86/kernel/apic/io_apic.c @@ -2116,6 +2116,7 @@ static inline void __init check_timer(void) if (idx != -1 && irq_trigger(idx)) unmask_ioapic_irq(irq_get_chip_data(0)); } + irq_domain_deactivate_irq(irq_data); irq_domain_activate_irq(irq_data); if (timer_irq_works()) { if (disable_timer_pin_1 > 0) @@ -2137,6 +2138,7 @@ static inline void __init check_timer(void) * legacy devices should be connected to IO APIC #0 */ replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2); + irq_domain_deactivate_irq(irq_data); irq_domain_activate_irq(irq_data); legacy_pic->unmask(0); if (timer_irq_works()) { diff --git a/kernel/arch/x86/kernel/apic/vector.c b/kernel/arch/x86/kernel/apic/vector.c index a35f6b547..0988e204f 100644 --- a/kernel/arch/x86/kernel/apic/vector.c +++ b/kernel/arch/x86/kernel/apic/vector.c @@ -211,6 +211,7 @@ update: */ cpumask_and(d->old_domain, d->old_domain, cpu_online_mask); d->move_in_progress = !cpumask_empty(d->old_domain); + d->cfg.old_vector = d->move_in_progress ? d->cfg.vector : 0; d->cfg.vector = vector; cpumask_copy(d->domain, vector_cpumask); success: @@ -253,7 +254,8 @@ static void clear_irq_vector(int irq, struct apic_chip_data *data) struct irq_desc *desc; int cpu, vector; - BUG_ON(!data->cfg.vector); + if (!data->cfg.vector) + return; vector = data->cfg.vector; for_each_cpu_and(cpu, data->domain, cpu_online_mask) @@ -653,46 +655,114 @@ void irq_complete_move(struct irq_cfg *cfg) } /* - * Called with @desc->lock held and interrupts disabled. + * Called from fixup_irqs() with @desc->lock held and interrupts disabled. */ void irq_force_complete_move(struct irq_desc *desc) { - struct irq_data *irqdata = irq_desc_get_irq_data(desc); - struct apic_chip_data *data = apic_chip_data(irqdata); - struct irq_cfg *cfg = data ? &data->cfg : NULL; + struct irq_data *irqdata; + struct apic_chip_data *data; + struct irq_cfg *cfg; + unsigned int cpu; - if (!cfg) + /* + * The function is called for all descriptors regardless of which + * irqdomain they belong to. For example if an IRQ is provided by + * an irq_chip as part of a GPIO driver, the chip data for that + * descriptor is specific to the irq_chip in question. + * + * Check first that the chip_data is what we expect + * (apic_chip_data) before touching it any further. + */ + irqdata = irq_domain_get_irq_data(x86_vector_domain, + irq_desc_get_irq(desc)); + if (!irqdata) return; - __irq_complete_move(cfg, cfg->vector); + data = apic_chip_data(irqdata); + cfg = data ? &data->cfg : NULL; + + if (!cfg) + return; /* * This is tricky. If the cleanup of @data->old_domain has not been * done yet, then the following setaffinity call will fail with * -EBUSY. This can leave the interrupt in a stale state. * - * The cleanup cannot make progress because we hold @desc->lock. So in - * case @data->old_domain is not yet cleaned up, we need to drop the - * lock and acquire it again. @desc cannot go away, because the - * hotplug code holds the sparse irq lock. + * All CPUs are stuck in stop machine with interrupts disabled so + * calling __irq_complete_move() would be completely pointless. */ raw_spin_lock(&vector_lock); - /* Clean out all offline cpus (including ourself) first. */ + /* + * Clean out all offline cpus (including the outgoing one) from the + * old_domain mask. + */ cpumask_and(data->old_domain, data->old_domain, cpu_online_mask); - while (!cpumask_empty(data->old_domain)) { + + /* + * If move_in_progress is cleared and the old_domain mask is empty, + * then there is nothing to cleanup. fixup_irqs() will take care of + * the stale vectors on the outgoing cpu. + */ + if (!data->move_in_progress && cpumask_empty(data->old_domain)) { raw_spin_unlock(&vector_lock); - raw_spin_unlock(&desc->lock); - cpu_relax(); - raw_spin_lock(&desc->lock); + return; + } + + /* + * 1) The interrupt is in move_in_progress state. That means that we + * have not seen an interrupt since the io_apic was reprogrammed to + * the new vector. + * + * 2) The interrupt has fired on the new vector, but the cleanup IPIs + * have not been processed yet. + */ + if (data->move_in_progress) { /* - * Reevaluate apic_chip_data. It might have been cleared after - * we dropped @desc->lock. + * In theory there is a race: + * + * set_ioapic(new_vector) <-- Interrupt is raised before update + * is effective, i.e. it's raised on + * the old vector. + * + * So if the target cpu cannot handle that interrupt before + * the old vector is cleaned up, we get a spurious interrupt + * and in the worst case the ioapic irq line becomes stale. + * + * But in case of cpu hotplug this should be a non issue + * because if the affinity update happens right before all + * cpus rendevouz in stop machine, there is no way that the + * interrupt can be blocked on the target cpu because all cpus + * loops first with interrupts enabled in stop machine, so the + * old vector is not yet cleaned up when the interrupt fires. + * + * So the only way to run into this issue is if the delivery + * of the interrupt on the apic/system bus would be delayed + * beyond the point where the target cpu disables interrupts + * in stop machine. I doubt that it can happen, but at least + * there is a theroretical chance. Virtualization might be + * able to expose this, but AFAICT the IOAPIC emulation is not + * as stupid as the real hardware. + * + * Anyway, there is nothing we can do about that at this point + * w/o refactoring the whole fixup_irq() business completely. + * We print at least the irq number and the old vector number, + * so we have the necessary information when a problem in that + * area arises. */ - data = apic_chip_data(irqdata); - if (!data) - return; - raw_spin_lock(&vector_lock); + pr_warn("IRQ fixup: irq %d move in progress, old vector %d\n", + irqdata->irq, cfg->old_vector); } + /* + * If old_domain is not empty, then other cpus still have the irq + * descriptor set in their vector array. Clean it up. + */ + for_each_cpu(cpu, data->old_domain) + per_cpu(vector_irq, cpu)[cfg->old_vector] = VECTOR_UNUSED; + + /* Cleanup the left overs of the (half finished) move */ + cpumask_clear(data->old_domain); + data->move_in_progress = 0; raw_spin_unlock(&vector_lock); } #endif diff --git a/kernel/arch/x86/kernel/cpu/amd.c b/kernel/arch/x86/kernel/cpu/amd.c index a8816b325..e2defc759 100644 --- a/kernel/arch/x86/kernel/cpu/amd.c +++ b/kernel/arch/x86/kernel/cpu/amd.c @@ -352,7 +352,6 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) #ifdef CONFIG_SMP unsigned bits; int cpu = smp_processor_id(); - unsigned int socket_id, core_complex_id; bits = c->x86_coreid_bits; /* Low order bits define the core id (index of core in socket) */ @@ -370,10 +369,7 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) if (c->x86 != 0x17 || !cpuid_edx(0x80000006)) return; - socket_id = (c->apicid >> bits) - 1; - core_complex_id = (c->apicid & ((1 << bits) - 1)) >> 3; - - per_cpu(cpu_llc_id, cpu) = (socket_id << 3) | core_complex_id; + per_cpu(cpu_llc_id, cpu) = c->apicid >> 3; #endif } @@ -656,6 +652,17 @@ static void init_amd_gh(struct cpuinfo_x86 *c) set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH); } +#define MSR_AMD64_DE_CFG 0xC0011029 + +static void init_amd_ln(struct cpuinfo_x86 *c) +{ + /* + * Apply erratum 665 fix unconditionally so machines without a BIOS + * fix work. + */ + msr_set_bit(MSR_AMD64_DE_CFG, 31); +} + static void init_amd_bd(struct cpuinfo_x86 *c) { u64 value; @@ -713,6 +720,7 @@ static void init_amd(struct cpuinfo_x86 *c) case 6: init_amd_k7(c); break; case 0xf: init_amd_k8(c); break; case 0x10: init_amd_gh(c); break; + case 0x12: init_amd_ln(c); break; case 0x15: init_amd_bd(c); break; } diff --git a/kernel/arch/x86/kernel/cpu/common.c b/kernel/arch/x86/kernel/cpu/common.c index c2b7522cb..637ca414d 100644 --- a/kernel/arch/x86/kernel/cpu/common.c +++ b/kernel/arch/x86/kernel/cpu/common.c @@ -737,21 +737,20 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c) identify_cpu_without_cpuid(c); /* cyrix could have cpuid enabled via c_identify()*/ - if (!have_cpuid_p()) - return; + if (have_cpuid_p()) { + cpu_detect(c); + get_cpu_vendor(c); + get_cpu_cap(c); - cpu_detect(c); - get_cpu_vendor(c); - get_cpu_cap(c); - - if (this_cpu->c_early_init) - this_cpu->c_early_init(c); + if (this_cpu->c_early_init) + this_cpu->c_early_init(c); - c->cpu_index = 0; - filter_cpuid_features(c, false); + c->cpu_index = 0; + filter_cpuid_features(c, false); - if (this_cpu->c_bsp_init) - this_cpu->c_bsp_init(c); + if (this_cpu->c_bsp_init) + this_cpu->c_bsp_init(c); + } setup_force_cpu_cap(X86_FEATURE_ALWAYS); fpu__init_system(c); @@ -1130,7 +1129,7 @@ static __init int setup_disablecpuid(char *arg) { int bit; - if (get_option(&arg, &bit) && bit < NCAPINTS*32) + if (get_option(&arg, &bit) && bit >= 0 && bit < NCAPINTS * 32) setup_clear_cpu_cap(bit); else return 0; diff --git a/kernel/arch/x86/kernel/cpu/mcheck/mce-genpool.c b/kernel/arch/x86/kernel/cpu/mcheck/mce-genpool.c index 0a850100c..2658e2af7 100644 --- a/kernel/arch/x86/kernel/cpu/mcheck/mce-genpool.c +++ b/kernel/arch/x86/kernel/cpu/mcheck/mce-genpool.c @@ -29,7 +29,7 @@ static char gen_pool_buf[MCE_POOLSZ]; void mce_gen_pool_process(void) { struct llist_node *head; - struct mce_evt_llist *node; + struct mce_evt_llist *node, *tmp; struct mce *mce; head = llist_del_all(&mce_event_llist); @@ -37,7 +37,7 @@ void mce_gen_pool_process(void) return; head = llist_reverse_order(head); - llist_for_each_entry(node, head, llnode) { + llist_for_each_entry_safe(node, tmp, head, llnode) { mce = &node->mce; atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, mce); gen_pool_free(mce_evt_pool, (unsigned long)node, sizeof(*node)); diff --git a/kernel/arch/x86/kernel/cpu/mcheck/mce.c b/kernel/arch/x86/kernel/cpu/mcheck/mce.c index a080b4939..430a4ec07 100644 --- a/kernel/arch/x86/kernel/cpu/mcheck/mce.c +++ b/kernel/arch/x86/kernel/cpu/mcheck/mce.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/kernel/arch/x86/kernel/cpu/mcheck/therm_throt.c b/kernel/arch/x86/kernel/cpu/mcheck/therm_throt.c index 2c5aaf8c2..05538582a 100644 --- a/kernel/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/kernel/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -385,6 +385,9 @@ static void intel_thermal_interrupt(void) { __u64 msr_val; + if (static_cpu_has(X86_FEATURE_HWP)) + wrmsrl_safe(MSR_HWP_STATUS, 0); + rdmsrl(MSR_IA32_THERM_STATUS, msr_val); /* Check for violation of core thermal thresholds*/ diff --git a/kernel/arch/x86/kernel/cpu/microcode/intel.c b/kernel/arch/x86/kernel/cpu/microcode/intel.c index ce47402eb..ac8975a65 100644 --- a/kernel/arch/x86/kernel/cpu/microcode/intel.c +++ b/kernel/arch/x86/kernel/cpu/microcode/intel.c @@ -555,10 +555,14 @@ scan_microcode(struct mc_saved_data *mc_saved_data, unsigned long *initrd, cd.data = NULL; cd.size = 0; - cd = find_cpio_data(p, (void *)start, size, &offset); - if (!cd.data) { + /* try built-in microcode if no initrd */ + if (!size) { if (!load_builtin_intel_microcode(&cd)) return UCODE_ERROR; + } else { + cd = find_cpio_data(p, (void *)start, size, &offset); + if (!cd.data) + return UCODE_ERROR; } return get_matching_model_microcode(0, start, cd.data, cd.size, @@ -694,7 +698,7 @@ int __init save_microcode_in_initrd_intel(void) if (count == 0) return ret; - copy_initrd_ptrs(mc_saved, mc_saved_in_initrd, initrd_start, count); + copy_initrd_ptrs(mc_saved, mc_saved_in_initrd, get_initrd_start(), count); ret = save_microcode(&mc_saved_data, mc_saved, count); if (ret) pr_err("Cannot save microcode patches from initrd.\n"); @@ -732,16 +736,20 @@ void __init load_ucode_intel_bsp(void) struct boot_params *p; p = (struct boot_params *)__pa_nodebug(&boot_params); - start = p->hdr.ramdisk_image; size = p->hdr.ramdisk_size; - _load_ucode_intel_bsp( - (struct mc_saved_data *)__pa_nodebug(&mc_saved_data), - (unsigned long *)__pa_nodebug(&mc_saved_in_initrd), - start, size); + /* + * Set start only if we have an initrd image. We cannot use initrd_start + * because it is not set that early yet. + */ + start = (size ? p->hdr.ramdisk_image : 0); + + _load_ucode_intel_bsp((struct mc_saved_data *)__pa_nodebug(&mc_saved_data), + (unsigned long *)__pa_nodebug(&mc_saved_in_initrd), + start, size); #else - start = boot_params.hdr.ramdisk_image + PAGE_OFFSET; size = boot_params.hdr.ramdisk_size; + start = (size ? boot_params.hdr.ramdisk_image + PAGE_OFFSET : 0); _load_ucode_intel_bsp(&mc_saved_data, mc_saved_in_initrd, start, size); #endif @@ -752,20 +760,14 @@ void load_ucode_intel_ap(void) struct mc_saved_data *mc_saved_data_p; struct ucode_cpu_info uci; unsigned long *mc_saved_in_initrd_p; - unsigned long initrd_start_addr; enum ucode_state ret; #ifdef CONFIG_X86_32 - unsigned long *initrd_start_p; - mc_saved_in_initrd_p = - (unsigned long *)__pa_nodebug(mc_saved_in_initrd); + mc_saved_in_initrd_p = (unsigned long *)__pa_nodebug(mc_saved_in_initrd); mc_saved_data_p = (struct mc_saved_data *)__pa_nodebug(&mc_saved_data); - initrd_start_p = (unsigned long *)__pa_nodebug(&initrd_start); - initrd_start_addr = (unsigned long)__pa_nodebug(*initrd_start_p); #else - mc_saved_data_p = &mc_saved_data; mc_saved_in_initrd_p = mc_saved_in_initrd; - initrd_start_addr = initrd_start; + mc_saved_data_p = &mc_saved_data; #endif /* @@ -777,7 +779,7 @@ void load_ucode_intel_ap(void) collect_cpu_info_early(&uci); ret = load_microcode(mc_saved_data_p, mc_saved_in_initrd_p, - initrd_start_addr, &uci); + get_initrd_start_addr(), &uci); if (ret != UCODE_OK) return; diff --git a/kernel/arch/x86/kernel/cpu/mshyperv.c b/kernel/arch/x86/kernel/cpu/mshyperv.c index 20e242ea1..cfc4a966e 100644 --- a/kernel/arch/x86/kernel/cpu/mshyperv.c +++ b/kernel/arch/x86/kernel/cpu/mshyperv.c @@ -152,6 +152,11 @@ static struct clocksource hyperv_cs = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +static unsigned char hv_get_nmi_reason(void) +{ + return 0; +} + static void __init ms_hyperv_init_platform(void) { /* @@ -191,6 +196,13 @@ static void __init ms_hyperv_init_platform(void) machine_ops.crash_shutdown = hv_machine_crash_shutdown; #endif mark_tsc_unstable("running on Hyper-V"); + + /* + * Generation 2 instances don't support reading the NMI status from + * 0x61 port. + */ + if (efi_enabled(EFI_BOOT)) + x86_platform.get_nmi_reason = hv_get_nmi_reason; } const __refconst struct hypervisor_x86 x86_hyper_ms_hyperv = { diff --git a/kernel/arch/x86/kernel/cpu/mtrr/generic.c b/kernel/arch/x86/kernel/cpu/mtrr/generic.c index 3b533cf37..b5624fafa 100644 --- a/kernel/arch/x86/kernel/cpu/mtrr/generic.c +++ b/kernel/arch/x86/kernel/cpu/mtrr/generic.c @@ -444,11 +444,24 @@ static void __init print_mtrr_state(void) pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20); } +/* PAT setup for BP. We need to go through sync steps here */ +void __init mtrr_bp_pat_init(void) +{ + unsigned long flags; + + local_irq_save(flags); + prepare_set(); + + pat_init(); + + post_set(); + local_irq_restore(flags); +} + /* Grab all of the MTRR state for this CPU into *state */ bool __init get_mtrr_state(void) { struct mtrr_var_range *vrs; - unsigned long flags; unsigned lo, dummy; unsigned int i; @@ -481,15 +494,6 @@ bool __init get_mtrr_state(void) mtrr_state_set = 1; - /* PAT setup for BP. We need to go through sync steps here */ - local_irq_save(flags); - prepare_set(); - - pat_init(); - - post_set(); - local_irq_restore(flags); - return !!(mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED); } diff --git a/kernel/arch/x86/kernel/cpu/mtrr/main.c b/kernel/arch/x86/kernel/cpu/mtrr/main.c index f891b4750..fa77ac829 100644 --- a/kernel/arch/x86/kernel/cpu/mtrr/main.c +++ b/kernel/arch/x86/kernel/cpu/mtrr/main.c @@ -752,6 +752,9 @@ void __init mtrr_bp_init(void) /* BIOS may override */ __mtrr_enabled = get_mtrr_state(); + if (mtrr_enabled()) + mtrr_bp_pat_init(); + if (mtrr_cleanup(phys_addr)) { changed_by_mtrr_cleanup = 1; mtrr_if->set_all(); @@ -759,8 +762,16 @@ void __init mtrr_bp_init(void) } } - if (!mtrr_enabled()) + if (!mtrr_enabled()) { pr_info("MTRR: Disabled\n"); + + /* + * PAT initialization relies on MTRR's rendezvous handler. + * Skip PAT init until the handler can initialize both + * features independently. + */ + pat_disable("MTRRs disabled, skipping PAT initialization too."); + } } void mtrr_ap_init(void) diff --git a/kernel/arch/x86/kernel/cpu/mtrr/mtrr.h b/kernel/arch/x86/kernel/cpu/mtrr/mtrr.h index 951884dcc..6c7ced07d 100644 --- a/kernel/arch/x86/kernel/cpu/mtrr/mtrr.h +++ b/kernel/arch/x86/kernel/cpu/mtrr/mtrr.h @@ -52,6 +52,7 @@ void set_mtrr_prepare_save(struct set_mtrr_context *ctxt); void fill_mtrr_var_range(unsigned int index, u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi); bool get_mtrr_state(void); +void mtrr_bp_pat_init(void); extern void set_mtrr_ops(const struct mtrr_ops *ops); diff --git a/kernel/arch/x86/kernel/cpu/perf_event.c b/kernel/arch/x86/kernel/cpu/perf_event.c index 2bf79d7c9..1a8256dd6 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event.c +++ b/kernel/arch/x86/kernel/cpu/perf_event.c @@ -67,7 +67,7 @@ u64 x86_perf_event_update(struct perf_event *event) int shift = 64 - x86_pmu.cntval_bits; u64 prev_raw_count, new_raw_count; int idx = hwc->idx; - s64 delta; + u64 delta; if (idx == INTEL_PMC_IDX_FIXED_BTS) return 0; @@ -593,6 +593,19 @@ void x86_pmu_disable_all(void) } } +/* + * There may be PMI landing after enabled=0. The PMI hitting could be before or + * after disable_all. + * + * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. + * It will not be re-enabled in the NMI handler again, because enabled=0. After + * handling the NMI, disable_all will be called, which will not change the + * state either. If PMI hits after disable_all, the PMU is already disabled + * before entering NMI handler. The NMI handler will not change the state + * either. + * + * So either situation is harmless. + */ static void x86_pmu_disable(struct pmu *pmu) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); diff --git a/kernel/arch/x86/kernel/cpu/perf_event.h b/kernel/arch/x86/kernel/cpu/perf_event.h index d0e35ebb2..ee70445fb 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event.h +++ b/kernel/arch/x86/kernel/cpu/perf_event.h @@ -591,6 +591,7 @@ struct x86_pmu { pebs_active :1, pebs_broken :1; int pebs_record_size; + int pebs_buffer_size; void (*drain_pebs)(struct pt_regs *regs); struct event_constraint *pebs_constraints; void (*pebs_aliases)(struct perf_event *event); @@ -907,6 +908,8 @@ void intel_pmu_lbr_init_hsw(void); void intel_pmu_lbr_init_skl(void); +void intel_pmu_pebs_data_source_nhm(void); + int intel_pmu_setup_lbr_filter(struct perf_event *event); void intel_pt_interrupt(void); diff --git a/kernel/arch/x86/kernel/cpu/perf_event_intel.c b/kernel/arch/x86/kernel/cpu/perf_event_intel.c index e2a430021..5cc2242d7 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_intel.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_intel.c @@ -1458,7 +1458,15 @@ static __initconst const u64 slm_hw_cache_event_ids }; /* - * Use from PMIs where the LBRs are already disabled. + * Used from PMIs where the LBRs are already disabled. + * + * This function could be called consecutively. It is required to remain in + * disabled state if called consecutively. + * + * During consecutive calls, the same disable value will be written to related + * registers, so the PMU state remains unchanged. hw.state in + * intel_bts_disable_local will remain PERF_HES_STOPPED too in consecutive + * calls. */ static void __intel_pmu_disable_all(void) { @@ -1840,6 +1848,16 @@ again: if (__test_and_clear_bit(62, (unsigned long *)&status)) { handled++; x86_pmu.drain_pebs(regs); + /* + * There are cases where, even though, the PEBS ovfl bit is set + * in GLOBAL_OVF_STATUS, the PEBS events may also have their + * overflow bits set for their counters. We must clear them + * here because they have been processed as exact samples in + * the drain_pebs() routine. They must not be processed again + * in the for_each_bit_set() loop for regular samples below. + */ + status &= ~cpuc->pebs_enabled; + status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI; } /* @@ -1885,7 +1903,10 @@ again: goto again; done: - __intel_pmu_enable_all(0, true); + /* Only restore PMU state when it's active. See x86_pmu_disable(). */ + if (cpuc->enabled) + __intel_pmu_enable_all(0, true); + /* * Only unmask the NMI after the overflow counters * have been reset. This avoids spurious NMIs on @@ -3315,6 +3336,7 @@ __init int intel_pmu_init(void) intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); + intel_pmu_pebs_data_source_nhm(); x86_add_quirk(intel_nehalem_quirk); pr_cont("Nehalem events, "); @@ -3377,6 +3399,7 @@ __init int intel_pmu_init(void) intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); + intel_pmu_pebs_data_source_nhm(); pr_cont("Westmere events, "); break; @@ -3578,7 +3601,7 @@ __init int intel_pmu_init(void) c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1; } c->idxmsk64 &= - ~(~0UL << (INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed)); + ~(~0ULL << (INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed)); c->weight = hweight64(c->idxmsk64); } } @@ -3613,7 +3636,7 @@ __init int intel_pmu_init(void) /* Support full width counters using alternative MSR range */ if (x86_pmu.intel_cap.full_width_write) { - x86_pmu.max_period = x86_pmu.cntval_mask; + x86_pmu.max_period = x86_pmu.cntval_mask >> 1; x86_pmu.perfctr = MSR_IA32_PMC0; pr_cont("full-width counters, "); } diff --git a/kernel/arch/x86/kernel/cpu/perf_event_intel_cqm.c b/kernel/arch/x86/kernel/cpu/perf_event_intel_cqm.c index a316ca96f..fc704ed58 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_intel_cqm.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_intel_cqm.c @@ -211,6 +211,20 @@ static void __put_rmid(u32 rmid) list_add_tail(&entry->list, &cqm_rmid_limbo_lru); } +static void cqm_cleanup(void) +{ + int i; + + if (!cqm_rmid_ptrs) + return; + + for (i = 0; i < cqm_max_rmid; i++) + kfree(cqm_rmid_ptrs[i]); + + kfree(cqm_rmid_ptrs); + cqm_rmid_ptrs = NULL; +} + static int intel_cqm_setup_rmid_cache(void) { struct cqm_rmid_entry *entry; @@ -218,7 +232,7 @@ static int intel_cqm_setup_rmid_cache(void) int r = 0; nr_rmids = cqm_max_rmid + 1; - cqm_rmid_ptrs = kmalloc(sizeof(struct cqm_rmid_entry *) * + cqm_rmid_ptrs = kzalloc(sizeof(struct cqm_rmid_entry *) * nr_rmids, GFP_KERNEL); if (!cqm_rmid_ptrs) return -ENOMEM; @@ -249,11 +263,9 @@ static int intel_cqm_setup_rmid_cache(void) mutex_unlock(&cache_mutex); return 0; -fail: - while (r--) - kfree(cqm_rmid_ptrs[r]); - kfree(cqm_rmid_ptrs); +fail: + cqm_cleanup(); return -ENOMEM; } @@ -281,9 +293,13 @@ static bool __match_event(struct perf_event *a, struct perf_event *b) /* * Events that target same task are placed into the same cache group. + * Mark it as a multi event group, so that we update ->count + * for every event rather than just the group leader later. */ - if (a->hw.target == b->hw.target) + if (a->hw.target == b->hw.target) { + b->hw.is_group_event = true; return true; + } /* * Are we an inherited event? @@ -849,6 +865,7 @@ static void intel_cqm_setup_event(struct perf_event *event, bool conflict = false; u32 rmid; + event->hw.is_group_event = false; list_for_each_entry(iter, &cache_groups, hw.cqm_groups_entry) { rmid = iter->hw.cqm_rmid; @@ -940,7 +957,9 @@ static u64 intel_cqm_event_count(struct perf_event *event) return __perf_event_count(event); /* - * Only the group leader gets to report values. This stops us + * Only the group leader gets to report values except in case of + * multiple events in the same group, we still need to read the + * other events.This stops us * reporting duplicate values to userspace, and gives us a clear * rule for which task gets to report the values. * @@ -948,7 +967,7 @@ static u64 intel_cqm_event_count(struct perf_event *event) * specific packages - we forfeit that ability when we create * task events. */ - if (!cqm_group_leader(event)) + if (!cqm_group_leader(event) && !event->hw.is_group_event) return 0; /* @@ -1315,7 +1334,7 @@ static const struct x86_cpu_id intel_cqm_match[] = { static int __init intel_cqm_init(void) { - char *str, scale[20]; + char *str = NULL, scale[20]; int i, cpu, ret; if (!x86_match_cpu(intel_cqm_match)) @@ -1375,16 +1394,25 @@ static int __init intel_cqm_init(void) cqm_pick_event_reader(i); } - __perf_cpu_notifier(intel_cqm_cpu_notifier); - ret = perf_pmu_register(&intel_cqm_pmu, "intel_cqm", -1); - if (ret) + if (ret) { pr_err("Intel CQM perf registration failed: %d\n", ret); - else - pr_info("Intel CQM monitoring enabled\n"); + goto out; + } + + pr_info("Intel CQM monitoring enabled\n"); + /* + * Register the hot cpu notifier once we are sure cqm + * is enabled to avoid notifier leak. + */ + __perf_cpu_notifier(intel_cqm_cpu_notifier); out: cpu_notifier_register_done(); + if (ret) { + kfree(str); + cqm_cleanup(); + } return ret; } diff --git a/kernel/arch/x86/kernel/cpu/perf_event_intel_ds.c b/kernel/arch/x86/kernel/cpu/perf_event_intel_ds.c index 5db1c7755..1e7de3cef 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_intel_ds.c @@ -51,7 +51,8 @@ union intel_x86_pebs_dse { #define OP_LH (P(OP, LOAD) | P(LVL, HIT)) #define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS)) -static const u64 pebs_data_source[] = { +/* Version for Sandy Bridge and later */ +static u64 pebs_data_source[] = { P(OP, LOAD) | P(LVL, MISS) | P(LVL, L3) | P(SNOOP, NA),/* 0x00:ukn L3 */ OP_LH | P(LVL, L1) | P(SNOOP, NONE), /* 0x01: L1 local */ OP_LH | P(LVL, LFB) | P(SNOOP, NONE), /* 0x02: LFB hit */ @@ -70,6 +71,14 @@ static const u64 pebs_data_source[] = { OP_LH | P(LVL, UNC) | P(SNOOP, NONE), /* 0x0f: uncached */ }; +/* Patch up minor differences in the bits */ +void __init intel_pmu_pebs_data_source_nhm(void) +{ + pebs_data_source[0x05] = OP_LH | P(LVL, L3) | P(SNOOP, HIT); + pebs_data_source[0x06] = OP_LH | P(LVL, L3) | P(SNOOP, HITM); + pebs_data_source[0x07] = OP_LH | P(LVL, L3) | P(SNOOP, HITM); +} + static u64 precise_store_data(u64 status) { union intel_x86_pebs_dse dse; @@ -269,7 +278,7 @@ static int alloc_pebs_buffer(int cpu) if (!x86_pmu.pebs) return 0; - buffer = kzalloc_node(PEBS_BUFFER_SIZE, GFP_KERNEL, node); + buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node); if (unlikely(!buffer)) return -ENOMEM; @@ -286,7 +295,7 @@ static int alloc_pebs_buffer(int cpu) per_cpu(insn_buffer, cpu) = ibuffer; } - max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size; + max = x86_pmu.pebs_buffer_size / x86_pmu.pebs_record_size; ds->pebs_buffer_base = (u64)(unsigned long)buffer; ds->pebs_index = ds->pebs_buffer_base; @@ -1101,6 +1110,13 @@ get_next_pebs_record_by_bit(void *base, void *top, int bit) void *at; u64 pebs_status; + /* + * fmt0 does not have a status bitfield (does not use + * perf_record_nhm format) + */ + if (x86_pmu.intel_cap.pebs_format < 1) + return base; + if (base == NULL) return NULL; @@ -1186,7 +1202,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs) if (!event->attr.precise_ip) return; - n = (top - at) / x86_pmu.pebs_record_size; + n = top - at; if (n <= 0) return; @@ -1296,6 +1312,7 @@ void __init intel_ds_init(void) x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS); x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS); + x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE; if (x86_pmu.pebs) { char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-'; int format = x86_pmu.intel_cap.pebs_format; @@ -1304,6 +1321,14 @@ void __init intel_ds_init(void) case 0: printk(KERN_CONT "PEBS fmt0%c, ", pebs_type); x86_pmu.pebs_record_size = sizeof(struct pebs_record_core); + /* + * Using >PAGE_SIZE buffers makes the WRMSR to + * PERF_GLOBAL_CTRL in intel_pmu_enable_all() + * mysteriously hang on Core2. + * + * As a workaround, we don't do this. + */ + x86_pmu.pebs_buffer_size = PAGE_SIZE; x86_pmu.drain_pebs = intel_pmu_drain_pebs_core; break; diff --git a/kernel/arch/x86/kernel/cpu/perf_event_intel_pt.c b/kernel/arch/x86/kernel/cpu/perf_event_intel_pt.c index 868e11943..49e35d003 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_intel_pt.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_intel_pt.c @@ -694,6 +694,7 @@ static int pt_buffer_reset_markers(struct pt_buffer *buf, /* clear STOP and INT from current entry */ buf->topa_index[buf->stop_pos]->stop = 0; + buf->topa_index[buf->stop_pos]->intr = 0; buf->topa_index[buf->intr_pos]->intr = 0; /* how many pages till the STOP marker */ @@ -718,6 +719,7 @@ static int pt_buffer_reset_markers(struct pt_buffer *buf, buf->intr_pos = idx; buf->topa_index[buf->stop_pos]->stop = 1; + buf->topa_index[buf->stop_pos]->intr = 1; buf->topa_index[buf->intr_pos]->intr = 1; return 0; diff --git a/kernel/arch/x86/kernel/cpu/perf_event_intel_rapl.c b/kernel/arch/x86/kernel/cpu/perf_event_intel_rapl.c index ed446bdcb..d2ac364e2 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_intel_rapl.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_intel_rapl.c @@ -117,7 +117,7 @@ static struct perf_pmu_events_attr event_attr_##v = { \ }; struct rapl_pmu { - spinlock_t lock; + raw_spinlock_t lock; int n_active; /* number of active events */ struct list_head active_list; struct pmu *pmu; /* pointer to rapl_pmu_class */ @@ -220,13 +220,13 @@ static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer) if (!pmu->n_active) return HRTIMER_NORESTART; - spin_lock_irqsave(&pmu->lock, flags); + raw_spin_lock_irqsave(&pmu->lock, flags); list_for_each_entry(event, &pmu->active_list, active_entry) { rapl_event_update(event); } - spin_unlock_irqrestore(&pmu->lock, flags); + raw_spin_unlock_irqrestore(&pmu->lock, flags); hrtimer_forward_now(hrtimer, pmu->timer_interval); @@ -263,9 +263,9 @@ static void rapl_pmu_event_start(struct perf_event *event, int mode) struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu); unsigned long flags; - spin_lock_irqsave(&pmu->lock, flags); + raw_spin_lock_irqsave(&pmu->lock, flags); __rapl_pmu_event_start(pmu, event); - spin_unlock_irqrestore(&pmu->lock, flags); + raw_spin_unlock_irqrestore(&pmu->lock, flags); } static void rapl_pmu_event_stop(struct perf_event *event, int mode) @@ -274,7 +274,7 @@ static void rapl_pmu_event_stop(struct perf_event *event, int mode) struct hw_perf_event *hwc = &event->hw; unsigned long flags; - spin_lock_irqsave(&pmu->lock, flags); + raw_spin_lock_irqsave(&pmu->lock, flags); /* mark event as deactivated and stopped */ if (!(hwc->state & PERF_HES_STOPPED)) { @@ -299,7 +299,7 @@ static void rapl_pmu_event_stop(struct perf_event *event, int mode) hwc->state |= PERF_HES_UPTODATE; } - spin_unlock_irqrestore(&pmu->lock, flags); + raw_spin_unlock_irqrestore(&pmu->lock, flags); } static int rapl_pmu_event_add(struct perf_event *event, int mode) @@ -308,14 +308,14 @@ static int rapl_pmu_event_add(struct perf_event *event, int mode) struct hw_perf_event *hwc = &event->hw; unsigned long flags; - spin_lock_irqsave(&pmu->lock, flags); + raw_spin_lock_irqsave(&pmu->lock, flags); hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; if (mode & PERF_EF_START) __rapl_pmu_event_start(pmu, event); - spin_unlock_irqrestore(&pmu->lock, flags); + raw_spin_unlock_irqrestore(&pmu->lock, flags); return 0; } @@ -603,7 +603,7 @@ static int rapl_cpu_prepare(int cpu) pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu)); if (!pmu) return -1; - spin_lock_init(&pmu->lock); + raw_spin_lock_init(&pmu->lock); INIT_LIST_HEAD(&pmu->active_list); diff --git a/kernel/arch/x86/kernel/cpu/perf_event_knc.c b/kernel/arch/x86/kernel/cpu/perf_event_knc.c index 5b0c232d1..b931095e8 100644 --- a/kernel/arch/x86/kernel/cpu/perf_event_knc.c +++ b/kernel/arch/x86/kernel/cpu/perf_event_knc.c @@ -263,7 +263,9 @@ again: goto again; done: - knc_pmu_enable_all(0); + /* Only restore PMU state when it's active. See x86_pmu_disable(). */ + if (cpuc->enabled) + knc_pmu_enable_all(0); return handled; } diff --git a/kernel/arch/x86/kernel/e820.c b/kernel/arch/x86/kernel/e820.c index 569c1e4f9..52a2526c3 100644 --- a/kernel/arch/x86/kernel/e820.c +++ b/kernel/arch/x86/kernel/e820.c @@ -753,7 +753,7 @@ u64 __init early_reserve_e820(u64 size, u64 align) /* * Find the highest page frame number we have available */ -static unsigned long __init e820_end_pfn(unsigned long limit_pfn) +static unsigned long __init e820_end_pfn(unsigned long limit_pfn, unsigned type) { int i; unsigned long last_pfn = 0; @@ -764,11 +764,7 @@ static unsigned long __init e820_end_pfn(unsigned long limit_pfn) unsigned long start_pfn; unsigned long end_pfn; - /* - * Persistent memory is accounted as ram for purposes of - * establishing max_pfn and mem_map. - */ - if (ei->type != E820_RAM && ei->type != E820_PRAM) + if (ei->type != type) continue; start_pfn = ei->addr >> PAGE_SHIFT; @@ -793,12 +789,12 @@ static unsigned long __init e820_end_pfn(unsigned long limit_pfn) } unsigned long __init e820_end_of_ram_pfn(void) { - return e820_end_pfn(MAX_ARCH_PFN); + return e820_end_pfn(MAX_ARCH_PFN, E820_RAM); } unsigned long __init e820_end_of_low_ram_pfn(void) { - return e820_end_pfn(1UL << (32-PAGE_SHIFT)); + return e820_end_pfn(1UL << (32 - PAGE_SHIFT), E820_RAM); } static void early_panic(char *msg) diff --git a/kernel/arch/x86/kernel/early-quirks.c b/kernel/arch/x86/kernel/early-quirks.c index db9a675e7..a257d6077 100644 --- a/kernel/arch/x86/kernel/early-quirks.c +++ b/kernel/arch/x86/kernel/early-quirks.c @@ -11,7 +11,11 @@ #include #include +#include +#include #include +#include +#include #include #include #include @@ -21,6 +25,9 @@ #include #include #include +#include + +#define dev_err(msg) pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg) static void __init fix_hypertransport_config(int num, int slot, int func) { @@ -75,6 +82,13 @@ static void __init nvidia_bugs(int num, int slot, int func) { #ifdef CONFIG_ACPI #ifdef CONFIG_X86_IO_APIC + /* + * Only applies to Nvidia root ports (bus 0) and not to + * Nvidia graphics cards with PCI ports on secondary buses. + */ + if (num) + return; + /* * All timer overrides on Nvidia are * wrong unless HPET is enabled. @@ -317,12 +331,11 @@ static u32 __init i85x_stolen_base(int num, int slot, int func, size_t stolen_si static u32 __init i865_stolen_base(int num, int slot, int func, size_t stolen_size) { - /* - * FIXME is the graphics stolen memory region - * always at TOUD? Ie. is it always the last - * one to be allocated by the BIOS? - */ - return read_pci_config_16(0, 0, 0, I865_TOUD) << 16; + u16 toud = 0; + + toud = read_pci_config_16(0, 0, 0, I865_TOUD); + + return (phys_addr_t)(toud << 16) + i845_tseg_size(); } static size_t __init i830_stolen_size(int num, int slot, int func) @@ -589,6 +602,61 @@ static void __init force_disable_hpet(int num, int slot, int func) #endif } +#define BCM4331_MMIO_SIZE 16384 +#define BCM4331_PM_CAP 0x40 +#define bcma_aread32(reg) ioread32(mmio + 1 * BCMA_CORE_SIZE + reg) +#define bcma_awrite32(reg, val) iowrite32(val, mmio + 1 * BCMA_CORE_SIZE + reg) + +static void __init apple_airport_reset(int bus, int slot, int func) +{ + void __iomem *mmio; + u16 pmcsr; + u64 addr; + int i; + + if (!dmi_match(DMI_SYS_VENDOR, "Apple Inc.")) + return; + + /* Card may have been put into PCI_D3hot by grub quirk */ + pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); + + if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { + pmcsr &= ~PCI_PM_CTRL_STATE_MASK; + write_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL, pmcsr); + mdelay(10); + + pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); + if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { + dev_err("Cannot power up Apple AirPort card\n"); + return; + } + } + + addr = read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_0); + addr |= (u64)read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_1) << 32; + addr &= PCI_BASE_ADDRESS_MEM_MASK; + + mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); + if (!mmio) { + dev_err("Cannot iomap Apple AirPort card\n"); + return; + } + + pr_info("Resetting Apple AirPort card (left enabled by EFI)\n"); + + for (i = 0; bcma_aread32(BCMA_RESET_ST) && i < 30; i++) + udelay(10); + + bcma_awrite32(BCMA_RESET_CTL, BCMA_RESET_CTL_RESET); + bcma_aread32(BCMA_RESET_CTL); + udelay(1); + + bcma_awrite32(BCMA_RESET_CTL, 0); + bcma_aread32(BCMA_RESET_CTL); + udelay(10); + + early_iounmap(mmio, BCM4331_MMIO_SIZE); +} #define QFLAG_APPLY_ONCE 0x1 #define QFLAG_APPLIED 0x2 @@ -602,12 +670,6 @@ struct chipset { void (*f)(int num, int slot, int func); }; -/* - * Only works for devices on the root bus. If you add any devices - * not on bus 0 readd another loop level in early_quirks(). But - * be careful because at least the Nvidia quirk here relies on - * only matching on bus 0. - */ static struct chipset early_qrk[] __initdata = { { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, nvidia_bugs }, @@ -637,9 +699,13 @@ static struct chipset early_qrk[] __initdata = { */ { PCI_VENDOR_ID_INTEL, 0x0f00, PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, force_disable_hpet}, + { PCI_VENDOR_ID_BROADCOM, 0x4331, + PCI_CLASS_NETWORK_OTHER, PCI_ANY_ID, 0, apple_airport_reset}, {} }; +static void __init early_pci_scan_bus(int bus); + /** * check_dev_quirk - apply early quirks to a given PCI device * @num: bus number @@ -648,7 +714,7 @@ static struct chipset early_qrk[] __initdata = { * * Check the vendor & device ID against the early quirks table. * - * If the device is single function, let early_quirks() know so we don't + * If the device is single function, let early_pci_scan_bus() know so we don't * poke at this device again. */ static int __init check_dev_quirk(int num, int slot, int func) @@ -657,6 +723,7 @@ static int __init check_dev_quirk(int num, int slot, int func) u16 vendor; u16 device; u8 type; + u8 sec; int i; class = read_pci_config_16(num, slot, func, PCI_CLASS_DEVICE); @@ -684,25 +751,36 @@ static int __init check_dev_quirk(int num, int slot, int func) type = read_pci_config_byte(num, slot, func, PCI_HEADER_TYPE); + + if ((type & 0x7f) == PCI_HEADER_TYPE_BRIDGE) { + sec = read_pci_config_byte(num, slot, func, PCI_SECONDARY_BUS); + if (sec > num) + early_pci_scan_bus(sec); + } + if (!(type & 0x80)) return -1; return 0; } -void __init early_quirks(void) +static void __init early_pci_scan_bus(int bus) { int slot, func; - if (!early_pci_allowed()) - return; - /* Poor man's PCI discovery */ - /* Only scan the root bus */ for (slot = 0; slot < 32; slot++) for (func = 0; func < 8; func++) { /* Only probe function 0 on single fn devices */ - if (check_dev_quirk(0, slot, func)) + if (check_dev_quirk(bus, slot, func)) break; } } + +void __init early_quirks(void) +{ + if (!early_pci_allowed()) + return; + + early_pci_scan_bus(0); +} diff --git a/kernel/arch/x86/kernel/head_32.S b/kernel/arch/x86/kernel/head_32.S index 6bc9ae24b..8f1a3f443 100644 --- a/kernel/arch/x86/kernel/head_32.S +++ b/kernel/arch/x86/kernel/head_32.S @@ -571,7 +571,7 @@ early_idt_handler_common: movl %eax,%ds movl %eax,%es - cmpl $(__KERNEL_CS),32(%esp) + cmpw $(__KERNEL_CS),32(%esp) jne 10f leal 28(%esp),%eax # Pointer to %eip diff --git a/kernel/arch/x86/kernel/hpet.c b/kernel/arch/x86/kernel/hpet.c index b8e6ff5cd..acc9b8f19 100644 --- a/kernel/arch/x86/kernel/hpet.c +++ b/kernel/arch/x86/kernel/hpet.c @@ -351,6 +351,7 @@ static int hpet_resume(struct clock_event_device *evt, int timer) } else { struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + irq_domain_deactivate_irq(irq_get_irq_data(hdev->irq)); irq_domain_activate_irq(irq_get_irq_data(hdev->irq)); disable_irq(hdev->irq); irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); diff --git a/kernel/arch/x86/kernel/ioport.c b/kernel/arch/x86/kernel/ioport.c index 37dae792d..589b3193f 100644 --- a/kernel/arch/x86/kernel/ioport.c +++ b/kernel/arch/x86/kernel/ioport.c @@ -96,9 +96,14 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on) SYSCALL_DEFINE1(iopl, unsigned int, level) { struct pt_regs *regs = current_pt_regs(); - unsigned int old = (regs->flags >> 12) & 3; struct thread_struct *t = ¤t->thread; + /* + * Careful: the IOPL bits in regs->flags are undefined under Xen PV + * and changing them has no effect. + */ + unsigned int old = t->iopl >> X86_EFLAGS_IOPL_BIT; + if (level > 3) return -EINVAL; /* Trying to gain more privileges? */ @@ -106,8 +111,9 @@ SYSCALL_DEFINE1(iopl, unsigned int, level) if (!capable(CAP_SYS_RAWIO)) return -EPERM; } - regs->flags = (regs->flags & ~X86_EFLAGS_IOPL) | (level << 12); - t->iopl = level << 12; + regs->flags = (regs->flags & ~X86_EFLAGS_IOPL) | + (level << X86_EFLAGS_IOPL_BIT); + t->iopl = level << X86_EFLAGS_IOPL_BIT; set_iopl_mask(t->iopl); return 0; diff --git a/kernel/arch/x86/kernel/kprobes/core.c b/kernel/arch/x86/kernel/kprobes/core.c index 1deffe6cc..023c442c3 100644 --- a/kernel/arch/x86/kernel/kprobes/core.c +++ b/kernel/arch/x86/kernel/kprobes/core.c @@ -959,7 +959,19 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr) * normal page fault. */ regs->ip = (unsigned long)cur->addr; + /* + * Trap flag (TF) has been set here because this fault + * happened where the single stepping will be done. + * So clear it by resetting the current kprobe: + */ + regs->flags &= ~X86_EFLAGS_TF; + + /* + * If the TF flag was set before the kprobe hit, + * don't touch it: + */ regs->flags |= kcb->kprobe_old_flags; + if (kcb->kprobe_status == KPROBE_REENTER) restore_previous_kprobe(kcb); else diff --git a/kernel/arch/x86/kernel/mcount_64.S b/kernel/arch/x86/kernel/mcount_64.S index 87e1762e2..5d9afbcb6 100644 --- a/kernel/arch/x86/kernel/mcount_64.S +++ b/kernel/arch/x86/kernel/mcount_64.S @@ -180,7 +180,8 @@ GLOBAL(ftrace_graph_call) jmp ftrace_stub #endif -GLOBAL(ftrace_stub) +/* This is weak to keep gas from relaxing the jumps */ +WEAK(ftrace_stub) retq END(ftrace_caller) diff --git a/kernel/arch/x86/kernel/nmi.c b/kernel/arch/x86/kernel/nmi.c index 697f90db0..424aec4a4 100644 --- a/kernel/arch/x86/kernel/nmi.c +++ b/kernel/arch/x86/kernel/nmi.c @@ -231,7 +231,7 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs) #endif if (panic_on_unrecovered_nmi) - panic("NMI: Not continuing"); + nmi_panic(regs, "NMI: Not continuing"); pr_emerg("Dazed and confused, but trying to continue\n"); @@ -255,8 +255,16 @@ io_check_error(unsigned char reason, struct pt_regs *regs) reason, smp_processor_id()); show_regs(regs); - if (panic_on_io_nmi) - panic("NMI IOCK error: Not continuing"); + if (panic_on_io_nmi) { + nmi_panic(regs, "NMI IOCK error: Not continuing"); + + /* + * If we end up here, it means we have received an NMI while + * processing panic(). Simply return without delaying and + * re-enabling NMIs. + */ + return; + } /* Re-enable the IOCK line, wait for a few seconds */ reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK; @@ -297,7 +305,7 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) pr_emerg("Do you have a strange power saving mode enabled?\n"); if (unknown_nmi_panic || panic_on_unrecovered_nmi) - panic("NMI: Not continuing"); + nmi_panic(regs, "NMI: Not continuing"); pr_emerg("Dazed and confused, but trying to continue\n"); } diff --git a/kernel/arch/x86/kernel/paravirt.c b/kernel/arch/x86/kernel/paravirt.c index c2130aef3..f534a0e3a 100644 --- a/kernel/arch/x86/kernel/paravirt.c +++ b/kernel/arch/x86/kernel/paravirt.c @@ -55,12 +55,12 @@ asm (".pushsection .entry.text, \"ax\"\n" ".popsection"); /* identity function, which can be inlined */ -u32 _paravirt_ident_32(u32 x) +u32 notrace _paravirt_ident_32(u32 x) { return x; } -u64 _paravirt_ident_64(u64 x) +u64 notrace _paravirt_ident_64(u64 x) { return x; } diff --git a/kernel/arch/x86/kernel/process_64.c b/kernel/arch/x86/kernel/process_64.c index e835d263a..4cbb60fbf 100644 --- a/kernel/arch/x86/kernel/process_64.c +++ b/kernel/arch/x86/kernel/process_64.c @@ -48,6 +48,7 @@ #include #include #include +#include asmlinkage extern void ret_from_fork(void); @@ -411,6 +412,17 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) __switch_to_xtra(prev_p, next_p, tss); +#ifdef CONFIG_XEN + /* + * On Xen PV, IOPL bits in pt_regs->flags have no effect, and + * current_pt_regs()->flags may not match the current task's + * intended IOPL. We need to switch it manually. + */ + if (unlikely(static_cpu_has(X86_FEATURE_XENPV) && + prev->iopl != next->iopl)) + xen_set_iopl_mask(next->iopl); +#endif + if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) { /* * AMD CPUs have a misfeature: SYSRET sets the SS selector but diff --git a/kernel/arch/x86/kernel/ptrace.c b/kernel/arch/x86/kernel/ptrace.c index 558f50ede..479a409dd 100644 --- a/kernel/arch/x86/kernel/ptrace.c +++ b/kernel/arch/x86/kernel/ptrace.c @@ -188,8 +188,8 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs) return sp; prev_esp = (u32 *)(context); - if (prev_esp) - return (unsigned long)prev_esp; + if (*prev_esp) + return (unsigned long)*prev_esp; return (unsigned long)regs; } diff --git a/kernel/arch/x86/kernel/pvclock.c b/kernel/arch/x86/kernel/pvclock.c index 2f355d229..bf0ce7573 100644 --- a/kernel/arch/x86/kernel/pvclock.c +++ b/kernel/arch/x86/kernel/pvclock.c @@ -66,6 +66,8 @@ u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src) do { version = __pvclock_read_cycles(src, &ret, &flags); + /* Make sure that the version double-check is last. */ + smp_rmb(); } while ((src->version & 1) || version != src->version); return flags & valid_flags; @@ -80,6 +82,8 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src) do { version = __pvclock_read_cycles(src, &ret, &flags); + /* Make sure that the version double-check is last. */ + smp_rmb(); } while ((src->version & 1) || version != src->version); if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) { diff --git a/kernel/arch/x86/kernel/reboot.c b/kernel/arch/x86/kernel/reboot.c index f660d63f4..8384207ad 100644 --- a/kernel/arch/x86/kernel/reboot.c +++ b/kernel/arch/x86/kernel/reboot.c @@ -726,6 +726,7 @@ static int crashing_cpu; static nmi_shootdown_cb shootdown_callback; static atomic_t waiting_for_crash_ipi; +static int crash_ipi_issued; static int crash_nmi_callback(unsigned int val, struct pt_regs *regs) { @@ -788,6 +789,9 @@ void nmi_shootdown_cpus(nmi_shootdown_cb callback) smp_send_nmi_allbutself(); + /* Kick CPUs looping in NMI context. */ + WRITE_ONCE(crash_ipi_issued, 1); + msecs = 1000; /* Wait at most a second for the other cpus to stop */ while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { mdelay(1); @@ -796,6 +800,22 @@ void nmi_shootdown_cpus(nmi_shootdown_cb callback) /* Leave the nmi callback set */ } + +/* Override the weak function in kernel/panic.c */ +void nmi_panic_self_stop(struct pt_regs *regs) +{ + while (1) { + /* + * Wait for the crash dumping IPI to be issued, and then + * call its callback directly. + */ + if (READ_ONCE(crash_ipi_issued)) + crash_nmi_callback(0, regs); /* Don't return */ + + cpu_relax(); + } +} + #else /* !CONFIG_SMP */ void nmi_shootdown_cpus(nmi_shootdown_cb callback) { diff --git a/kernel/arch/x86/kernel/sysfb_efi.c b/kernel/arch/x86/kernel/sysfb_efi.c index b285d4e8c..5da924bbf 100644 --- a/kernel/arch/x86/kernel/sysfb_efi.c +++ b/kernel/arch/x86/kernel/sysfb_efi.c @@ -106,14 +106,24 @@ static int __init efifb_set_system(const struct dmi_system_id *id) continue; for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { resource_size_t start, end; + unsigned long flags; + + flags = pci_resource_flags(dev, i); + if (!(flags & IORESOURCE_MEM)) + continue; + + if (flags & IORESOURCE_UNSET) + continue; + + if (pci_resource_len(dev, i) == 0) + continue; start = pci_resource_start(dev, i); - if (start == 0) - break; end = pci_resource_end(dev, i); if (screen_info.lfb_base >= start && screen_info.lfb_base < end) { found_bar = 1; + break; } } } diff --git a/kernel/arch/x86/kernel/traps.c b/kernel/arch/x86/kernel/traps.c index ade185a46..679302c31 100644 --- a/kernel/arch/x86/kernel/traps.c +++ b/kernel/arch/x86/kernel/traps.c @@ -109,6 +109,12 @@ static inline void preempt_conditional_cli(struct pt_regs *regs) preempt_count_dec(); } +/* + * In IST context, we explicitly disable preemption. This serves two + * purposes: it makes it much less likely that we would accidentally + * schedule in IST context and it will force a warning if we somehow + * manage to schedule by accident. + */ void ist_enter(struct pt_regs *regs) { if (user_mode(regs)) { @@ -123,13 +129,7 @@ void ist_enter(struct pt_regs *regs) rcu_nmi_enter(); } - /* - * We are atomic because we're on the IST stack; or we're on - * x86_32, in which case we still shouldn't schedule; or we're - * on x86_64 and entered from user mode, in which case we're - * still atomic unless ist_begin_non_atomic is called. - */ - preempt_count_add(HARDIRQ_OFFSET); + preempt_disable(); /* This code is a bit fragile. Test it. */ RCU_LOCKDEP_WARN(!rcu_is_watching(), "ist_enter didn't work"); @@ -137,7 +137,7 @@ void ist_enter(struct pt_regs *regs) void ist_exit(struct pt_regs *regs) { - preempt_count_sub(HARDIRQ_OFFSET); + preempt_enable_no_resched(); if (!user_mode(regs)) rcu_nmi_exit(); @@ -168,7 +168,7 @@ void ist_begin_non_atomic(struct pt_regs *regs) BUG_ON((unsigned long)(current_top_of_stack() - current_stack_pointer()) >= THREAD_SIZE); - preempt_count_sub(HARDIRQ_OFFSET); + preempt_enable_no_resched(); } /** @@ -178,7 +178,7 @@ void ist_begin_non_atomic(struct pt_regs *regs) */ void ist_end_non_atomic(void) { - preempt_count_add(HARDIRQ_OFFSET); + preempt_disable(); } static nokprobe_inline int diff --git a/kernel/arch/x86/kernel/tsc_msr.c b/kernel/arch/x86/kernel/tsc_msr.c index 92ae6acac..6aa0f4d9e 100644 --- a/kernel/arch/x86/kernel/tsc_msr.c +++ b/kernel/arch/x86/kernel/tsc_msr.c @@ -92,7 +92,7 @@ unsigned long try_msr_calibrate_tsc(void) if (freq_desc_tables[cpu_index].msr_plat) { rdmsr(MSR_PLATFORM_INFO, lo, hi); - ratio = (lo >> 8) & 0x1f; + ratio = (lo >> 8) & 0xff; } else { rdmsr(MSR_IA32_PERF_STATUS, lo, hi); ratio = (hi >> 8) & 0x1f; diff --git a/kernel/arch/x86/kernel/uprobes.c b/kernel/arch/x86/kernel/uprobes.c index bf4db6eae..c6aace2bb 100644 --- a/kernel/arch/x86/kernel/uprobes.c +++ b/kernel/arch/x86/kernel/uprobes.c @@ -357,20 +357,22 @@ static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn) *cursor &= 0xfe; } /* - * Similar treatment for VEX3 prefix. - * TODO: add XOP/EVEX treatment when insn decoder supports them + * Similar treatment for VEX3/EVEX prefix. + * TODO: add XOP treatment when insn decoder supports them */ - if (insn->vex_prefix.nbytes == 3) { + if (insn->vex_prefix.nbytes >= 3) { /* * vex2: c5 rvvvvLpp (has no b bit) * vex3/xop: c4/8f rxbmmmmm wvvvvLpp * evex: 62 rxbR00mm wvvvv1pp zllBVaaa - * (evex will need setting of both b and x since - * in non-sib encoding evex.x is 4th bit of MODRM.rm) - * Setting VEX3.b (setting because it has inverted meaning): + * Setting VEX3.b (setting because it has inverted meaning). + * Setting EVEX.x since (in non-SIB encoding) EVEX.x + * is the 4th bit of MODRM.rm, and needs the same treatment. + * For VEX3-encoded insns, VEX3.x value has no effect in + * non-SIB encoding, the change is superfluous but harmless. */ cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1; - *cursor |= 0x20; + *cursor |= 0x60; } /* @@ -415,12 +417,10 @@ static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn) reg = MODRM_REG(insn); /* Fetch modrm.reg */ reg2 = 0xff; /* Fetch vex.vvvv */ - if (insn->vex_prefix.nbytes == 2) - reg2 = insn->vex_prefix.bytes[1]; - else if (insn->vex_prefix.nbytes == 3) + if (insn->vex_prefix.nbytes) reg2 = insn->vex_prefix.bytes[2]; /* - * TODO: add XOP, EXEV vvvv reading. + * TODO: add XOP vvvv reading. * * vex.vvvv field is in bits 6-3, bits are inverted. * But in 32-bit mode, high-order bit may be ignored. diff --git a/kernel/arch/x86/kvm/cpuid.c b/kernel/arch/x86/kvm/cpuid.c index 6525e926f..2e1fd586b 100644 --- a/kernel/arch/x86/kvm/cpuid.c +++ b/kernel/arch/x86/kvm/cpuid.c @@ -509,6 +509,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, do_cpuid_1_ent(&entry[i], function, idx); if (idx == 1) { entry[i].eax &= kvm_supported_word10_x86_features; + cpuid_mask(&entry[i].eax, 10); entry[i].ebx = 0; if (entry[i].eax & (F(XSAVES)|F(XSAVEC))) entry[i].ebx = diff --git a/kernel/arch/x86/kvm/emulate.c b/kernel/arch/x86/kvm/emulate.c index b9b09fec1..1dcea2259 100644 --- a/kernel/arch/x86/kvm/emulate.c +++ b/kernel/arch/x86/kvm/emulate.c @@ -172,6 +172,7 @@ #define NearBranch ((u64)1 << 52) /* Near branches */ #define No16 ((u64)1 << 53) /* No 16 bit operand */ #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */ +#define Aligned16 ((u64)1 << 55) /* Aligned to 16 byte boundary (e.g. FXSAVE) */ #define DstXacc (DstAccLo | SrcAccHi | SrcWrite) @@ -434,6 +435,26 @@ FOP_END; FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET FOP_END; +/* + * XXX: inoutclob user must know where the argument is being expanded. + * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault. + */ +#define asm_safe(insn, inoutclob...) \ +({ \ + int _fault = 0; \ + \ + asm volatile("1:" insn "\n" \ + "2:\n" \ + ".pushsection .fixup, \"ax\"\n" \ + "3: movl $1, %[_fault]\n" \ + " jmp 2b\n" \ + ".popsection\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [_fault] "+qm"(_fault) inoutclob ); \ + \ + _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ +}) + static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt, enum x86_intercept intercept, enum x86_intercept_stage stage) @@ -620,21 +641,24 @@ static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector, * depending on whether they're AVX encoded or not. * * Also included is CMPXCHG16B which is not a vector instruction, yet it is - * subject to the same check. + * subject to the same check. FXSAVE and FXRSTOR are checked here too as their + * 512 bytes of data must be aligned to a 16 byte boundary. */ -static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size) +static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) { if (likely(size < 16)) - return false; + return 1; if (ctxt->d & Aligned) - return true; + return size; else if (ctxt->d & Unaligned) - return false; + return 1; else if (ctxt->d & Avx) - return false; + return 1; + else if (ctxt->d & Aligned16) + return 16; else - return true; + return size; } static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, @@ -692,7 +716,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, } break; } - if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0)) + if (la & (insn_alignment(ctxt, size) - 1)) return emulate_gp(ctxt, 0); return X86EMUL_CONTINUE; bad: @@ -779,6 +803,20 @@ static int segmented_read_std(struct x86_emulate_ctxt *ctxt, return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception); } +static int segmented_write_std(struct x86_emulate_ctxt *ctxt, + struct segmented_address addr, + void *data, + unsigned int size) +{ + int rc; + ulong linear; + + rc = linearize(ctxt, addr, size, true, &linear); + if (rc != X86EMUL_CONTINUE) + return rc; + return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception); +} + /* * Prefetch the remaining bytes of the instruction without crossing page * boundary if they are not in fetch_cache yet. @@ -1532,7 +1570,6 @@ static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt, &ctxt->exception); } -/* Does not support long mode */ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt, u16 selector, int seg, u8 cpl, enum x86_transfer_type transfer, @@ -1569,20 +1606,34 @@ static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt, rpl = selector & 3; - /* NULL selector is not valid for TR, CS and SS (except for long mode) */ - if ((seg == VCPU_SREG_CS - || (seg == VCPU_SREG_SS - && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)) - || seg == VCPU_SREG_TR) - && null_selector) - goto exception; - /* TR should be in GDT only */ if (seg == VCPU_SREG_TR && (selector & (1 << 2))) goto exception; - if (null_selector) /* for NULL selector skip all following checks */ + /* NULL selector is not valid for TR, CS and (except for long mode) SS */ + if (null_selector) { + if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR) + goto exception; + + if (seg == VCPU_SREG_SS) { + if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) + goto exception; + + /* + * ctxt->ops->set_segment expects the CPL to be in + * SS.DPL, so fake an expand-up 32-bit data segment. + */ + seg_desc.type = 3; + seg_desc.p = 1; + seg_desc.s = 1; + seg_desc.dpl = cpl; + seg_desc.d = 1; + seg_desc.g = 1; + } + + /* Skip all following checks */ goto load; + } ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr); if (ret != X86EMUL_CONTINUE) @@ -1698,6 +1749,21 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, u16 selector, int seg) { u8 cpl = ctxt->ops->cpl(ctxt); + + /* + * None of MOV, POP and LSS can load a NULL selector in CPL=3, but + * they can load it at CPL<3 (Intel's manual says only LSS can, + * but it's wrong). + * + * However, the Intel manual says that putting IST=1/DPL=3 in + * an interrupt gate will result in SS=3 (the AMD manual instead + * says it doesn't), so allow SS=3 in __load_segment_descriptor + * and only forbid it here. + */ + if (seg == VCPU_SREG_SS && selector == 3 && + ctxt->mode == X86EMUL_MODE_PROT64) + return emulate_exception(ctxt, GP_VECTOR, 0, true); + return __load_segment_descriptor(ctxt, selector, seg, cpl, X86_TRANSFER_NONE, NULL); } @@ -2093,16 +2159,10 @@ static int em_iret(struct x86_emulate_ctxt *ctxt) static int em_jmp_far(struct x86_emulate_ctxt *ctxt) { int rc; - unsigned short sel, old_sel; - struct desc_struct old_desc, new_desc; - const struct x86_emulate_ops *ops = ctxt->ops; + unsigned short sel; + struct desc_struct new_desc; u8 cpl = ctxt->ops->cpl(ctxt); - /* Assignment of RIP may only fail in 64-bit mode */ - if (ctxt->mode == X86EMUL_MODE_PROT64) - ops->get_segment(ctxt, &old_sel, &old_desc, NULL, - VCPU_SREG_CS); - memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, @@ -2112,12 +2172,10 @@ static int em_jmp_far(struct x86_emulate_ctxt *ctxt) return rc; rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc); - if (rc != X86EMUL_CONTINUE) { - WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64); - /* assigning eip failed; restore the old cs */ - ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS); - return rc; - } + /* Error handling is not implemented. */ + if (rc != X86EMUL_CONTINUE) + return X86EMUL_UNHANDLEABLE; + return rc; } @@ -2177,14 +2235,8 @@ static int em_ret_far(struct x86_emulate_ctxt *ctxt) { int rc; unsigned long eip, cs; - u16 old_cs; int cpl = ctxt->ops->cpl(ctxt); - struct desc_struct old_desc, new_desc; - const struct x86_emulate_ops *ops = ctxt->ops; - - if (ctxt->mode == X86EMUL_MODE_PROT64) - ops->get_segment(ctxt, &old_cs, &old_desc, NULL, - VCPU_SREG_CS); + struct desc_struct new_desc; rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); if (rc != X86EMUL_CONTINUE) @@ -2201,10 +2253,10 @@ static int em_ret_far(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; rc = assign_eip_far(ctxt, eip, &new_desc); - if (rc != X86EMUL_CONTINUE) { - WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64); - ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS); - } + /* Error handling is not implemented. */ + if (rc != X86EMUL_CONTINUE) + return X86EMUL_UNHANDLEABLE; + return rc; } @@ -3660,8 +3712,8 @@ static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt, } /* Disable writeback. */ ctxt->dst.type = OP_NONE; - return segmented_write(ctxt, ctxt->dst.addr.mem, - &desc_ptr, 2 + ctxt->op_bytes); + return segmented_write_std(ctxt, ctxt->dst.addr.mem, + &desc_ptr, 2 + ctxt->op_bytes); } static int em_sgdt(struct x86_emulate_ctxt *ctxt) @@ -3844,6 +3896,131 @@ static int em_movsxd(struct x86_emulate_ctxt *ctxt) return X86EMUL_CONTINUE; } +static int check_fxsr(struct x86_emulate_ctxt *ctxt) +{ + u32 eax = 1, ebx, ecx = 0, edx; + + ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); + if (!(edx & FFL(FXSR))) + return emulate_ud(ctxt); + + if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) + return emulate_nm(ctxt); + + /* + * Don't emulate a case that should never be hit, instead of working + * around a lack of fxsave64/fxrstor64 on old compilers. + */ + if (ctxt->mode >= X86EMUL_MODE_PROT64) + return X86EMUL_UNHANDLEABLE; + + return X86EMUL_CONTINUE; +} + +/* + * FXSAVE and FXRSTOR have 4 different formats depending on execution mode, + * 1) 16 bit mode + * 2) 32 bit mode + * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs + * preserve whole 32 bit values, though, so (1) and (2) are the same wrt. + * save and restore + * 3) 64-bit mode with REX.W prefix + * - like (2), but XMM 8-15 are being saved and restored + * 4) 64-bit mode without REX.W prefix + * - like (3), but FIP and FDP are 64 bit + * + * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the + * desired result. (4) is not emulated. + * + * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS + * and FPU DS) should match. + */ +static int em_fxsave(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + size_t size; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + ctxt->ops->get_fpu(ctxt); + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + if (rc != X86EMUL_CONTINUE) + return rc; + + if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR) + size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]); + else + size = offsetof(struct fxregs_state, xmm_space[0]); + + return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, size); +} + +static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt, + struct fxregs_state *new) +{ + int rc = X86EMUL_CONTINUE; + struct fxregs_state old; + + rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old)); + if (rc != X86EMUL_CONTINUE) + return rc; + + /* + * 64 bit host will restore XMM 8-15, which is not correct on non-64 + * bit guests. Load the current values in order to preserve 64 bit + * XMMs after fxrstor. + */ +#ifdef CONFIG_X86_64 + /* XXX: accessing XMM 8-15 very awkwardly */ + memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16); +#endif + + /* + * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but + * does save and restore MXCSR. + */ + if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)) + memcpy(new->xmm_space, old.xmm_space, 8 * 16); + + return rc; +} + +static int em_fxrstor(struct x86_emulate_ctxt *ctxt) +{ + struct fxregs_state fx_state; + int rc; + + rc = check_fxsr(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; + + rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, 512); + if (rc != X86EMUL_CONTINUE) + return rc; + + if (fx_state.mxcsr >> 16) + return emulate_gp(ctxt, 0); + + ctxt->ops->get_fpu(ctxt); + + if (ctxt->mode < X86EMUL_MODE_PROT64) + rc = fxrstor_fixup(ctxt, &fx_state); + + if (rc == X86EMUL_CONTINUE) + rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state)); + + ctxt->ops->put_fpu(ctxt); + + return rc; +} + static bool valid_cr(int nr) { switch (nr) { @@ -4196,7 +4373,9 @@ static const struct gprefix pfx_0f_ae_7 = { }; static const struct group_dual group15 = { { - N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7), + I(ModRM | Aligned16, em_fxsave), + I(ModRM | Aligned16, em_fxrstor), + N, N, N, N, N, GP(0, &pfx_0f_ae_7), }, { N, N, N, N, N, N, N, N, } }; @@ -5033,7 +5212,7 @@ done_prefixes: /* Decode and fetch the destination operand: register or memory. */ rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask); - if (ctxt->rip_relative) + if (ctxt->rip_relative && likely(ctxt->memopp)) ctxt->memopp->addr.mem.ea = address_mask(ctxt, ctxt->memopp->addr.mem.ea + ctxt->_eip); @@ -5068,21 +5247,13 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt) static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt) { - bool fault = false; + int rc; ctxt->ops->get_fpu(ctxt); - asm volatile("1: fwait \n\t" - "2: \n\t" - ".pushsection .fixup,\"ax\" \n\t" - "3: \n\t" - "movb $1, %[fault] \n\t" - "jmp 2b \n\t" - ".popsection \n\t" - _ASM_EXTABLE(1b, 3b) - : [fault]"+qm"(fault)); + rc = asm_safe("fwait"); ctxt->ops->put_fpu(ctxt); - if (unlikely(fault)) + if (unlikely(rc != X86EMUL_CONTINUE)) return emulate_exception(ctxt, MF_VECTOR, 0, false); return X86EMUL_CONTINUE; diff --git a/kernel/arch/x86/kvm/i8254.c b/kernel/arch/x86/kvm/i8254.c index b0ea42b78..ab5318727 100644 --- a/kernel/arch/x86/kvm/i8254.c +++ b/kernel/arch/x86/kvm/i8254.c @@ -245,7 +245,7 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian) * PIC is being reset. Handle it gracefully here */ atomic_inc(&ps->pending); - else if (value > 0) + else if (value > 0 && ps->reinject) /* in this case, we had multiple outstanding pit interrupts * that we needed to inject. Reinject */ @@ -288,7 +288,9 @@ static void pit_do_work(struct kthread_work *work) * last one has been acked. */ spin_lock(&ps->inject_lock); - if (ps->irq_ack) { + if (!ps->reinject) + inject = 1; + else if (ps->irq_ack) { ps->irq_ack = 0; inject = 1; } @@ -317,10 +319,10 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) struct kvm_kpit_state *ps = container_of(data, struct kvm_kpit_state, timer); struct kvm_pit *pt = ps->kvm->arch.vpit; - if (ps->reinject || !atomic_read(&ps->pending)) { + if (ps->reinject) atomic_inc(&ps->pending); - queue_kthread_work(&pt->worker, &pt->expired); - } + + queue_kthread_work(&pt->worker, &pt->expired); if (ps->is_periodic) { hrtimer_add_expires_ns(&ps->timer, ps->period); diff --git a/kernel/arch/x86/kvm/ioapic.c b/kernel/arch/x86/kvm/ioapic.c index 88d0a92d3..3aab53f8c 100644 --- a/kernel/arch/x86/kvm/ioapic.c +++ b/kernel/arch/x86/kvm/ioapic.c @@ -580,7 +580,7 @@ static void kvm_ioapic_reset(struct kvm_ioapic *ioapic) ioapic->irr = 0; ioapic->irr_delivered = 0; ioapic->id = 0; - memset(ioapic->irq_eoi, 0x00, IOAPIC_NUM_PINS); + memset(ioapic->irq_eoi, 0x00, sizeof(ioapic->irq_eoi)); rtc_irq_eoi_tracking_reset(ioapic); } diff --git a/kernel/arch/x86/kvm/irq_comm.c b/kernel/arch/x86/kvm/irq_comm.c index 84b96d319..d09544e82 100644 --- a/kernel/arch/x86/kvm/irq_comm.c +++ b/kernel/arch/x86/kvm/irq_comm.c @@ -38,6 +38,15 @@ static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, bool line_status) { struct kvm_pic *pic = pic_irqchip(kvm); + + /* + * XXX: rejecting pic routes when pic isn't in use would be better, + * but the default routing table is installed while kvm->arch.vpic is + * NULL and KVM_CREATE_IRQCHIP can race with KVM_IRQ_LINE. + */ + if (!pic) + return -1; + return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); } @@ -46,6 +55,10 @@ static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, bool line_status) { struct kvm_ioapic *ioapic = kvm->arch.vioapic; + + if (!ioapic) + return -1; + return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, line_status); } diff --git a/kernel/arch/x86/kvm/lapic.c b/kernel/arch/x86/kvm/lapic.c index 3d1b17068..6cd4f4b32 100644 --- a/kernel/arch/x86/kvm/lapic.c +++ b/kernel/arch/x86/kvm/lapic.c @@ -2266,3 +2266,9 @@ void kvm_lapic_init(void) jump_label_rate_limit(&apic_hw_disabled, HZ); jump_label_rate_limit(&apic_sw_disabled, HZ); } + +void kvm_lapic_exit(void) +{ + static_key_deferred_flush(&apic_hw_disabled); + static_key_deferred_flush(&apic_sw_disabled); +} diff --git a/kernel/arch/x86/kvm/lapic.h b/kernel/arch/x86/kvm/lapic.h index 640ad275b..04141a598 100644 --- a/kernel/arch/x86/kvm/lapic.h +++ b/kernel/arch/x86/kvm/lapic.h @@ -96,6 +96,7 @@ static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu) int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data); void kvm_lapic_init(void); +void kvm_lapic_exit(void); static inline u32 kvm_apic_get_reg(struct kvm_lapic *apic, int reg_off) { diff --git a/kernel/arch/x86/kvm/mtrr.c b/kernel/arch/x86/kvm/mtrr.c index 3f8c73211..0149ac59c 100644 --- a/kernel/arch/x86/kvm/mtrr.c +++ b/kernel/arch/x86/kvm/mtrr.c @@ -44,8 +44,6 @@ static bool msr_mtrr_valid(unsigned msr) case MSR_MTRRdefType: case MSR_IA32_CR_PAT: return true; - case 0x2f8: - return true; } return false; } @@ -541,6 +539,7 @@ static void mtrr_lookup_var_start(struct mtrr_iter *iter) iter->fixed = false; iter->start_max = iter->start; + iter->range = NULL; iter->range = list_prepare_entry(iter->range, &mtrr_state->head, node); __mtrr_lookup_var_next(iter); diff --git a/kernel/arch/x86/kvm/vmx.c b/kernel/arch/x86/kvm/vmx.c index a722f724c..8d744a82f 100644 --- a/kernel/arch/x86/kvm/vmx.c +++ b/kernel/arch/x86/kvm/vmx.c @@ -415,6 +415,7 @@ struct nested_vmx { struct list_head vmcs02_pool; int vmcs02_num; u64 vmcs01_tsc_offset; + bool change_vmcs01_virtual_x2apic_mode; /* L2 must run next, and mustn't decide to exit to L1. */ bool nested_run_pending; /* @@ -1308,10 +1309,10 @@ static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12) return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR; } -static inline bool is_exception(u32 intr_info) +static inline bool is_nmi(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) - == (INTR_TYPE_HARD_EXCEPTION | INTR_INFO_VALID_MASK); + == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK); } static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, @@ -2699,8 +2700,15 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx) } else vmx->nested.nested_vmx_ept_caps = 0; + /* + * Old versions of KVM use the single-context version without + * checking for support, so declare that it is supported even + * though it is treated as global context. The alternative is + * not failing the single-context invvpid, and it is worse. + */ if (enable_vpid) vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT | + VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT; else vmx->nested.nested_vmx_vpid_caps = 0; @@ -5016,8 +5024,8 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET; - vmx_set_cr0(vcpu, cr0); /* enter rmode */ vmx->vcpu.arch.cr0 = cr0; + vmx_set_cr0(vcpu, cr0); /* enter rmode */ vmx_set_cr4(vcpu, 0); vmx_set_efer(vcpu, 0); vmx_fpu_activate(vcpu); @@ -5295,7 +5303,7 @@ static int handle_exception(struct kvm_vcpu *vcpu) if (is_machine_check(intr_info)) return handle_machine_check(vcpu); - if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR) + if (is_nmi(intr_info)) return 1; /* already handled by vmx_vcpu_run() */ if (is_no_device(intr_info)) { @@ -6652,7 +6660,13 @@ static int get_vmx_mem_address(struct kvm_vcpu *vcpu, /* Checks for #GP/#SS exceptions. */ exn = false; - if (is_protmode(vcpu)) { + if (is_long_mode(vcpu)) { + /* Long mode: #GP(0)/#SS(0) if the memory address is in a + * non-canonical form. This is the only check on the memory + * destination for long mode! + */ + exn = is_noncanonical_address(*ret); + } else if (is_protmode(vcpu)) { /* Protected mode: apply checks for segment validity in the * following order: * - segment type check (#GP(0) may be thrown) @@ -6669,17 +6683,10 @@ static int get_vmx_mem_address(struct kvm_vcpu *vcpu, * execute-only code segment */ exn = ((s.type & 0xa) == 8); - } - if (exn) { - kvm_queue_exception_e(vcpu, GP_VECTOR, 0); - return 1; - } - if (is_long_mode(vcpu)) { - /* Long mode: #GP(0)/#SS(0) if the memory address is in a - * non-canonical form. This is an only check for long mode. - */ - exn = is_noncanonical_address(*ret); - } else if (is_protmode(vcpu)) { + if (exn) { + kvm_queue_exception_e(vcpu, GP_VECTOR, 0); + return 1; + } /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. */ exn = (s.unusable != 0); @@ -7420,6 +7427,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) if (!(types & (1UL << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + skip_emulated_instruction(vcpu); return 1; } @@ -7478,6 +7486,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) if (!(types & (1UL << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + skip_emulated_instruction(vcpu); return 1; } @@ -7494,12 +7503,17 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) } switch (type) { + case VMX_VPID_EXTENT_SINGLE_CONTEXT: + /* + * Old versions of KVM use the single-context version so we + * have to support it; just treat it the same as all-context. + */ case VMX_VPID_EXTENT_ALL_CONTEXT: __vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02); nested_vmx_succeed(vcpu); break; default: - /* Trap single context invalidation invvpid calls */ + /* Trap individual address invalidation invvpid calls */ BUG_ON(1); break; } @@ -7795,7 +7809,7 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu) switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: - if (!is_exception(intr_info)) + if (is_nmi(intr_info)) return false; else if (is_page_fault(intr_info)) return enable_ept; @@ -8200,6 +8214,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) if ((vectoring_info & VECTORING_INFO_VALID_MASK) && (exit_reason != EXIT_REASON_EXCEPTION_NMI && exit_reason != EXIT_REASON_EPT_VIOLATION && + exit_reason != EXIT_REASON_PML_FULL && exit_reason != EXIT_REASON_TASK_SWITCH)) { vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; @@ -8259,6 +8274,12 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set) { u32 sec_exec_control; + /* Postpone execution until vmcs01 is the current VMCS. */ + if (is_guest_mode(vcpu)) { + to_vmx(vcpu)->nested.change_vmcs01_virtual_x2apic_mode = true; + return; + } + /* * There is not point to enable virtualize x2apic without enable * apicv @@ -8397,8 +8418,7 @@ static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx) kvm_machine_check(); /* We need to handle NMIs before interrupts are enabled */ - if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR && - (exit_intr_info & INTR_INFO_VALID_MASK)) { + if (is_nmi(exit_intr_info)) { kvm_before_handle_nmi(&vmx->vcpu); asm("int $2"); kvm_after_handle_nmi(&vmx->vcpu); @@ -8834,6 +8854,22 @@ static void vmx_load_vmcs01(struct kvm_vcpu *vcpu) put_cpu(); } +/* + * Ensure that the current vmcs of the logical processor is the + * vmcs01 of the vcpu before calling free_nested(). + */ +static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + int r; + + r = vcpu_load(vcpu); + BUG_ON(r); + vmx_load_vmcs01(vcpu); + free_nested(vmx); + vcpu_put(vcpu); +} + static void vmx_free_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -8842,8 +8878,7 @@ static void vmx_free_vcpu(struct kvm_vcpu *vcpu) vmx_destroy_pml_buffer(vmx); free_vpid(vmx->vpid); leave_guest_mode(vcpu); - vmx_load_vmcs01(vcpu); - free_nested(vmx); + vmx_free_vcpu_nested(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); kfree(vmx->guest_msrs); kvm_vcpu_uninit(vcpu); @@ -10576,6 +10611,12 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_VMX_PREEMPTION_TIMER); + if (vmx->nested.change_vmcs01_virtual_x2apic_mode) { + vmx->nested.change_vmcs01_virtual_x2apic_mode = false; + vmx_set_virtual_x2apic_mode(vcpu, + vcpu->arch.apic_base & X2APIC_ENABLE); + } + /* This is needed for same reason as it was needed in prepare_vmcs02 */ vmx->host_rsp = 0; diff --git a/kernel/arch/x86/kvm/x86.c b/kernel/arch/x86/kvm/x86.c index c7695cea4..c265bb0c2 100644 --- a/kernel/arch/x86/kvm/x86.c +++ b/kernel/arch/x86/kvm/x86.c @@ -200,7 +200,18 @@ static void kvm_on_user_return(struct user_return_notifier *urn) struct kvm_shared_msrs *locals = container_of(urn, struct kvm_shared_msrs, urn); struct kvm_shared_msr_values *values; + unsigned long flags; + /* + * Disabling irqs at this point since the following code could be + * interrupted and executed through kvm_arch_hardware_disable() + */ + local_irq_save(flags); + if (locals->registered) { + locals->registered = false; + user_return_notifier_unregister(urn); + } + local_irq_restore(flags); for (slot = 0; slot < shared_msrs_global.nr; ++slot) { values = &locals->values[slot]; if (values->host != values->curr) { @@ -208,8 +219,6 @@ static void kvm_on_user_return(struct user_return_notifier *urn) values->curr = values->host; } } - locals->registered = false; - user_return_notifier_unregister(urn); } static void shared_msr_update(unsigned slot, u32 msr) @@ -698,7 +707,6 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) return 1; } - kvm_put_guest_xcr0(vcpu); vcpu->arch.xcr0 = xcr0; if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) @@ -2947,6 +2955,8 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, memset(&events->reserved, 0, sizeof(events->reserved)); } +static void kvm_set_hflags(struct kvm_vcpu *vcpu, unsigned emul_flags); + static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) { @@ -2979,10 +2989,13 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, vcpu->arch.apic->sipi_vector = events->sipi_vector; if (events->flags & KVM_VCPUEVENT_VALID_SMM) { + u32 hflags = vcpu->arch.hflags; if (events->smi.smm) - vcpu->arch.hflags |= HF_SMM_MASK; + hflags |= HF_SMM_MASK; else - vcpu->arch.hflags &= ~HF_SMM_MASK; + hflags &= ~HF_SMM_MASK; + kvm_set_hflags(vcpu, hflags); + vcpu->arch.smi_pending = events->smi.pending; if (events->smi.smm_inside_nmi) vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; @@ -3020,6 +3033,11 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, if (dbgregs->flags) return -EINVAL; + if (dbgregs->dr6 & ~0xffffffffull) + return -EINVAL; + if (dbgregs->dr7 & ~0xffffffffull) + return -EINVAL; + memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); kvm_update_dr0123(vcpu); vcpu->arch.dr6 = dbgregs->dr6; @@ -3045,6 +3063,7 @@ static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) memcpy(dest, xsave, XSAVE_HDR_OFFSET); /* Set XSTATE_BV */ + xstate_bv &= vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FPSSE; *(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv; /* @@ -3319,6 +3338,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, }; case KVM_SET_VAPIC_ADDR: { struct kvm_vapic_addr va; + int idx; r = -EINVAL; if (!lapic_in_kernel(vcpu)) @@ -3326,7 +3346,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&va, argp, sizeof va)) goto out; + idx = srcu_read_lock(&vcpu->kvm->srcu); r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); + srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } case KVM_X86_SETUP_MCE: { @@ -5834,6 +5856,7 @@ out: void kvm_arch_exit(void) { + kvm_lapic_exit(); perf_unregister_guest_info_callbacks(&kvm_guest_cbs); if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) @@ -6036,12 +6059,10 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) } /* try to inject new event if pending */ - if (vcpu->arch.nmi_pending) { - if (kvm_x86_ops->nmi_allowed(vcpu)) { - --vcpu->arch.nmi_pending; - vcpu->arch.nmi_injected = true; - kvm_x86_ops->set_nmi(vcpu); - } + if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) { + --vcpu->arch.nmi_pending; + vcpu->arch.nmi_injected = true; + kvm_x86_ops->set_nmi(vcpu); } else if (kvm_cpu_has_injectable_intr(vcpu)) { /* * Because interrupts can be injected asynchronously, we are @@ -6486,10 +6507,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (inject_pending_event(vcpu, req_int_win) != 0) req_immediate_exit = true; /* enable NMI/IRQ window open exits if needed */ - else if (vcpu->arch.nmi_pending) - kvm_x86_ops->enable_nmi_window(vcpu); - else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) - kvm_x86_ops->enable_irq_window(vcpu); + else { + if (vcpu->arch.nmi_pending) + kvm_x86_ops->enable_nmi_window(vcpu); + if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) + kvm_x86_ops->enable_irq_window(vcpu); + } if (kvm_lapic_enabled(vcpu)) { update_cr8_intercept(vcpu); @@ -6507,8 +6530,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_ops->prepare_guest_switch(vcpu); if (vcpu->fpu_active) kvm_load_guest_fpu(vcpu); - kvm_load_guest_xcr0(vcpu); - vcpu->mode = IN_GUEST_MODE; srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); @@ -6531,6 +6552,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto cancel_injection; } + kvm_load_guest_xcr0(vcpu); + if (req_immediate_exit) smp_send_reschedule(vcpu->cpu); @@ -6580,6 +6603,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->mode = OUTSIDE_GUEST_MODE; smp_wmb(); + kvm_put_guest_xcr0(vcpu); + /* Interrupt is enabled by handle_external_intr() */ kvm_x86_ops->handle_external_intr(vcpu); @@ -7227,7 +7252,6 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) * and assume host would use all available bits. * Guest xcr0 would be loaded later. */ - kvm_put_guest_xcr0(vcpu); vcpu->guest_fpu_loaded = 1; __kernel_fpu_begin(); __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu.state); @@ -7236,8 +7260,6 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) { - kvm_put_guest_xcr0(vcpu); - if (!vcpu->guest_fpu_loaded) { vcpu->fpu_counter = 0; return; @@ -7262,10 +7284,12 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) { + void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask; + kvmclock_reset(vcpu); - free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); kvm_x86_ops->vcpu_free(vcpu); + free_cpumask_var(wbinvd_dirty_mask); } struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, diff --git a/kernel/arch/x86/mm/kmmio.c b/kernel/arch/x86/mm/kmmio.c index 637ab34ed..ddb2244b0 100644 --- a/kernel/arch/x86/mm/kmmio.c +++ b/kernel/arch/x86/mm/kmmio.c @@ -33,7 +33,7 @@ struct kmmio_fault_page { struct list_head list; struct kmmio_fault_page *release_next; - unsigned long page; /* location of the fault page */ + unsigned long addr; /* the requested address */ pteval_t old_presence; /* page presence prior to arming */ bool armed; @@ -70,9 +70,16 @@ unsigned int kmmio_count; static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE]; static LIST_HEAD(kmmio_probes); -static struct list_head *kmmio_page_list(unsigned long page) +static struct list_head *kmmio_page_list(unsigned long addr) { - return &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)]; + unsigned int l; + pte_t *pte = lookup_address(addr, &l); + + if (!pte) + return NULL; + addr &= page_level_mask(l); + + return &kmmio_page_table[hash_long(addr, KMMIO_PAGE_HASH_BITS)]; } /* Accessed per-cpu */ @@ -98,15 +105,19 @@ static struct kmmio_probe *get_kmmio_probe(unsigned long addr) } /* You must be holding RCU read lock. */ -static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page) +static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long addr) { struct list_head *head; struct kmmio_fault_page *f; + unsigned int l; + pte_t *pte = lookup_address(addr, &l); - page &= PAGE_MASK; - head = kmmio_page_list(page); + if (!pte) + return NULL; + addr &= page_level_mask(l); + head = kmmio_page_list(addr); list_for_each_entry_rcu(f, head, list) { - if (f->page == page) + if (f->addr == addr) return f; } return NULL; @@ -137,10 +148,10 @@ static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old) static int clear_page_presence(struct kmmio_fault_page *f, bool clear) { unsigned int level; - pte_t *pte = lookup_address(f->page, &level); + pte_t *pte = lookup_address(f->addr, &level); if (!pte) { - pr_err("no pte for page 0x%08lx\n", f->page); + pr_err("no pte for addr 0x%08lx\n", f->addr); return -1; } @@ -156,7 +167,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear) return -1; } - __flush_tlb_one(f->page); + __flush_tlb_one(f->addr); return 0; } @@ -176,12 +187,12 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f) int ret; WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n")); if (f->armed) { - pr_warning("double-arm: page 0x%08lx, ref %d, old %d\n", - f->page, f->count, !!f->old_presence); + pr_warning("double-arm: addr 0x%08lx, ref %d, old %d\n", + f->addr, f->count, !!f->old_presence); } ret = clear_page_presence(f, true); - WARN_ONCE(ret < 0, KERN_ERR pr_fmt("arming 0x%08lx failed.\n"), - f->page); + WARN_ONCE(ret < 0, KERN_ERR pr_fmt("arming at 0x%08lx failed.\n"), + f->addr); f->armed = true; return ret; } @@ -191,7 +202,7 @@ static void disarm_kmmio_fault_page(struct kmmio_fault_page *f) { int ret = clear_page_presence(f, false); WARN_ONCE(ret < 0, - KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page); + KERN_ERR "kmmio disarming at 0x%08lx failed.\n", f->addr); f->armed = false; } @@ -215,6 +226,12 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) struct kmmio_context *ctx; struct kmmio_fault_page *faultpage; int ret = 0; /* default to fault not handled */ + unsigned long page_base = addr; + unsigned int l; + pte_t *pte = lookup_address(addr, &l); + if (!pte) + return -EINVAL; + page_base &= page_level_mask(l); /* * Preemption is now disabled to prevent process switch during @@ -227,7 +244,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) preempt_disable(); rcu_read_lock(); - faultpage = get_kmmio_fault_page(addr); + faultpage = get_kmmio_fault_page(page_base); if (!faultpage) { /* * Either this page fault is not caused by kmmio, or @@ -239,7 +256,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ctx = &get_cpu_var(kmmio_ctx); if (ctx->active) { - if (addr == ctx->addr) { + if (page_base == ctx->addr) { /* * A second fault on the same page means some other * condition needs handling by do_page_fault(), the @@ -267,9 +284,9 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ctx->active++; ctx->fpage = faultpage; - ctx->probe = get_kmmio_probe(addr); + ctx->probe = get_kmmio_probe(page_base); ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF)); - ctx->addr = addr; + ctx->addr = page_base; if (ctx->probe && ctx->probe->pre_handler) ctx->probe->pre_handler(ctx->probe, regs, addr); @@ -354,12 +371,11 @@ out: } /* You must be holding kmmio_lock. */ -static int add_kmmio_fault_page(unsigned long page) +static int add_kmmio_fault_page(unsigned long addr) { struct kmmio_fault_page *f; - page &= PAGE_MASK; - f = get_kmmio_fault_page(page); + f = get_kmmio_fault_page(addr); if (f) { if (!f->count) arm_kmmio_fault_page(f); @@ -372,26 +388,25 @@ static int add_kmmio_fault_page(unsigned long page) return -1; f->count = 1; - f->page = page; + f->addr = addr; if (arm_kmmio_fault_page(f)) { kfree(f); return -1; } - list_add_rcu(&f->list, kmmio_page_list(f->page)); + list_add_rcu(&f->list, kmmio_page_list(f->addr)); return 0; } /* You must be holding kmmio_lock. */ -static void release_kmmio_fault_page(unsigned long page, +static void release_kmmio_fault_page(unsigned long addr, struct kmmio_fault_page **release_list) { struct kmmio_fault_page *f; - page &= PAGE_MASK; - f = get_kmmio_fault_page(page); + f = get_kmmio_fault_page(addr); if (!f) return; @@ -420,18 +435,27 @@ int register_kmmio_probe(struct kmmio_probe *p) int ret = 0; unsigned long size = 0; const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK); + unsigned int l; + pte_t *pte; spin_lock_irqsave(&kmmio_lock, flags); if (get_kmmio_probe(p->addr)) { ret = -EEXIST; goto out; } + + pte = lookup_address(p->addr, &l); + if (!pte) { + ret = -EINVAL; + goto out; + } + kmmio_count++; list_add_rcu(&p->list, &kmmio_probes); while (size < size_lim) { if (add_kmmio_fault_page(p->addr + size)) pr_err("Unable to set page fault.\n"); - size += PAGE_SIZE; + size += page_level_size(l); } out: spin_unlock_irqrestore(&kmmio_lock, flags); @@ -506,11 +530,17 @@ void unregister_kmmio_probe(struct kmmio_probe *p) const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK); struct kmmio_fault_page *release_list = NULL; struct kmmio_delayed_release *drelease; + unsigned int l; + pte_t *pte; + + pte = lookup_address(p->addr, &l); + if (!pte) + return; spin_lock_irqsave(&kmmio_lock, flags); while (size < size_lim) { release_kmmio_fault_page(p->addr + size, &release_list); - size += PAGE_SIZE; + size += page_level_size(l); } list_del_rcu(&p->list); kmmio_count--; diff --git a/kernel/arch/x86/mm/mmap.c b/kernel/arch/x86/mm/mmap.c index 844b06d67..307f60ecf 100644 --- a/kernel/arch/x86/mm/mmap.c +++ b/kernel/arch/x86/mm/mmap.c @@ -93,18 +93,6 @@ static unsigned long mmap_base(unsigned long rnd) return PAGE_ALIGN(TASK_SIZE - gap - rnd); } -/* - * Bottom-up (legacy) layout on X86_32 did not support randomization, X86_64 - * does, but not when emulating X86_32 - */ -static unsigned long mmap_legacy_base(unsigned long rnd) -{ - if (mmap_is_ia32()) - return TASK_UNMAPPED_BASE; - else - return TASK_UNMAPPED_BASE + rnd; -} - /* * This function, called very early during the creation of a new * process VM image, sets up which VM layout function to use: @@ -116,7 +104,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm) if (current->flags & PF_RANDOMIZE) random_factor = arch_mmap_rnd(); - mm->mmap_legacy_base = mmap_legacy_base(random_factor); + mm->mmap_legacy_base = TASK_UNMAPPED_BASE + random_factor; if (mmap_is_legacy()) { mm->mmap_base = mm->mmap_legacy_base; diff --git a/kernel/arch/x86/mm/pat.c b/kernel/arch/x86/mm/pat.c index 188e3e07e..6ad687d10 100644 --- a/kernel/arch/x86/mm/pat.c +++ b/kernel/arch/x86/mm/pat.c @@ -39,11 +39,22 @@ static bool boot_cpu_done; static int __read_mostly __pat_enabled = IS_ENABLED(CONFIG_X86_PAT); +static void init_cache_modes(void); -static inline void pat_disable(const char *reason) +void pat_disable(const char *reason) { + if (!__pat_enabled) + return; + + if (boot_cpu_done) { + WARN_ONCE(1, "x86/PAT: PAT cannot be disabled after initialization\n"); + return; + } + __pat_enabled = 0; pr_info("x86/PAT: %s\n", reason); + + init_cache_modes(); } static int __init nopat(char *str) @@ -180,7 +191,7 @@ static enum page_cache_mode pat_get_cache_mode(unsigned pat_val, char *msg) * configuration. * Using lower indices is preferred, so we start with highest index. */ -void pat_init_cache_modes(u64 pat) +static void __init_cache_modes(u64 pat) { enum page_cache_mode cache; char pat_msg[33]; @@ -201,14 +212,11 @@ static void pat_bsp_init(u64 pat) { u64 tmp_pat; - if (!cpu_has_pat) { + if (!boot_cpu_has(X86_FEATURE_PAT)) { pat_disable("PAT not supported by CPU."); return; } - if (!pat_enabled()) - goto done; - rdmsrl(MSR_IA32_CR_PAT, tmp_pat); if (!tmp_pat) { pat_disable("PAT MSR is 0, disabled."); @@ -217,16 +225,12 @@ static void pat_bsp_init(u64 pat) wrmsrl(MSR_IA32_CR_PAT, pat); -done: - pat_init_cache_modes(pat); + __init_cache_modes(pat); } static void pat_ap_init(u64 pat) { - if (!pat_enabled()) - return; - - if (!cpu_has_pat) { + if (!boot_cpu_has(X86_FEATURE_PAT)) { /* * If this happens we are on a secondary CPU, but switched to * PAT on the boot CPU. We have no way to undo PAT. @@ -237,18 +241,32 @@ static void pat_ap_init(u64 pat) wrmsrl(MSR_IA32_CR_PAT, pat); } -void pat_init(void) +static void init_cache_modes(void) { - u64 pat; - struct cpuinfo_x86 *c = &boot_cpu_data; + u64 pat = 0; + static int init_cm_done; - if (!pat_enabled()) { + if (init_cm_done) + return; + + if (boot_cpu_has(X86_FEATURE_PAT)) { + /* + * CPU supports PAT. Set PAT table to be consistent with + * PAT MSR. This case supports "nopat" boot option, and + * virtual machine environments which support PAT without + * MTRRs. In specific, Xen has unique setup to PAT MSR. + * + * If PAT MSR returns 0, it is considered invalid and emulates + * as No PAT. + */ + rdmsrl(MSR_IA32_CR_PAT, pat); + } + + if (!pat) { /* * No PAT. Emulate the PAT table that corresponds to the two - * cache bits, PWT (Write Through) and PCD (Cache Disable). This - * setup is the same as the BIOS default setup when the system - * has PAT but the "nopat" boot option has been specified. This - * emulated PAT table is used when MSR_IA32_CR_PAT returns 0. + * cache bits, PWT (Write Through) and PCD (Cache Disable). + * This setup is also the same as the BIOS default setup. * * PTE encoding: * @@ -265,10 +283,36 @@ void pat_init(void) */ pat = PAT(0, WB) | PAT(1, WT) | PAT(2, UC_MINUS) | PAT(3, UC) | PAT(4, WB) | PAT(5, WT) | PAT(6, UC_MINUS) | PAT(7, UC); + } + + __init_cache_modes(pat); + + init_cm_done = 1; +} + +/** + * pat_init - Initialize PAT MSR and PAT table + * + * This function initializes PAT MSR and PAT table with an OS-defined value + * to enable additional cache attributes, WC and WT. + * + * This function must be called on all CPUs using the specific sequence of + * operations defined in Intel SDM. mtrr_rendezvous_handler() provides this + * procedure for PAT. + */ +void pat_init(void) +{ + u64 pat; + struct cpuinfo_x86 *c = &boot_cpu_data; + + if (!pat_enabled()) { + init_cache_modes(); + return; + } - } else if ((c->x86_vendor == X86_VENDOR_INTEL) && - (((c->x86 == 0x6) && (c->x86_model <= 0xd)) || - ((c->x86 == 0xf) && (c->x86_model <= 0x6)))) { + if ((c->x86_vendor == X86_VENDOR_INTEL) && + (((c->x86 == 0x6) && (c->x86_model <= 0xd)) || + ((c->x86 == 0xf) && (c->x86_model <= 0x6)))) { /* * PAT support with the lower four entries. Intel Pentium 2, * 3, M, and 4 are affected by PAT errata, which makes the @@ -733,25 +777,6 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, if (file->f_flags & O_DSYNC) pcm = _PAGE_CACHE_MODE_UC_MINUS; -#ifdef CONFIG_X86_32 - /* - * On the PPro and successors, the MTRRs are used to set - * memory types for physical addresses outside main memory, - * so blindly setting UC or PWT on those pages is wrong. - * For Pentiums and earlier, the surround logic should disable - * caching for the high addresses through the KEN pin, but - * we maintain the tradition of paranoia in this code. - */ - if (!pat_enabled() && - !(boot_cpu_has(X86_FEATURE_MTRR) || - boot_cpu_has(X86_FEATURE_K6_MTRR) || - boot_cpu_has(X86_FEATURE_CYRIX_ARR) || - boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) && - (pfn << PAGE_SHIFT) >= __pa(high_memory)) { - pcm = _PAGE_CACHE_MODE_UC; - } -#endif - *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) | cachemode2protval(pcm)); return 1; diff --git a/kernel/arch/x86/mm/tlb.c b/kernel/arch/x86/mm/tlb.c index 8f4cc3dfa..5fb6adaaa 100644 --- a/kernel/arch/x86/mm/tlb.c +++ b/kernel/arch/x86/mm/tlb.c @@ -106,8 +106,6 @@ static void flush_tlb_func(void *info) if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) return; - if (!f->flush_end) - f->flush_end = f->flush_start + PAGE_SIZE; count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { @@ -135,12 +133,20 @@ void native_flush_tlb_others(const struct cpumask *cpumask, unsigned long end) { struct flush_tlb_info info; + + if (end == 0) + end = start + PAGE_SIZE; info.flush_mm = mm; info.flush_start = start; info.flush_end = end; count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); - trace_tlb_flush(TLB_REMOTE_SEND_IPI, end - start); + if (end == TLB_FLUSH_ALL) + trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL); + else + trace_tlb_flush(TLB_REMOTE_SEND_IPI, + (end - start) >> PAGE_SHIFT); + if (is_uv_system()) { unsigned int cpu; diff --git a/kernel/arch/x86/pci/acpi.c b/kernel/arch/x86/pci/acpi.c index 3cd69832d..3961103e9 100644 --- a/kernel/arch/x86/pci/acpi.c +++ b/kernel/arch/x86/pci/acpi.c @@ -114,6 +114,16 @@ static const struct dmi_system_id pci_crs_quirks[] __initconst = { DMI_MATCH(DMI_BIOS_VERSION, "6JET85WW (1.43 )"), }, }, + /* https://bugzilla.kernel.org/show_bug.cgi?id=42606 */ + { + .callback = set_nouse_crs, + .ident = "Supermicro X8DTH", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Supermicro"), + DMI_MATCH(DMI_PRODUCT_NAME, "X8DTH-i/6/iF/6F"), + DMI_MATCH(DMI_BIOS_VERSION, "2.0a"), + }, + }, /* https://bugzilla.kernel.org/show_bug.cgi?id=15362 */ { diff --git a/kernel/arch/x86/pci/common.c b/kernel/arch/x86/pci/common.c index eccd4d99e..8fd6f44ae 100644 --- a/kernel/arch/x86/pci/common.c +++ b/kernel/arch/x86/pci/common.c @@ -673,28 +673,22 @@ int pcibios_add_device(struct pci_dev *dev) return 0; } -int pcibios_alloc_irq(struct pci_dev *dev) +int pcibios_enable_device(struct pci_dev *dev, int mask) { - /* - * If the PCI device was already claimed by core code and has - * MSI enabled, probing of the pcibios IRQ will overwrite - * dev->irq. So bail out if MSI is already enabled. - */ - if (pci_dev_msi_enabled(dev)) - return -EBUSY; + int err; - return pcibios_enable_irq(dev); -} + if ((err = pci_enable_resources(dev, mask)) < 0) + return err; -void pcibios_free_irq(struct pci_dev *dev) -{ - if (pcibios_disable_irq) - pcibios_disable_irq(dev); + if (!pci_dev_msi_enabled(dev)) + return pcibios_enable_irq(dev); + return 0; } -int pcibios_enable_device(struct pci_dev *dev, int mask) +void pcibios_disable_device (struct pci_dev *dev) { - return pci_enable_resources(dev, mask); + if (!pci_dev_msi_enabled(dev) && pcibios_disable_irq) + pcibios_disable_irq(dev); } int pci_ext_cfg_avail(void) diff --git a/kernel/arch/x86/pci/fixup.c b/kernel/arch/x86/pci/fixup.c index e58565556..0ae7e9fa3 100644 --- a/kernel/arch/x86/pci/fixup.c +++ b/kernel/arch/x86/pci/fixup.c @@ -540,3 +540,10 @@ static void twinhead_reserve_killing_zone(struct pci_dev *dev) } } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x27B9, twinhead_reserve_killing_zone); + +static void pci_bdwep_bar(struct pci_dev *dev) +{ + dev->non_compliant_bars = 1; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x6fa0, pci_bdwep_bar); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x6fc0, pci_bdwep_bar); diff --git a/kernel/arch/x86/pci/intel_mid_pci.c b/kernel/arch/x86/pci/intel_mid_pci.c index 0d24e7c10..ae97f24a4 100644 --- a/kernel/arch/x86/pci/intel_mid_pci.c +++ b/kernel/arch/x86/pci/intel_mid_pci.c @@ -37,6 +37,7 @@ /* Quirks for the listed devices */ #define PCI_DEVICE_ID_INTEL_MRFL_MMC 0x1190 +#define PCI_DEVICE_ID_INTEL_MRFL_HSU 0x1191 /* Fixed BAR fields */ #define PCIE_VNDR_CAP_ID_FIXED_BAR 0x00 /* Fixed BAR (TBD) */ @@ -215,7 +216,7 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) int polarity; int ret; - if (pci_has_managed_irq(dev)) + if (dev->irq_managed && dev->irq > 0) return 0; switch (intel_mid_identify_cpu()) { @@ -224,14 +225,21 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) /* Special treatment for IRQ0 */ if (dev->irq == 0) { + /* + * Skip HS UART common registers device since it has + * IRQ0 assigned and not used by the kernel. + */ + if (dev->device == PCI_DEVICE_ID_INTEL_MRFL_HSU) + return -EBUSY; /* * TNG has IRQ0 assigned to eMMC controller. But there * are also other devices with bogus PCI configuration * that have IRQ0 assigned. This check ensures that - * eMMC gets it. + * eMMC gets it. The rest of devices still could be + * enabled without interrupt line being allocated. */ if (dev->device != PCI_DEVICE_ID_INTEL_MRFL_MMC) - return -EBUSY; + return 0; } break; default: @@ -256,13 +264,10 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) static void intel_mid_pci_irq_disable(struct pci_dev *dev) { - if (pci_has_managed_irq(dev)) { + if (!mp_should_keep_irq(&dev->dev) && dev->irq_managed && + dev->irq > 0) { mp_unmap_irq(dev->irq); dev->irq_managed = 0; - /* - * Don't reset dev->irq here, otherwise - * intel_mid_pci_irq_enable() will fail on next call. - */ } } diff --git a/kernel/arch/x86/pci/irq.c b/kernel/arch/x86/pci/irq.c index 32e70343e..9bd115484 100644 --- a/kernel/arch/x86/pci/irq.c +++ b/kernel/arch/x86/pci/irq.c @@ -1202,7 +1202,7 @@ static int pirq_enable_irq(struct pci_dev *dev) struct pci_dev *temp_dev; int irq; - if (pci_has_managed_irq(dev)) + if (dev->irq_managed && dev->irq > 0) return 0; irq = IO_APIC_get_PCI_irq_vector(dev->bus->number, @@ -1230,7 +1230,8 @@ static int pirq_enable_irq(struct pci_dev *dev) } dev = temp_dev; if (irq >= 0) { - pci_set_managed_irq(dev, irq); + dev->irq_managed = 1; + dev->irq = irq; dev_info(&dev->dev, "PCI->APIC IRQ transform: " "INT %c -> IRQ %d\n", 'A' + pin - 1, irq); return 0; @@ -1256,10 +1257,24 @@ static int pirq_enable_irq(struct pci_dev *dev) return 0; } +bool mp_should_keep_irq(struct device *dev) +{ + if (dev->power.is_prepared) + return true; +#ifdef CONFIG_PM + if (dev->power.runtime_status == RPM_SUSPENDING) + return true; +#endif + + return false; +} + static void pirq_disable_irq(struct pci_dev *dev) { - if (io_apic_assign_pci_irqs && pci_has_managed_irq(dev)) { + if (io_apic_assign_pci_irqs && !mp_should_keep_irq(&dev->dev) && + dev->irq_managed && dev->irq) { mp_unmap_irq(dev->irq); - pci_reset_managed_irq(dev); + dev->irq = 0; + dev->irq_managed = 0; } } diff --git a/kernel/arch/x86/pci/xen.c b/kernel/arch/x86/pci/xen.c index ff31ab464..c6d6efed3 100644 --- a/kernel/arch/x86/pci/xen.c +++ b/kernel/arch/x86/pci/xen.c @@ -488,8 +488,11 @@ int __init pci_xen_initial_domain(void) #endif __acpi_register_gsi = acpi_register_gsi_xen; __acpi_unregister_gsi = NULL; - /* Pre-allocate legacy irqs */ - for (irq = 0; irq < nr_legacy_irqs(); irq++) { + /* + * Pre-allocate the legacy IRQs. Use NR_LEGACY_IRQS here + * because we don't have a PIC and thus nr_legacy_irqs() is zero. + */ + for (irq = 0; irq < NR_IRQS_LEGACY; irq++) { int trigger, polarity; if (acpi_get_override_irq(irq, &trigger, &polarity) == -1) diff --git a/kernel/arch/x86/purgatory/Makefile b/kernel/arch/x86/purgatory/Makefile index 2c835e356..d445c5f1a 100644 --- a/kernel/arch/x86/purgatory/Makefile +++ b/kernel/arch/x86/purgatory/Makefile @@ -12,6 +12,7 @@ targets += purgatory.ro KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes -fno-zero-initialized-in-bss -fno-builtin -ffreestanding -c -MD -Os -mcmodel=large KBUILD_CFLAGS += -m$(BITS) +KBUILD_CFLAGS += $(call cc-option,-fno-PIE) $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld) diff --git a/kernel/arch/x86/xen/enlighten.c b/kernel/arch/x86/xen/enlighten.c index b7de78bdc..ffa41591b 100644 --- a/kernel/arch/x86/xen/enlighten.c +++ b/kernel/arch/x86/xen/enlighten.c @@ -74,7 +74,6 @@ #include #include #include -#include #include #ifdef CONFIG_ACPI @@ -961,7 +960,7 @@ static void xen_load_sp0(struct tss_struct *tss, tss->x86_tss.sp0 = thread->sp0; } -static void xen_set_iopl_mask(unsigned mask) +void xen_set_iopl_mask(unsigned mask) { struct physdev_set_iopl set_iopl; @@ -1519,7 +1518,6 @@ asmlinkage __visible void __init xen_start_kernel(void) { struct physdev_set_iopl set_iopl; unsigned long initrd_start = 0; - u64 pat; int rc; if (!xen_start_info) @@ -1627,13 +1625,6 @@ asmlinkage __visible void __init xen_start_kernel(void) xen_start_info->nr_pages); xen_reserve_special_pages(); - /* - * Modify the cache mode translation tables to match Xen's PAT - * configuration. - */ - rdmsrl(MSR_IA32_CR_PAT, pat); - pat_init_cache_modes(pat); - /* keep using Xen gdt for now; no urgent need to change it */ #ifdef CONFIG_X86_32 diff --git a/kernel/arch/x86/xen/mmu.c b/kernel/arch/x86/xen/mmu.c index cb5e266a8..1e56ff583 100644 --- a/kernel/arch/x86/xen/mmu.c +++ b/kernel/arch/x86/xen/mmu.c @@ -1113,7 +1113,7 @@ static void __init xen_cleanhighmap(unsigned long vaddr, /* NOTE: The loop is more greedy than the cleanup_highmap variant. * We include the PMD passed in on _both_ boundaries. */ - for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE)); + for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PTRS_PER_PMD)); pmd++, vaddr += PMD_SIZE) { if (pmd_none(*pmd)) continue; diff --git a/kernel/arch/x86/xen/setup.c b/kernel/arch/x86/xen/setup.c index 7ab29518a..e34589145 100644 --- a/kernel/arch/x86/xen/setup.c +++ b/kernel/arch/x86/xen/setup.c @@ -393,6 +393,9 @@ static unsigned long __init xen_set_identity_and_remap_chunk( unsigned long i = 0; unsigned long n = end_pfn - start_pfn; + if (remap_pfn == 0) + remap_pfn = nr_pages; + while (i < n) { unsigned long cur_pfn = start_pfn + i; unsigned long left = n - i; @@ -438,17 +441,29 @@ static unsigned long __init xen_set_identity_and_remap_chunk( return remap_pfn; } -static void __init xen_set_identity_and_remap(unsigned long nr_pages) +static unsigned long __init xen_count_remap_pages( + unsigned long start_pfn, unsigned long end_pfn, unsigned long nr_pages, + unsigned long remap_pages) +{ + if (start_pfn >= nr_pages) + return remap_pages; + + return remap_pages + min(end_pfn, nr_pages) - start_pfn; +} + +static unsigned long __init xen_foreach_remap_area(unsigned long nr_pages, + unsigned long (*func)(unsigned long start_pfn, unsigned long end_pfn, + unsigned long nr_pages, unsigned long last_val)) { phys_addr_t start = 0; - unsigned long last_pfn = nr_pages; + unsigned long ret_val = 0; const struct e820entry *entry = xen_e820_map; int i; /* * Combine non-RAM regions and gaps until a RAM region (or the - * end of the map) is reached, then set the 1:1 map and - * remap the memory in those non-RAM regions. + * end of the map) is reached, then call the provided function + * to perform its duty on the non-RAM region. * * The combined non-RAM regions are rounded to a whole number * of pages so any partial pages are accessible via the 1:1 @@ -466,14 +481,13 @@ static void __init xen_set_identity_and_remap(unsigned long nr_pages) end_pfn = PFN_UP(entry->addr); if (start_pfn < end_pfn) - last_pfn = xen_set_identity_and_remap_chunk( - start_pfn, end_pfn, nr_pages, - last_pfn); + ret_val = func(start_pfn, end_pfn, nr_pages, + ret_val); start = end; } } - pr_info("Released %ld page(s)\n", xen_released_pages); + return ret_val; } /* @@ -596,35 +610,6 @@ static void __init xen_ignore_unusable(void) } } -static unsigned long __init xen_count_remap_pages(unsigned long max_pfn) -{ - unsigned long extra = 0; - unsigned long start_pfn, end_pfn; - const struct e820entry *entry = xen_e820_map; - int i; - - end_pfn = 0; - for (i = 0; i < xen_e820_map_entries; i++, entry++) { - start_pfn = PFN_DOWN(entry->addr); - /* Adjacent regions on non-page boundaries handling! */ - end_pfn = min(end_pfn, start_pfn); - - if (start_pfn >= max_pfn) - return extra + max_pfn - end_pfn; - - /* Add any holes in map to result. */ - extra += start_pfn - end_pfn; - - end_pfn = PFN_UP(entry->addr + entry->size); - end_pfn = min(end_pfn, max_pfn); - - if (entry->type != E820_RAM) - extra += end_pfn - start_pfn; - } - - return extra; -} - bool __init xen_is_e820_reserved(phys_addr_t start, phys_addr_t size) { struct e820entry *entry; @@ -804,7 +789,7 @@ char * __init xen_memory_setup(void) max_pages = xen_get_max_pages(); /* How many extra pages do we need due to remapping? */ - max_pages += xen_count_remap_pages(max_pfn); + max_pages += xen_foreach_remap_area(max_pfn, xen_count_remap_pages); if (max_pages > max_pfn) extra_pages += max_pages - max_pfn; @@ -922,7 +907,9 @@ char * __init xen_memory_setup(void) * Set identity map on non-RAM pages and prepare remapping the * underlying RAM. */ - xen_set_identity_and_remap(max_pfn); + xen_foreach_remap_area(max_pfn, xen_set_identity_and_remap_chunk); + + pr_info("Released %ld page(s)\n", xen_released_pages); return "Xen"; } diff --git a/kernel/arch/xtensa/kernel/head.S b/kernel/arch/xtensa/kernel/head.S index 9ed55649a..05e1df943 100644 --- a/kernel/arch/xtensa/kernel/head.S +++ b/kernel/arch/xtensa/kernel/head.S @@ -128,7 +128,7 @@ ENTRY(_startup) wsr a0, icountlevel .set _index, 0 - .rept XCHAL_NUM_DBREAK - 1 + .rept XCHAL_NUM_DBREAK wsr a0, SREG_DBREAKC + _index .set _index, _index + 1 .endr diff --git a/kernel/arch/xtensa/mm/cache.c b/kernel/arch/xtensa/mm/cache.c index d75aa1476..1a804a2f9 100644 --- a/kernel/arch/xtensa/mm/cache.c +++ b/kernel/arch/xtensa/mm/cache.c @@ -97,11 +97,11 @@ void clear_user_highpage(struct page *page, unsigned long vaddr) unsigned long paddr; void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr); - pagefault_disable(); + preempt_disable(); kmap_invalidate_coherent(page, vaddr); set_bit(PG_arch_1, &page->flags); clear_page_alias(kvaddr, paddr); - pagefault_enable(); + preempt_enable(); } void copy_user_highpage(struct page *dst, struct page *src, @@ -113,11 +113,11 @@ void copy_user_highpage(struct page *dst, struct page *src, void *src_vaddr = coherent_kvaddr(src, TLBTEMP_BASE_2, vaddr, &src_paddr); - pagefault_disable(); + preempt_disable(); kmap_invalidate_coherent(dst, vaddr); set_bit(PG_arch_1, &dst->flags); copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr); - pagefault_enable(); + preempt_enable(); } #endif /* DCACHE_WAY_SIZE > PAGE_SIZE */ diff --git a/kernel/arch/xtensa/platforms/iss/console.c b/kernel/arch/xtensa/platforms/iss/console.c index 70cb408bc..92d785fef 100644 --- a/kernel/arch/xtensa/platforms/iss/console.c +++ b/kernel/arch/xtensa/platforms/iss/console.c @@ -100,21 +100,23 @@ static void rs_poll(unsigned long priv) { struct tty_port *port = (struct tty_port *)priv; int i = 0; + int rd = 1; unsigned char c; spin_lock(&timer_lock); while (simc_poll(0)) { - simc_read(0, &c, 1); + rd = simc_read(0, &c, 1); + if (rd <= 0) + break; tty_insert_flip_char(port, c, TTY_NORMAL); i++; } if (i) tty_flip_buffer_push(port); - - - mod_timer(&serial_timer, jiffies + SERIAL_TIMER_VALUE); + if (rd) + mod_timer(&serial_timer, jiffies + SERIAL_TIMER_VALUE); spin_unlock(&timer_lock); } diff --git a/kernel/block/bio.c b/kernel/block/bio.c index d4d144363..46e2cc1d4 100644 --- a/kernel/block/bio.c +++ b/kernel/block/bio.c @@ -584,6 +584,8 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) bio->bi_rw = bio_src->bi_rw; bio->bi_iter = bio_src->bi_iter; bio->bi_io_vec = bio_src->bi_io_vec; + + bio_clone_blkcg_association(bio, bio_src); } EXPORT_SYMBOL(__bio_clone_fast); @@ -689,6 +691,8 @@ integrity_clone: } } + bio_clone_blkcg_association(bio, bio_src); + return bio; } EXPORT_SYMBOL(bio_clone_bioset); @@ -2014,6 +2018,17 @@ void bio_disassociate_task(struct bio *bio) } } +/** + * bio_clone_blkcg_association - clone blkcg association from src to dst bio + * @dst: destination bio + * @src: source bio + */ +void bio_clone_blkcg_association(struct bio *dst, struct bio *src) +{ + if (src->bi_css) + WARN_ON(bio_associate_blkcg(dst, src->bi_css)); +} + #endif /* CONFIG_BLK_CGROUP */ static void __init biovec_init_slabs(void) diff --git a/kernel/block/blk-cgroup.c b/kernel/block/blk-cgroup.c index 5a37188b5..9d359e05f 100644 --- a/kernel/block/blk-cgroup.c +++ b/kernel/block/blk-cgroup.c @@ -1331,10 +1331,8 @@ int blkcg_policy_register(struct blkcg_policy *pol) struct blkcg_policy_data *cpd; cpd = pol->cpd_alloc_fn(GFP_KERNEL); - if (!cpd) { - mutex_unlock(&blkcg_pol_mutex); + if (!cpd) goto err_free_cpds; - } blkcg->cpd[pol->plid] = cpd; cpd->blkcg = blkcg; diff --git a/kernel/block/blk-core.c b/kernel/block/blk-core.c index 2f7afb98e..52d2fe2fe 100644 --- a/kernel/block/blk-core.c +++ b/kernel/block/blk-core.c @@ -518,7 +518,9 @@ EXPORT_SYMBOL_GPL(blk_queue_bypass_end); void blk_set_queue_dying(struct request_queue *q) { - queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); + spin_lock_irq(q->queue_lock); + queue_flag_set(QUEUE_FLAG_DYING, q); + spin_unlock_irq(q->queue_lock); if (q->mq_ops) blk_mq_wake_waiters(q); @@ -2192,7 +2194,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) if (q->mq_ops) { if (blk_queue_io_stat(q)) blk_account_io_start(rq, true); - blk_mq_insert_request(rq, false, true, true); + blk_mq_insert_request(rq, false, true, false); return 0; } diff --git a/kernel/block/blk-map.c b/kernel/block/blk-map.c index f565e11f4..69953bd97 100644 --- a/kernel/block/blk-map.c +++ b/kernel/block/blk-map.c @@ -90,6 +90,9 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, if (!iter || !iter->count) return -EINVAL; + if (!iter_is_iovec(iter)) + return -EINVAL; + iov_for_each(iov, i, *iter) { unsigned long uaddr = (unsigned long) iov.iov_base; diff --git a/kernel/block/blk-merge.c b/kernel/block/blk-merge.c index b966db8f3..7225511cf 100644 --- a/kernel/block/blk-merge.c +++ b/kernel/block/blk-merge.c @@ -92,8 +92,30 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, bool do_split = true; struct bio *new = NULL; const unsigned max_sectors = get_max_io_size(q, bio); + unsigned bvecs = 0; bio_for_each_segment(bv, bio, iter) { + /* + * With arbitrary bio size, the incoming bio may be very + * big. We have to split the bio into small bios so that + * each holds at most BIO_MAX_PAGES bvecs because + * bio_clone() can fail to allocate big bvecs. + * + * It should have been better to apply the limit per + * request queue in which bio_clone() is involved, + * instead of globally. The biggest blocker is the + * bio_clone() in bio bounce. + * + * If bio is splitted by this reason, we should have + * allowed to continue bios merging, but don't do + * that now for making the change simple. + * + * TODO: deal with bio bounce's bio_clone() gracefully + * and convert the global limit into per-queue limit. + */ + if (bvecs++ >= BIO_MAX_PAGES) + goto split; + /* * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. diff --git a/kernel/block/blk-mq.c b/kernel/block/blk-mq.c index 7cdf19e4a..2e21e4c04 100644 --- a/kernel/block/blk-mq.c +++ b/kernel/block/blk-mq.c @@ -621,8 +621,10 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, * If a request wasn't started before the queue was * marked dying, kill it here or it'll go unnoticed. */ - if (unlikely(blk_queue_dying(rq->q))) - blk_mq_complete_request(rq, -EIO); + if (unlikely(blk_queue_dying(rq->q))) { + rq->errors = -EIO; + blk_mq_end_request(rq, rq->errors); + } return; } if (rq->cmd_flags & REQ_NO_TIMEOUT) @@ -798,7 +800,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) switch (ret) { case BLK_MQ_RQ_QUEUE_OK: queued++; - continue; + break; case BLK_MQ_RQ_QUEUE_BUSY: list_add(&rq->queuelist, &rq_list); __blk_mq_requeue_request(rq); @@ -860,7 +862,7 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) return WORK_CPU_UNBOUND; if (--hctx->next_cpu_batch <= 0) { - int cpu = hctx->next_cpu, next_cpu; + int next_cpu; next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); if (next_cpu >= nr_cpu_ids) @@ -868,8 +870,6 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) hctx->next_cpu = next_cpu; hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; - - return cpu; } return hctx->next_cpu; @@ -1331,9 +1331,9 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) blk_mq_put_ctx(data.ctx); if (!old_rq) goto done; - if (!blk_mq_direct_issue_request(old_rq, &cookie)) - goto done; - blk_mq_insert_request(old_rq, false, true, true); + if (test_bit(BLK_MQ_S_STOPPED, &data.hctx->state) || + blk_mq_direct_issue_request(old_rq, &cookie) != 0) + blk_mq_insert_request(old_rq, false, true, true); goto done; } diff --git a/kernel/block/bsg.c b/kernel/block/bsg.c index d214e929c..b9a53615b 100644 --- a/kernel/block/bsg.c +++ b/kernel/block/bsg.c @@ -655,6 +655,9 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) dprintk("%s: write %Zd bytes\n", bd->name, count); + if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + return -EINVAL; + bsg_set_block(bd, file); bytes_written = 0; diff --git a/kernel/block/cfq-iosched.c b/kernel/block/cfq-iosched.c index 1f9093e90..e04a7b849 100644 --- a/kernel/block/cfq-iosched.c +++ b/kernel/block/cfq-iosched.c @@ -1572,7 +1572,7 @@ static struct blkcg_policy_data *cfq_cpd_alloc(gfp_t gfp) { struct cfq_group_data *cgd; - cgd = kzalloc(sizeof(*cgd), GFP_KERNEL); + cgd = kzalloc(sizeof(*cgd), gfp); if (!cgd) return NULL; return &cgd->cpd; @@ -3003,7 +3003,6 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq) if (time_before(jiffies, rq->fifo_time)) rq = NULL; - cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); return rq; } @@ -3377,6 +3376,9 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) { unsigned int max_dispatch; + if (cfq_cfqq_must_dispatch(cfqq)) + return true; + /* * Drain async requests before we start sync IO */ @@ -3468,15 +3470,20 @@ static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq) BUG_ON(RB_EMPTY_ROOT(&cfqq->sort_list)); + rq = cfq_check_fifo(cfqq); + if (rq) + cfq_mark_cfqq_must_dispatch(cfqq); + if (!cfq_may_dispatch(cfqd, cfqq)) return false; /* * follow expired path, else get first next available */ - rq = cfq_check_fifo(cfqq); if (!rq) rq = cfqq->next_rq; + else + cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); /* * insert request into driver dispatch list @@ -3944,7 +3951,7 @@ cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq, * if the new request is sync, but the currently running queue is * not, let the sync request have priority. */ - if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq)) + if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq) && !cfq_cfqq_must_dispatch(cfqq)) return true; if (new_cfqq->cfqg != cfqq->cfqg) diff --git a/kernel/block/genhd.c b/kernel/block/genhd.c index e5cafa515..a5bed6bc8 100644 --- a/kernel/block/genhd.c +++ b/kernel/block/genhd.c @@ -612,7 +612,7 @@ void add_disk(struct gendisk *disk) /* Register BDI before referencing it from bdev */ bdi = &disk->queue->backing_dev_info; - bdi_register_dev(bdi, disk_devt(disk)); + bdi_register_owner(bdi, disk_to_dev(disk)); blk_register_region(disk_devt(disk), disk->minors, NULL, exact_match, exact_lock, disk); @@ -831,6 +831,7 @@ static void disk_seqf_stop(struct seq_file *seqf, void *v) if (iter) { class_dev_iter_exit(iter); kfree(iter); + seqf->private = NULL; } } diff --git a/kernel/block/ioprio.c b/kernel/block/ioprio.c index cc7800e9e..01b811629 100644 --- a/kernel/block/ioprio.c +++ b/kernel/block/ioprio.c @@ -150,8 +150,10 @@ static int get_task_ioprio(struct task_struct *p) if (ret) goto out; ret = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, IOPRIO_NORM); + task_lock(p); if (p->io_context) ret = p->io_context->ioprio; + task_unlock(p); out: return ret; } diff --git a/kernel/block/partition-generic.c b/kernel/block/partition-generic.c index 746935a59..a241e3900 100644 --- a/kernel/block/partition-generic.c +++ b/kernel/block/partition-generic.c @@ -349,15 +349,20 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, goto out_del; } + err = hd_ref_init(p); + if (err) { + if (flags & ADDPART_FLAG_WHOLEDISK) + goto out_remove_file; + goto out_del; + } + /* everything is up and running, commence */ rcu_assign_pointer(ptbl->part[partno], p); /* suppress uevent if the disk suppresses it */ if (!dev_get_uevent_suppress(ddev)) kobject_uevent(&pdev->kobj, KOBJ_ADD); - - if (!hd_ref_init(p)) - return p; + return p; out_free_info: free_part_info(p); @@ -366,6 +371,8 @@ out_free_stats: out_free: kfree(p); return ERR_PTR(err); +out_remove_file: + device_remove_file(pdev, &dev_attr_whole_disk); out_del: kobject_put(p->holder_dir); device_del(pdev); diff --git a/kernel/crypto/Makefile b/kernel/crypto/Makefile index f7aba9234..82fbff180 100644 --- a/kernel/crypto/Makefile +++ b/kernel/crypto/Makefile @@ -33,6 +33,7 @@ obj-$(CONFIG_CRYPTO_AKCIPHER2) += akcipher.o $(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h $(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h +$(obj)/rsa_helper.o: $(obj)/rsapubkey-asn1.h $(obj)/rsaprivkey-asn1.h clean-files += rsapubkey-asn1.c rsapubkey-asn1.h clean-files += rsaprivkey-asn1.c rsaprivkey-asn1.h diff --git a/kernel/crypto/ahash.c b/kernel/crypto/ahash.c index d19b52324..dac1c24e9 100644 --- a/kernel/crypto/ahash.c +++ b/kernel/crypto/ahash.c @@ -69,8 +69,9 @@ static int hash_walk_new_entry(struct crypto_hash_walk *walk) struct scatterlist *sg; sg = walk->sg; - walk->pg = sg_page(sg); walk->offset = sg->offset; + walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); + walk->offset = offset_in_page(walk->offset); walk->entrylen = sg->length; if (walk->entrylen > walk->total) diff --git a/kernel/crypto/algapi.c b/kernel/crypto/algapi.c index f98e79c8c..b54fd9187 100644 --- a/kernel/crypto/algapi.c +++ b/kernel/crypto/algapi.c @@ -357,6 +357,7 @@ int crypto_register_alg(struct crypto_alg *alg) struct crypto_larval *larval; int err; + alg->cra_flags &= ~CRYPTO_ALG_DEAD; err = crypto_check_alg(alg); if (err) return err; diff --git a/kernel/crypto/asymmetric_keys/Kconfig b/kernel/crypto/asymmetric_keys/Kconfig index 4870f2840..05bfe568c 100644 --- a/kernel/crypto/asymmetric_keys/Kconfig +++ b/kernel/crypto/asymmetric_keys/Kconfig @@ -14,6 +14,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE select MPILIB select PUBLIC_KEY_ALGO_RSA select CRYPTO_HASH_INFO + select CRYPTO_AKCIPHER help This option provides support for asymmetric public key type handling. If signature generation and/or verification are to be used, diff --git a/kernel/crypto/asymmetric_keys/pkcs7_parser.c b/kernel/crypto/asymmetric_keys/pkcs7_parser.c index 758acabf2..8f3056cd0 100644 --- a/kernel/crypto/asymmetric_keys/pkcs7_parser.c +++ b/kernel/crypto/asymmetric_keys/pkcs7_parser.c @@ -547,9 +547,7 @@ int pkcs7_sig_note_set_of_authattrs(void *context, size_t hdrlen, struct pkcs7_signed_info *sinfo = ctx->sinfo; if (!test_bit(sinfo_has_content_type, &sinfo->aa_set) || - !test_bit(sinfo_has_message_digest, &sinfo->aa_set) || - (ctx->msg->data_type == OID_msIndirectData && - !test_bit(sinfo_has_ms_opus_info, &sinfo->aa_set))) { + !test_bit(sinfo_has_message_digest, &sinfo->aa_set)) { pr_warn("Missing required AuthAttr\n"); return -EBADMSG; } diff --git a/kernel/crypto/asymmetric_keys/pkcs7_trust.c b/kernel/crypto/asymmetric_keys/pkcs7_trust.c index 90d6d4796..ecdb5a2ce 100644 --- a/kernel/crypto/asymmetric_keys/pkcs7_trust.c +++ b/kernel/crypto/asymmetric_keys/pkcs7_trust.c @@ -178,6 +178,8 @@ int pkcs7_validate_trust(struct pkcs7_message *pkcs7, int cached_ret = -ENOKEY; int ret; + *_trusted = false; + for (p = pkcs7->certs; p; p = p->next) p->seen = false; diff --git a/kernel/crypto/asymmetric_keys/x509_cert_parser.c b/kernel/crypto/asymmetric_keys/x509_cert_parser.c index 021d39c0b..13c4e5a5f 100644 --- a/kernel/crypto/asymmetric_keys/x509_cert_parser.c +++ b/kernel/crypto/asymmetric_keys/x509_cert_parser.c @@ -494,7 +494,7 @@ int x509_decode_time(time64_t *_t, size_t hdrlen, unsigned char tag, const unsigned char *value, size_t vlen) { - static const unsigned char month_lengths[] = { 31, 29, 31, 30, 31, 30, + static const unsigned char month_lengths[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; const unsigned char *p = value; unsigned year, mon, day, hour, min, sec, mon_len; @@ -540,9 +540,9 @@ int x509_decode_time(time64_t *_t, size_t hdrlen, if (year % 4 == 0) { mon_len = 29; if (year % 100 == 0) { - year /= 100; - if (year % 4 != 0) - mon_len = 28; + mon_len = 28; + if (year % 400 == 0) + mon_len = 29; } } } diff --git a/kernel/crypto/async_tx/async_pq.c b/kernel/crypto/async_tx/async_pq.c index c0748bbd4..84f8d4d8b 100644 --- a/kernel/crypto/async_tx/async_pq.c +++ b/kernel/crypto/async_tx/async_pq.c @@ -368,8 +368,6 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, dma_set_unmap(tx, unmap); async_tx_submit(chan, tx, submit); - - return tx; } else { struct page *p_src = P(blocks, disks); struct page *q_src = Q(blocks, disks); @@ -424,9 +422,11 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, submit->cb_param = cb_param_orig; submit->flags = flags_orig; async_tx_sync_epilog(submit); - - return NULL; + tx = NULL; } + dmaengine_unmap_put(unmap); + + return tx; } EXPORT_SYMBOL_GPL(async_syndrome_val); diff --git a/kernel/crypto/blkcipher.c b/kernel/crypto/blkcipher.c index 8cc1622b2..dca7bc87d 100644 --- a/kernel/crypto/blkcipher.c +++ b/kernel/crypto/blkcipher.c @@ -234,6 +234,8 @@ static int blkcipher_walk_next(struct blkcipher_desc *desc, return blkcipher_walk_done(desc, walk, -EINVAL); } + bsize = min(walk->walk_blocksize, n); + walk->flags &= ~(BLKCIPHER_WALK_SLOW | BLKCIPHER_WALK_COPY | BLKCIPHER_WALK_DIFF); if (!scatterwalk_aligned(&walk->in, walk->alignmask) || @@ -246,7 +248,6 @@ static int blkcipher_walk_next(struct blkcipher_desc *desc, } } - bsize = min(walk->walk_blocksize, n); n = scatterwalk_clamp(&walk->in, n); n = scatterwalk_clamp(&walk->out, n); diff --git a/kernel/crypto/cryptd.c b/kernel/crypto/cryptd.c index c81861b13..e7aa904cb 100644 --- a/kernel/crypto/cryptd.c +++ b/kernel/crypto/cryptd.c @@ -594,9 +594,14 @@ static int cryptd_hash_export(struct ahash_request *req, void *out) static int cryptd_hash_import(struct ahash_request *req, const void *in) { - struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct shash_desc *desc = cryptd_shash_desc(req); + + desc->tfm = ctx->child; + desc->flags = req->base.flags; - return crypto_shash_import(&rctx->desc, in); + return crypto_shash_import(desc, in); } static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, diff --git a/kernel/crypto/crypto_user.c b/kernel/crypto/crypto_user.c index 43fe85f20..7097a3395 100644 --- a/kernel/crypto/crypto_user.c +++ b/kernel/crypto/crypto_user.c @@ -455,6 +455,7 @@ static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = { [CRYPTO_MSG_NEWALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_DELALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_UPDATEALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), + [CRYPTO_MSG_GETALG - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg), [CRYPTO_MSG_DELRNG - CRYPTO_MSG_BASE] = 0, }; diff --git a/kernel/crypto/echainiv.c b/kernel/crypto/echainiv.c index b96a84560..343a74e96 100644 --- a/kernel/crypto/echainiv.c +++ b/kernel/crypto/echainiv.c @@ -1,8 +1,8 @@ /* * echainiv: Encrypted Chain IV Generator * - * This generator generates an IV based on a sequence number by xoring it - * with a salt and then encrypting it with the same key as used to encrypt + * This generator generates an IV based on a sequence number by multiplying + * it with a salt and then encrypting it with the same key as used to encrypt * the plain text. This algorithm requires that the block size be equal * to the IV size. It is mainly useful for CBC. * @@ -23,81 +23,17 @@ #include #include #include -#include #include -#include -#include +#include #include -#define MAX_IV_SIZE 16 - -static DEFINE_PER_CPU(u32 [MAX_IV_SIZE / sizeof(u32)], echainiv_iv); - -/* We don't care if we get preempted and read/write IVs from the next CPU. */ -static void echainiv_read_iv(u8 *dst, unsigned size) -{ - u32 *a = (u32 *)dst; - u32 __percpu *b = echainiv_iv; - - for (; size >= 4; size -= 4) { - *a++ = this_cpu_read(*b); - b++; - } -} - -static void echainiv_write_iv(const u8 *src, unsigned size) -{ - const u32 *a = (const u32 *)src; - u32 __percpu *b = echainiv_iv; - - for (; size >= 4; size -= 4) { - this_cpu_write(*b, *a); - a++; - b++; - } -} - -static void echainiv_encrypt_complete2(struct aead_request *req, int err) -{ - struct aead_request *subreq = aead_request_ctx(req); - struct crypto_aead *geniv; - unsigned int ivsize; - - if (err == -EINPROGRESS) - return; - - if (err) - goto out; - - geniv = crypto_aead_reqtfm(req); - ivsize = crypto_aead_ivsize(geniv); - - echainiv_write_iv(subreq->iv, ivsize); - - if (req->iv != subreq->iv) - memcpy(req->iv, subreq->iv, ivsize); - -out: - if (req->iv != subreq->iv) - kzfree(subreq->iv); -} - -static void echainiv_encrypt_complete(struct crypto_async_request *base, - int err) -{ - struct aead_request *req = base->data; - - echainiv_encrypt_complete2(req, err); - aead_request_complete(req, err); -} - static int echainiv_encrypt(struct aead_request *req) { struct crypto_aead *geniv = crypto_aead_reqtfm(req); struct aead_geniv_ctx *ctx = crypto_aead_ctx(geniv); struct aead_request *subreq = aead_request_ctx(req); - crypto_completion_t compl; - void *data; + __be64 nseqno; + u64 seqno; u8 *info; unsigned int ivsize = crypto_aead_ivsize(geniv); int err; @@ -107,8 +43,6 @@ static int echainiv_encrypt(struct aead_request *req) aead_request_set_tfm(subreq, ctx->child); - compl = echainiv_encrypt_complete; - data = req; info = req->iv; if (req->src != req->dst) { @@ -123,29 +57,30 @@ static int echainiv_encrypt(struct aead_request *req) return err; } - if (unlikely(!IS_ALIGNED((unsigned long)info, - crypto_aead_alignmask(geniv) + 1))) { - info = kmalloc(ivsize, req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL: - GFP_ATOMIC); - if (!info) - return -ENOMEM; - - memcpy(info, req->iv, ivsize); - } - - aead_request_set_callback(subreq, req->base.flags, compl, data); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); aead_request_set_crypt(subreq, req->dst, req->dst, req->cryptlen, info); aead_request_set_ad(subreq, req->assoclen); - crypto_xor(info, ctx->salt, ivsize); + memcpy(&nseqno, info + ivsize - 8, 8); + seqno = be64_to_cpu(nseqno); + memset(info, 0, ivsize); + scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1); - echainiv_read_iv(info, ivsize); - err = crypto_aead_encrypt(subreq); - echainiv_encrypt_complete2(req, err); - return err; + do { + u64 a; + + memcpy(&a, ctx->salt + ivsize - 8, 8); + + a |= 1; + a *= seqno; + + memcpy(info + ivsize - 8, &a, 8); + } while ((ivsize -= 8)); + + return crypto_aead_encrypt(subreq); } static int echainiv_decrypt(struct aead_request *req) @@ -192,8 +127,7 @@ static int echainiv_aead_create(struct crypto_template *tmpl, alg = crypto_spawn_aead_alg(spawn); err = -EINVAL; - if (inst->alg.ivsize & (sizeof(u32) - 1) || - inst->alg.ivsize > MAX_IV_SIZE) + if (inst->alg.ivsize & (sizeof(u64) - 1) || !inst->alg.ivsize) goto free_inst; inst->alg.encrypt = echainiv_encrypt; @@ -202,7 +136,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl, inst->alg.init = aead_init_geniv; inst->alg.exit = aead_exit_geniv; - inst->alg.base.cra_alignmask |= __alignof__(u32) - 1; inst->alg.base.cra_ctxsize = sizeof(struct aead_geniv_ctx); inst->alg.base.cra_ctxsize += inst->alg.ivsize; diff --git a/kernel/crypto/gcm.c b/kernel/crypto/gcm.c index bec329b3d..1238b3c5a 100644 --- a/kernel/crypto/gcm.c +++ b/kernel/crypto/gcm.c @@ -117,7 +117,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, struct crypto_ablkcipher *ctr = ctx->ctr; struct { be128 hash; - u8 iv[8]; + u8 iv[16]; struct crypto_gcm_setkey_result result; @@ -639,7 +639,9 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_AHASH_MASK); + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, + algt->mask)); if (IS_ERR(ghash_alg)) return PTR_ERR(ghash_alg); diff --git a/kernel/crypto/ghash-generic.c b/kernel/crypto/ghash-generic.c index bac70995e..12ad3e3a8 100644 --- a/kernel/crypto/ghash-generic.c +++ b/kernel/crypto/ghash-generic.c @@ -14,24 +14,13 @@ #include #include +#include #include #include #include #include #include -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 - -struct ghash_ctx { - struct gf128mul_4k *gf128; -}; - -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; -}; - static int ghash_init(struct shash_desc *desc) { struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); diff --git a/kernel/crypto/keywrap.c b/kernel/crypto/keywrap.c index b1d106ce5..72014f963 100644 --- a/kernel/crypto/keywrap.c +++ b/kernel/crypto/keywrap.c @@ -212,7 +212,7 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc, SEMIBSIZE)) ret = -EBADMSG; - memzero_explicit(&block, sizeof(struct crypto_kw_block)); + memzero_explicit(block, sizeof(struct crypto_kw_block)); return ret; } @@ -297,7 +297,7 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc, /* establish the IV for the caller to pick up */ memcpy(desc->info, block->A, SEMIBSIZE); - memzero_explicit(&block, sizeof(struct crypto_kw_block)); + memzero_explicit(block, sizeof(struct crypto_kw_block)); return 0; } diff --git a/kernel/crypto/mcryptd.c b/kernel/crypto/mcryptd.c index fe5b495a4..a0ceb41d5 100644 --- a/kernel/crypto/mcryptd.c +++ b/kernel/crypto/mcryptd.c @@ -258,18 +258,22 @@ out_free_inst: goto out; } -static inline void mcryptd_check_internal(struct rtattr **tb, u32 *type, +static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type, u32 *mask) { struct crypto_attr_type *algt; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return; - if ((algt->type & CRYPTO_ALG_INTERNAL)) - *type |= CRYPTO_ALG_INTERNAL; - if ((algt->mask & CRYPTO_ALG_INTERNAL)) - *mask |= CRYPTO_ALG_INTERNAL; + return false; + + *type |= algt->type & CRYPTO_ALG_INTERNAL; + *mask |= algt->mask & CRYPTO_ALG_INTERNAL; + + if (*type & *mask & CRYPTO_ALG_INTERNAL) + return true; + else + return false; } static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) @@ -498,7 +502,8 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, u32 mask = 0; int err; - mcryptd_check_internal(tb, &type, &mask); + if (!mcryptd_check_internal(tb, &type, &mask)) + return -EINVAL; salg = shash_attr_alg(tb[1], type, mask); if (IS_ERR(salg)) diff --git a/kernel/crypto/scatterwalk.c b/kernel/crypto/scatterwalk.c index ea5815c5e..bc769c448 100644 --- a/kernel/crypto/scatterwalk.c +++ b/kernel/crypto/scatterwalk.c @@ -72,7 +72,8 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out, void scatterwalk_done(struct scatter_walk *walk, int out, int more) { - if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) || !more) + if (!more || walk->offset >= walk->sg->offset + walk->sg->length || + !(walk->offset & (PAGE_SIZE - 1))) scatterwalk_pagedone(walk, out, more); } EXPORT_SYMBOL_GPL(scatterwalk_done); diff --git a/kernel/crypto/testmgr.c b/kernel/crypto/testmgr.c index ae8c57fd8..d4944318c 100644 --- a/kernel/crypto/testmgr.c +++ b/kernel/crypto/testmgr.c @@ -1849,6 +1849,7 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver, static int do_test_rsa(struct crypto_akcipher *tfm, struct akcipher_testvec *vecs) { + char *xbuf[XBUFSIZE]; struct akcipher_request *req; void *outbuf_enc = NULL; void *outbuf_dec = NULL; @@ -1857,9 +1858,12 @@ static int do_test_rsa(struct crypto_akcipher *tfm, int err = -ENOMEM; struct scatterlist src, dst, src_tab[2]; + if (testmgr_alloc_buf(xbuf)) + return err; + req = akcipher_request_alloc(tfm, GFP_KERNEL); if (!req) - return err; + goto free_xbuf; init_completion(&result.completion); @@ -1877,9 +1881,14 @@ static int do_test_rsa(struct crypto_akcipher *tfm, if (!outbuf_enc) goto free_req; + if (WARN_ON(vecs->m_size > PAGE_SIZE)) + goto free_all; + + memcpy(xbuf[0], vecs->m, vecs->m_size); + sg_init_table(src_tab, 2); - sg_set_buf(&src_tab[0], vecs->m, 8); - sg_set_buf(&src_tab[1], vecs->m + 8, vecs->m_size - 8); + sg_set_buf(&src_tab[0], xbuf[0], 8); + sg_set_buf(&src_tab[1], xbuf[0] + 8, vecs->m_size - 8); sg_init_one(&dst, outbuf_enc, out_len_max); akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size, out_len_max); @@ -1898,7 +1907,7 @@ static int do_test_rsa(struct crypto_akcipher *tfm, goto free_all; } /* verify that encrypted message is equal to expected */ - if (memcmp(vecs->c, sg_virt(req->dst), vecs->c_size)) { + if (memcmp(vecs->c, outbuf_enc, vecs->c_size)) { pr_err("alg: rsa: encrypt test failed. Invalid output\n"); err = -EINVAL; goto free_all; @@ -1913,7 +1922,13 @@ static int do_test_rsa(struct crypto_akcipher *tfm, err = -ENOMEM; goto free_all; } - sg_init_one(&src, vecs->c, vecs->c_size); + + if (WARN_ON(vecs->c_size > PAGE_SIZE)) + goto free_all; + + memcpy(xbuf[0], vecs->c, vecs->c_size); + + sg_init_one(&src, xbuf[0], vecs->c_size); sg_init_one(&dst, outbuf_dec, out_len_max); init_completion(&result.completion); akcipher_request_set_crypt(req, &src, &dst, vecs->c_size, out_len_max); @@ -1940,6 +1955,8 @@ free_all: kfree(outbuf_enc); free_req: akcipher_request_free(req); +free_xbuf: + testmgr_free_buf(xbuf); return err; } diff --git a/kernel/drivers/acpi/acpi_processor.c b/kernel/drivers/acpi/acpi_processor.c index 6979186db..9f7794365 100644 --- a/kernel/drivers/acpi/acpi_processor.c +++ b/kernel/drivers/acpi/acpi_processor.c @@ -491,6 +491,58 @@ static void acpi_processor_remove(struct acpi_device *device) } #endif /* CONFIG_ACPI_HOTPLUG_CPU */ +#ifdef CONFIG_X86 +static bool acpi_hwp_native_thermal_lvt_set; +static acpi_status __init acpi_hwp_native_thermal_lvt_osc(acpi_handle handle, + u32 lvl, + void *context, + void **rv) +{ + u8 sb_uuid_str[] = "4077A616-290C-47BE-9EBD-D87058713953"; + u32 capbuf[2]; + struct acpi_osc_context osc_context = { + .uuid_str = sb_uuid_str, + .rev = 1, + .cap.length = 8, + .cap.pointer = capbuf, + }; + + if (acpi_hwp_native_thermal_lvt_set) + return AE_CTRL_TERMINATE; + + capbuf[0] = 0x0000; + capbuf[1] = 0x1000; /* set bit 12 */ + + if (ACPI_SUCCESS(acpi_run_osc(handle, &osc_context))) { + if (osc_context.ret.pointer && osc_context.ret.length > 1) { + u32 *capbuf_ret = osc_context.ret.pointer; + + if (capbuf_ret[1] & 0x1000) { + acpi_handle_info(handle, + "_OSC native thermal LVT Acked\n"); + acpi_hwp_native_thermal_lvt_set = true; + } + } + kfree(osc_context.ret.pointer); + } + + return AE_OK; +} + +void __init acpi_early_processor_osc(void) +{ + if (boot_cpu_has(X86_FEATURE_HWP)) { + acpi_walk_namespace(ACPI_TYPE_PROCESSOR, ACPI_ROOT_OBJECT, + ACPI_UINT32_MAX, + acpi_hwp_native_thermal_lvt_osc, + NULL, NULL, NULL); + acpi_get_devices(ACPI_PROCESSOR_DEVICE_HID, + acpi_hwp_native_thermal_lvt_osc, + NULL, NULL); + } +} +#endif + /* * The following ACPI IDs are known to be suitable for representing as * processor devices. diff --git a/kernel/drivers/acpi/acpica/dsmethod.c b/kernel/drivers/acpi/acpica/dsmethod.c index bc32f3194..28c50c6b5 100644 --- a/kernel/drivers/acpi/acpica/dsmethod.c +++ b/kernel/drivers/acpi/acpica/dsmethod.c @@ -417,6 +417,9 @@ acpi_ds_begin_method_execution(struct acpi_namespace_node *method_node, obj_desc->method.mutex->mutex. original_sync_level = obj_desc->method.mutex->mutex.sync_level; + + obj_desc->method.mutex->mutex.thread_id = + acpi_os_get_thread_id(); } } diff --git a/kernel/drivers/acpi/acpica/hwxface.c b/kernel/drivers/acpi/acpica/hwxface.c index 8c017f15d..ff007084d 100644 --- a/kernel/drivers/acpi/acpica/hwxface.c +++ b/kernel/drivers/acpi/acpica/hwxface.c @@ -504,11 +504,20 @@ acpi_get_sleep_type_data(u8 sleep_state, u8 *sleep_type_a, u8 *sleep_type_b) * Evaluate the \_Sx namespace object containing the register values * for this state */ - info->relative_pathname = - ACPI_CAST_PTR(char, acpi_gbl_sleep_state_names[sleep_state]); + info->relative_pathname = ACPI_CAST_PTR(char, + acpi_gbl_sleep_state_names + [sleep_state]); + status = acpi_ns_evaluate(info); if (ACPI_FAILURE(status)) { - goto cleanup; + if (status == AE_NOT_FOUND) { + + /* The _Sx states are optional, ignore NOT_FOUND */ + + goto final_cleanup; + } + + goto warning_cleanup; } /* Must have a return object */ @@ -517,7 +526,7 @@ acpi_get_sleep_type_data(u8 sleep_state, u8 *sleep_type_a, u8 *sleep_type_b) ACPI_ERROR((AE_INFO, "No Sleep State object returned from [%s]", info->relative_pathname)); status = AE_AML_NO_RETURN_VALUE; - goto cleanup; + goto warning_cleanup; } /* Return object must be of type Package */ @@ -526,7 +535,7 @@ acpi_get_sleep_type_data(u8 sleep_state, u8 *sleep_type_a, u8 *sleep_type_b) ACPI_ERROR((AE_INFO, "Sleep State return object is not a Package")); status = AE_AML_OPERAND_TYPE; - goto cleanup1; + goto return_value_cleanup; } /* @@ -570,16 +579,17 @@ acpi_get_sleep_type_data(u8 sleep_state, u8 *sleep_type_a, u8 *sleep_type_b) break; } -cleanup1: +return_value_cleanup: acpi_ut_remove_reference(info->return_object); -cleanup: +warning_cleanup: if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "While evaluating Sleep State [%s]", info->relative_pathname)); } +final_cleanup: ACPI_FREE(info); return_ACPI_STATUS(status); } diff --git a/kernel/drivers/acpi/apei/ghes.c b/kernel/drivers/acpi/apei/ghes.c index 3dd9c462d..eac4f3b02 100644 --- a/kernel/drivers/acpi/apei/ghes.c +++ b/kernel/drivers/acpi/apei/ghes.c @@ -657,7 +657,7 @@ static int ghes_proc(struct ghes *ghes) ghes_do_proc(ghes, ghes->estatus); out: ghes_clear_estatus(ghes); - return 0; + return rc; } static void ghes_add_timer(struct ghes *ghes) @@ -847,6 +847,8 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) if (ghes_read_estatus(ghes, 1)) { ghes_clear_estatus(ghes); continue; + } else { + ret = NMI_HANDLED; } sev = ghes_severity(ghes->estatus->error_severity); @@ -858,12 +860,11 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) __process_error(ghes); ghes_clear_estatus(ghes); - - ret = NMI_HANDLED; } #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG - irq_work_queue(&ghes_proc_irq_work); + if (ret == NMI_HANDLED) + irq_work_queue(&ghes_proc_irq_work); #endif atomic_dec(&ghes_in_nmi); return ret; diff --git a/kernel/drivers/acpi/bus.c b/kernel/drivers/acpi/bus.c index a212cefae..ca4f28432 100644 --- a/kernel/drivers/acpi/bus.c +++ b/kernel/drivers/acpi/bus.c @@ -1004,6 +1004,9 @@ static int __init acpi_bus_init(void) goto error1; } + /* Set capability bits for _OSC under processor scope */ + acpi_early_processor_osc(); + /* * _OSC method may exist in module level code, * so it must be run after ACPI_FULL_INITIALIZATION diff --git a/kernel/drivers/acpi/cppc_acpi.c b/kernel/drivers/acpi/cppc_acpi.c index 6730f965b..0afd1981e 100644 --- a/kernel/drivers/acpi/cppc_acpi.c +++ b/kernel/drivers/acpi/cppc_acpi.c @@ -216,8 +216,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; cpc_ptr = per_cpu(cpc_desc_ptr, i); - if (!cpc_ptr) - continue; + if (!cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } pdomain = &(cpc_ptr->domain_info); cpumask_set_cpu(i, pr->shared_cpu_map); @@ -239,8 +241,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; match_cpc_ptr = per_cpu(cpc_desc_ptr, j); - if (!match_cpc_ptr) - continue; + if (!match_cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } match_pdomain = &(match_cpc_ptr->domain_info); if (match_pdomain->domain != pdomain->domain) @@ -270,8 +274,10 @@ int acpi_get_psd_map(struct cpudata **all_cpu_data) continue; match_cpc_ptr = per_cpu(cpc_desc_ptr, j); - if (!match_cpc_ptr) - continue; + if (!match_cpc_ptr) { + retval = -EFAULT; + goto err_ret; + } match_pdomain = &(match_cpc_ptr->domain_info); if (match_pdomain->domain != pdomain->domain) @@ -502,9 +508,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) /* Store CPU Logical ID */ cpc_ptr->cpu_id = pr->id; - /* Plug it into this CPUs CPC descriptor. */ - per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr; - /* Parse PSD data for this CPU */ ret = acpi_get_psd(cpc_ptr, handle); if (ret) @@ -517,6 +520,9 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) goto out_free; } + /* Plug PSD data into this CPUs CPC descriptor. */ + per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr; + /* Everything looks okay */ pr_debug("Parsed CPC struct for CPU: %d\n", pr->id); diff --git a/kernel/drivers/acpi/ec.c b/kernel/drivers/acpi/ec.c index b420fb466..43f20328f 100644 --- a/kernel/drivers/acpi/ec.c +++ b/kernel/drivers/acpi/ec.c @@ -101,6 +101,7 @@ enum ec_command { #define ACPI_EC_UDELAY_POLL 550 /* Wait 1ms for EC transaction polling */ #define ACPI_EC_CLEAR_MAX 100 /* Maximum number of events to query * when trying to clear the EC */ +#define ACPI_EC_MAX_QUERIES 16 /* Maximum number of parallel queries */ enum { EC_FLAGS_QUERY_PENDING, /* Query is pending */ @@ -121,6 +122,10 @@ static unsigned int ec_delay __read_mostly = ACPI_EC_DELAY; module_param(ec_delay, uint, 0644); MODULE_PARM_DESC(ec_delay, "Timeout(ms) waited until an EC command completes"); +static unsigned int ec_max_queries __read_mostly = ACPI_EC_MAX_QUERIES; +module_param(ec_max_queries, uint, 0644); +MODULE_PARM_DESC(ec_max_queries, "Maximum parallel _Qxx evaluations"); + static bool ec_busy_polling __read_mostly; module_param(ec_busy_polling, bool, 0644); MODULE_PARM_DESC(ec_busy_polling, "Use busy polling to advance EC transaction"); @@ -174,6 +179,7 @@ static void acpi_ec_event_processor(struct work_struct *work); struct acpi_ec *boot_ec, *first_ec; EXPORT_SYMBOL(first_ec); +static struct workqueue_struct *ec_query_wq; static int EC_FLAGS_VALIDATE_ECDT; /* ASUStec ECDTs need to be validated */ static int EC_FLAGS_SKIP_DSDT_SCAN; /* Not all BIOS survive early DSDT scan */ @@ -1097,7 +1103,7 @@ static int acpi_ec_query(struct acpi_ec *ec, u8 *data) * work queue execution. */ ec_dbg_evt("Query(0x%02x) scheduled", value); - if (!schedule_work(&q->work)) { + if (!queue_work(ec_query_wq, &q->work)) { ec_dbg_evt("Query(0x%02x) overlapped", value); result = -EBUSY; } @@ -1657,15 +1663,41 @@ static struct acpi_driver acpi_ec_driver = { }, }; +static inline int acpi_ec_query_init(void) +{ + if (!ec_query_wq) { + ec_query_wq = alloc_workqueue("kec_query", 0, + ec_max_queries); + if (!ec_query_wq) + return -ENODEV; + } + return 0; +} + +static inline void acpi_ec_query_exit(void) +{ + if (ec_query_wq) { + destroy_workqueue(ec_query_wq); + ec_query_wq = NULL; + } +} + int __init acpi_ec_init(void) { - int result = 0; + int result; + /* register workqueue for _Qxx evaluations */ + result = acpi_ec_query_init(); + if (result) + goto err_exit; /* Now register the driver for the EC */ result = acpi_bus_register_driver(&acpi_ec_driver); - if (result < 0) - return -ENODEV; + if (result) + goto err_exit; +err_exit: + if (result) + acpi_ec_query_exit(); return result; } @@ -1675,5 +1707,6 @@ static void __exit acpi_ec_exit(void) { acpi_bus_unregister_driver(&acpi_ec_driver); + acpi_ec_query_exit(); } #endif /* 0 */ diff --git a/kernel/drivers/acpi/internal.h b/kernel/drivers/acpi/internal.h index 11d87bf67..0f3f41c13 100644 --- a/kernel/drivers/acpi/internal.h +++ b/kernel/drivers/acpi/internal.h @@ -130,6 +130,12 @@ void acpi_early_processor_set_pdc(void); static inline void acpi_early_processor_set_pdc(void) {} #endif +#ifdef CONFIG_X86 +void acpi_early_processor_osc(void); +#else +static inline void acpi_early_processor_osc(void) {} +#endif + /* -------------------------------------------------------------------------- Embedded Controller -------------------------------------------------------------------------- */ diff --git a/kernel/drivers/acpi/nfit.c b/kernel/drivers/acpi/nfit.c index 11d8209e6..c097f477c 100644 --- a/kernel/drivers/acpi/nfit.c +++ b/kernel/drivers/acpi/nfit.c @@ -1072,11 +1072,12 @@ static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) { struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; u64 offset = nfit_blk->stat_offset + mmio->size * bw; + const u32 STATUS_MASK = 0x80000037; if (mmio->num_lines) offset = to_interleave_offset(offset, mmio); - return readl(mmio->addr.base + offset); + return readl(mmio->addr.base + offset) & STATUS_MASK; } static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, @@ -1805,6 +1806,9 @@ static void acpi_nfit_notify(struct acpi_device *adev, u32 event) dev_dbg(dev, "%s: event: %d\n", __func__, event); + if (event != NFIT_NOTIFY_UPDATE) + return; + device_lock(dev); if (!dev->driver) { /* dev->driver may be null if we're being removed */ diff --git a/kernel/drivers/acpi/nfit.h b/kernel/drivers/acpi/nfit.h index 3d549a383..13d6ec1ff 100644 --- a/kernel/drivers/acpi/nfit.h +++ b/kernel/drivers/acpi/nfit.h @@ -45,6 +45,10 @@ enum { ND_BLK_DCR_LATCH = 2, }; +enum nfit_root_notifiers { + NFIT_NOTIFY_UPDATE = 0x80, +}; + struct nfit_spa { struct acpi_nfit_system_address *spa; struct list_head list; diff --git a/kernel/drivers/acpi/numa.c b/kernel/drivers/acpi/numa.c index 72b6e9ef0..d176e0ece 100644 --- a/kernel/drivers/acpi/numa.c +++ b/kernel/drivers/acpi/numa.c @@ -327,10 +327,18 @@ int __init acpi_numa_init(void) /* SRAT: Static Resource Affinity Table */ if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) { - acpi_table_parse_srat(ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY, - acpi_parse_x2apic_affinity, 0); - acpi_table_parse_srat(ACPI_SRAT_TYPE_CPU_AFFINITY, - acpi_parse_processor_affinity, 0); + struct acpi_subtable_proc srat_proc[2]; + + memset(srat_proc, 0, sizeof(srat_proc)); + srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY; + srat_proc[0].handler = acpi_parse_processor_affinity; + srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY; + srat_proc[1].handler = acpi_parse_x2apic_affinity; + + acpi_table_parse_entries_array(ACPI_SIG_SRAT, + sizeof(struct acpi_table_srat), + srat_proc, ARRAY_SIZE(srat_proc), 0); + cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY, acpi_parse_memory_affinity, NR_NODE_MEMBLKS); diff --git a/kernel/drivers/acpi/osl.c b/kernel/drivers/acpi/osl.c index 32d684af0..a000ecb99 100644 --- a/kernel/drivers/acpi/osl.c +++ b/kernel/drivers/acpi/osl.c @@ -135,7 +135,7 @@ static struct osi_linux { unsigned int enable:1; unsigned int dmi:1; unsigned int cmdline:1; - unsigned int default_disabling:1; + u8 default_disabling; } osi_linux = {0, 0, 0, 0}; static u32 acpi_osi_handler(acpi_string interface, u32 supported) @@ -1444,10 +1444,13 @@ void __init acpi_osi_setup(char *str) if (*str == '!') { str++; if (*str == '\0') { - osi_linux.default_disabling = 1; + /* Do not override acpi_osi=!* */ + if (!osi_linux.default_disabling) + osi_linux.default_disabling = + ACPI_DISABLE_ALL_VENDOR_STRINGS; return; } else if (*str == '*') { - acpi_update_interfaces(ACPI_DISABLE_ALL_STRINGS); + osi_linux.default_disabling = ACPI_DISABLE_ALL_STRINGS; for (i = 0; i < OSI_STRING_ENTRIES_MAX; i++) { osi = &osi_setup_entries[i]; osi->enable = false; @@ -1520,10 +1523,13 @@ static void __init acpi_osi_setup_late(void) acpi_status status; if (osi_linux.default_disabling) { - status = acpi_update_interfaces(ACPI_DISABLE_ALL_VENDOR_STRINGS); + status = acpi_update_interfaces(osi_linux.default_disabling); if (ACPI_SUCCESS(status)) - printk(KERN_INFO PREFIX "Disabled all _OSI OS vendors\n"); + printk(KERN_INFO PREFIX "Disabled all _OSI OS vendors%s\n", + osi_linux.default_disabling == + ACPI_DISABLE_ALL_STRINGS ? + " and feature groups" : ""); } for (i = 0; i < OSI_STRING_ENTRIES_MAX; i++) { diff --git a/kernel/drivers/acpi/pci_irq.c b/kernel/drivers/acpi/pci_irq.c index c9336751e..8a10a7ae6 100644 --- a/kernel/drivers/acpi/pci_irq.c +++ b/kernel/drivers/acpi/pci_irq.c @@ -409,7 +409,7 @@ int acpi_pci_irq_enable(struct pci_dev *dev) return 0; } - if (pci_has_managed_irq(dev)) + if (dev->irq_managed && dev->irq > 0) return 0; entry = acpi_pci_irq_lookup(dev, pin); @@ -454,7 +454,8 @@ int acpi_pci_irq_enable(struct pci_dev *dev) kfree(entry); return rc; } - pci_set_managed_irq(dev, rc); + dev->irq = rc; + dev->irq_managed = 1; if (link) snprintf(link_desc, sizeof(link_desc), " -> Link[%s]", link); @@ -477,9 +478,17 @@ void acpi_pci_irq_disable(struct pci_dev *dev) u8 pin; pin = dev->pin; - if (!pin || !pci_has_managed_irq(dev)) + if (!pin || !dev->irq_managed || dev->irq <= 0) return; + /* Keep IOAPIC pin configuration when suspending */ + if (dev->dev.power.is_prepared) + return; +#ifdef CONFIG_PM + if (dev->dev.power.runtime_status == RPM_SUSPENDING) + return; +#endif + entry = acpi_pci_irq_lookup(dev, pin); if (!entry) return; @@ -499,6 +508,6 @@ void acpi_pci_irq_disable(struct pci_dev *dev) dev_dbg(&dev->dev, "PCI INT %c disabled\n", pin_name(pin)); if (gsi >= 0) { acpi_unregister_gsi(gsi); - pci_reset_managed_irq(dev); + dev->irq_managed = 0; } } diff --git a/kernel/drivers/acpi/resource.c b/kernel/drivers/acpi/resource.c index cdc5c2599..627f8fbb5 100644 --- a/kernel/drivers/acpi/resource.c +++ b/kernel/drivers/acpi/resource.c @@ -26,8 +26,20 @@ #ifdef CONFIG_X86 #define valid_IRQ(i) (((i) != 0) && ((i) != 2)) +static inline bool acpi_iospace_resource_valid(struct resource *res) +{ + /* On X86 IO space is limited to the [0 - 64K] IO port range */ + return res->end < 0x10003; +} #else #define valid_IRQ(i) (true) +/* + * ACPI IO descriptors on arches other than X86 contain MMIO CPU physical + * addresses mapping IO space in CPU physical address space, IO space + * resources can be placed anywhere in the 64-bit physical address space. + */ +static inline bool +acpi_iospace_resource_valid(struct resource *res) { return true; } #endif static bool acpi_dev_resource_len_valid(u64 start, u64 end, u64 len, bool io) @@ -126,7 +138,7 @@ static void acpi_dev_ioresource_flags(struct resource *res, u64 len, if (!acpi_dev_resource_len_valid(res->start, res->end, len, true)) res->flags |= IORESOURCE_DISABLED | IORESOURCE_UNSET; - if (res->end >= 0x10003) + if (!acpi_iospace_resource_valid(res)) res->flags |= IORESOURCE_DISABLED | IORESOURCE_UNSET; if (io_decode == ACPI_DECODE_16) diff --git a/kernel/drivers/acpi/scan.c b/kernel/drivers/acpi/scan.c index 78d5f02a0..dcb3d6245 100644 --- a/kernel/drivers/acpi/scan.c +++ b/kernel/drivers/acpi/scan.c @@ -1958,7 +1958,7 @@ int __init acpi_scan_init(void) static struct acpi_probe_entry *ape; static int acpi_probe_count; -static DEFINE_SPINLOCK(acpi_probe_lock); +static DEFINE_MUTEX(acpi_probe_mutex); static int __init acpi_match_madt(struct acpi_subtable_header *header, const unsigned long end) @@ -1977,7 +1977,7 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr) if (acpi_disabled) return 0; - spin_lock(&acpi_probe_lock); + mutex_lock(&acpi_probe_mutex); for (ape = ap_head; nr; ape++, nr--) { if (ACPI_COMPARE_NAME(ACPI_SIG_MADT, ape->id)) { acpi_probe_count = 0; @@ -1990,7 +1990,7 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr) count++; } } - spin_unlock(&acpi_probe_lock); + mutex_unlock(&acpi_probe_mutex); return count; } diff --git a/kernel/drivers/acpi/sleep.c b/kernel/drivers/acpi/sleep.c index 0d94621dc..e3322adaa 100644 --- a/kernel/drivers/acpi/sleep.c +++ b/kernel/drivers/acpi/sleep.c @@ -714,6 +714,7 @@ static int acpi_hibernation_enter(void) static void acpi_hibernation_leave(void) { + pm_set_resume_via_firmware(); /* * If ACPI is not enabled by the BIOS and the boot kernel, we need to * enable it here. diff --git a/kernel/drivers/acpi/sysfs.c b/kernel/drivers/acpi/sysfs.c index 0243d375c..4b3a9e27f 100644 --- a/kernel/drivers/acpi/sysfs.c +++ b/kernel/drivers/acpi/sysfs.c @@ -555,23 +555,22 @@ static void acpi_global_event_handler(u32 event_type, acpi_handle device, static int get_status(u32 index, acpi_event_status *status, acpi_handle *handle) { - int result = 0; + int result; if (index >= num_gpes + ACPI_NUM_FIXED_EVENTS) - goto end; + return -EINVAL; if (index < num_gpes) { result = acpi_get_gpe_device(index, handle); if (result) { ACPI_EXCEPTION((AE_INFO, AE_NOT_FOUND, "Invalid GPE 0x%x", index)); - goto end; + return result; } result = acpi_get_gpe_status(*handle, index, status); } else if (index < (num_gpes + ACPI_NUM_FIXED_EVENTS)) result = acpi_get_event_status(index - num_gpes, status); -end: return result; } diff --git a/kernel/drivers/acpi/video_detect.c b/kernel/drivers/acpi/video_detect.c index 80e55cb08..b48ecbfc4 100644 --- a/kernel/drivers/acpi/video_detect.c +++ b/kernel/drivers/acpi/video_detect.c @@ -271,6 +271,26 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro12,1"), }, }, + { + /* https://bugzilla.redhat.com/show_bug.cgi?id=1123661 */ + .callback = video_detect_force_native, + .ident = "Dell XPS 17 L702X", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell System XPS L702X"), + }, + }, + { + /* https://bugzilla.redhat.com/show_bug.cgi?id=1204476 */ + /* https://bugs.launchpad.net/ubuntu/+source/linux-lts-trusty/+bug/1416940 */ + .callback = video_detect_force_native, + .ident = "HP Pavilion dv6", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP Pavilion dv6 Notebook PC"), + }, + }, + { }, }; diff --git a/kernel/drivers/android/binder.c b/kernel/drivers/android/binder.c index 7d00b7a01..47ddfefe2 100644 --- a/kernel/drivers/android/binder.c +++ b/kernel/drivers/android/binder.c @@ -1003,7 +1003,7 @@ static int binder_dec_node(struct binder_node *node, int strong, int internal) static struct binder_ref *binder_get_ref(struct binder_proc *proc, - uint32_t desc) + u32 desc, bool need_strong_ref) { struct rb_node *n = proc->refs_by_desc.rb_node; struct binder_ref *ref; @@ -1011,12 +1011,16 @@ static struct binder_ref *binder_get_ref(struct binder_proc *proc, while (n) { ref = rb_entry(n, struct binder_ref, rb_node_desc); - if (desc < ref->desc) + if (desc < ref->desc) { n = n->rb_left; - else if (desc > ref->desc) + } else if (desc > ref->desc) { n = n->rb_right; - else + } else if (need_strong_ref && !ref->strong) { + binder_user_error("tried to use weak ref as strong ref\n"); + return NULL; + } else { return ref; + } } return NULL; } @@ -1286,7 +1290,10 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref; + + ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { pr_err("transaction release %d bad handle %d\n", @@ -1380,7 +1387,7 @@ static void binder_transaction(struct binder_proc *proc, if (tr->target.handle) { struct binder_ref *ref; - ref = binder_get_ref(proc, tr->target.handle); + ref = binder_get_ref(proc, tr->target.handle, true); if (ref == NULL) { binder_user_error("%d:%d got transaction to invalid handle\n", proc->pid, thread->pid); @@ -1571,7 +1578,9 @@ static void binder_transaction(struct binder_proc *proc, fp->type = BINDER_TYPE_HANDLE; else fp->type = BINDER_TYPE_WEAK_HANDLE; + fp->binder = 0; fp->handle = ref->desc; + fp->cookie = 0; binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, &thread->todo); @@ -1583,7 +1592,10 @@ static void binder_transaction(struct binder_proc *proc, } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref; + + ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { binder_user_error("%d:%d got transaction with invalid handle, %d\n", @@ -1618,7 +1630,9 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + fp->binder = 0; fp->handle = new_ref->desc; + fp->cookie = 0; binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); trace_binder_transaction_ref_to_ref(t, ref, new_ref); @@ -1672,6 +1686,7 @@ static void binder_transaction(struct binder_proc *proc, binder_debug(BINDER_DEBUG_TRANSACTION, " fd %d -> %d\n", fp->handle, target_fd); /* TODO: fput? */ + fp->binder = 0; fp->handle = target_fd; } break; @@ -1794,7 +1809,9 @@ static int binder_thread_write(struct binder_proc *proc, ref->desc); } } else - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, + cmd == BC_ACQUIRE || + cmd == BC_RELEASE); if (ref == NULL) { binder_user_error("%d:%d refcount change on invalid ref %d\n", proc->pid, thread->pid, target); @@ -1990,7 +2007,7 @@ static int binder_thread_write(struct binder_proc *proc, if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, false); if (ref == NULL) { binder_user_error("%d:%d %s invalid ref %d\n", proc->pid, thread->pid, diff --git a/kernel/drivers/ata/ahci_platform.c b/kernel/drivers/ata/ahci_platform.c index 04975b851..639adb1f8 100644 --- a/kernel/drivers/ata/ahci_platform.c +++ b/kernel/drivers/ata/ahci_platform.c @@ -51,6 +51,9 @@ static int ahci_probe(struct platform_device *pdev) if (rc) return rc; + of_property_read_u32(dev->of_node, + "ports-implemented", &hpriv->force_port_map); + if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci")) hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ; diff --git a/kernel/drivers/ata/ahci_xgene.c b/kernel/drivers/ata/ahci_xgene.c index e2c6d9e0c..e916bff6c 100644 --- a/kernel/drivers/ata/ahci_xgene.c +++ b/kernel/drivers/ata/ahci_xgene.c @@ -739,9 +739,9 @@ static int xgene_ahci_probe(struct platform_device *pdev) dev_warn(&pdev->dev, "%s: Error reading device info. Assume version1\n", __func__); version = XGENE_AHCI_V1; - } - if (info->valid & ACPI_VALID_CID) + } else if (info->valid & ACPI_VALID_CID) { version = XGENE_AHCI_V2; + } } } #endif diff --git a/kernel/drivers/ata/libahci.c b/kernel/drivers/ata/libahci.c index 998c6a85a..9628fa131 100644 --- a/kernel/drivers/ata/libahci.c +++ b/kernel/drivers/ata/libahci.c @@ -467,6 +467,7 @@ void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) dev_info(dev, "forcing port_map 0x%x -> 0x%x\n", port_map, hpriv->force_port_map); port_map = hpriv->force_port_map; + hpriv->saved_port_map = port_map; } if (hpriv->mask_port_map) { diff --git a/kernel/drivers/ata/libata-core.c b/kernel/drivers/ata/libata-core.c index b79cb10e2..b0b77b61c 100644 --- a/kernel/drivers/ata/libata-core.c +++ b/kernel/drivers/ata/libata-core.c @@ -4138,6 +4138,12 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { */ { "ST380013AS", "3.20", ATA_HORKAGE_MAX_SEC_1024 }, + /* + * These devices time out with higher max sects. + * https://bugzilla.kernel.org/show_bug.cgi?id=121671 + */ + { "LITEON CX1-JB*-HP", NULL, ATA_HORKAGE_MAX_SEC_1024 }, + /* Devices we expect to fail diagnostics */ /* Devices where NCQ should be avoided */ diff --git a/kernel/drivers/ata/libata-eh.c b/kernel/drivers/ata/libata-eh.c index 961acc788..91a9e6af2 100644 --- a/kernel/drivers/ata/libata-eh.c +++ b/kernel/drivers/ata/libata-eh.c @@ -606,7 +606,7 @@ void ata_scsi_error(struct Scsi_Host *host) ata_scsi_port_error_handler(host, ap); /* finish or retry handled scmd's and clean up */ - WARN_ON(host->host_failed || !list_empty(&eh_work_q)); + WARN_ON(!list_empty(&eh_work_q)); DPRINTK("EXIT\n"); } diff --git a/kernel/drivers/ata/sata_mv.c b/kernel/drivers/ata/sata_mv.c index bd74ee555..729f26322 100644 --- a/kernel/drivers/ata/sata_mv.c +++ b/kernel/drivers/ata/sata_mv.c @@ -4121,6 +4121,9 @@ static int mv_platform_probe(struct platform_device *pdev) host->iomap = NULL; hpriv->base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!hpriv->base) + return -ENOMEM; + hpriv->base -= SATAHC0_REG_BASE; hpriv->clk = clk_get(&pdev->dev, NULL); diff --git a/kernel/drivers/base/core.c b/kernel/drivers/base/core.c index b7d56c5ea..f18856f59 100644 --- a/kernel/drivers/base/core.c +++ b/kernel/drivers/base/core.c @@ -836,11 +836,29 @@ static struct kobject *get_device_parent(struct device *dev, return NULL; } +static inline bool live_in_glue_dir(struct kobject *kobj, + struct device *dev) +{ + if (!kobj || !dev->class || + kobj->kset != &dev->class->p->glue_dirs) + return false; + return true; +} + +static inline struct kobject *get_glue_dir(struct device *dev) +{ + return dev->kobj.parent; +} + +/* + * make sure cleaning up dir as the last step, we need to make + * sure .release handler of kobject is run with holding the + * global lock + */ static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir) { /* see if we live in a "glue" directory */ - if (!glue_dir || !dev->class || - glue_dir->kset != &dev->class->p->glue_dirs) + if (!live_in_glue_dir(glue_dir, dev)) return; mutex_lock(&gdp_mutex); @@ -848,11 +866,6 @@ static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir) mutex_unlock(&gdp_mutex); } -static void cleanup_device_parent(struct device *dev) -{ - cleanup_glue_dir(dev, dev->kobj.parent); -} - static int device_add_class_symlinks(struct device *dev) { struct device_node *of_node = dev_of_node(dev); @@ -1028,6 +1041,7 @@ int device_add(struct device *dev) struct kobject *kobj; struct class_interface *class_intf; int error = -EINVAL; + struct kobject *glue_dir = NULL; dev = get_device(dev); if (!dev) @@ -1072,8 +1086,10 @@ int device_add(struct device *dev) /* first, register with generic layer. */ /* we require the name to be set before, and pass NULL */ error = kobject_add(&dev->kobj, dev->kobj.parent, NULL); - if (error) + if (error) { + glue_dir = get_glue_dir(dev); goto Error; + } /* notify platform of device entry */ if (platform_notify) @@ -1154,9 +1170,10 @@ done: device_remove_file(dev, &dev_attr_uevent); attrError: kobject_uevent(&dev->kobj, KOBJ_REMOVE); + glue_dir = get_glue_dir(dev); kobject_del(&dev->kobj); Error: - cleanup_device_parent(dev); + cleanup_glue_dir(dev, glue_dir); put_device(parent); name_error: kfree(dev->p); @@ -1232,6 +1249,7 @@ EXPORT_SYMBOL_GPL(put_device); void device_del(struct device *dev) { struct device *parent = dev->parent; + struct kobject *glue_dir = NULL; struct class_interface *class_intf; /* Notify clients of device removal. This call must come @@ -1276,8 +1294,9 @@ void device_del(struct device *dev) blocking_notifier_call_chain(&dev->bus->p->bus_notifier, BUS_NOTIFY_REMOVED_DEVICE, dev); kobject_uevent(&dev->kobj, KOBJ_REMOVE); - cleanup_device_parent(dev); + glue_dir = get_glue_dir(dev); kobject_del(&dev->kobj); + cleanup_glue_dir(dev, glue_dir); put_device(parent); } EXPORT_SYMBOL_GPL(device_del); diff --git a/kernel/drivers/base/dma-mapping.c b/kernel/drivers/base/dma-mapping.c index d95c5971c..a00f7b792 100644 --- a/kernel/drivers/base/dma-mapping.c +++ b/kernel/drivers/base/dma-mapping.c @@ -335,7 +335,7 @@ void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags) return; } - unmap_kernel_range((unsigned long)cpu_addr, size); + unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size)); vunmap(cpu_addr); } #endif diff --git a/kernel/drivers/base/firmware_class.c b/kernel/drivers/base/firmware_class.c index 8524450e7..ccfd26814 100644 --- a/kernel/drivers/base/firmware_class.c +++ b/kernel/drivers/base/firmware_class.c @@ -942,13 +942,14 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, timeout = MAX_JIFFY_OFFSET; } - retval = wait_for_completion_interruptible_timeout(&buf->completion, + timeout = wait_for_completion_interruptible_timeout(&buf->completion, timeout); - if (retval == -ERESTARTSYS || !retval) { + if (timeout == -ERESTARTSYS || !timeout) { + retval = timeout; mutex_lock(&fw_lock); fw_load_abort(fw_priv); mutex_unlock(&fw_lock); - } else if (retval > 0) { + } else if (timeout > 0) { retval = 0; } diff --git a/kernel/drivers/base/memory.c b/kernel/drivers/base/memory.c index 25425d3f2..48c0a1d0d 100644 --- a/kernel/drivers/base/memory.c +++ b/kernel/drivers/base/memory.c @@ -388,30 +388,29 @@ static ssize_t show_valid_zones(struct device *dev, { struct memory_block *mem = to_memory_block(dev); unsigned long start_pfn, end_pfn; + unsigned long valid_start, valid_end; unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; - struct page *first_page; struct zone *zone; start_pfn = section_nr_to_pfn(mem->start_section_nr); end_pfn = start_pfn + nr_pages; - first_page = pfn_to_page(start_pfn); /* The block contains more than one zone can not be offlined. */ - if (!test_pages_in_a_zone(start_pfn, end_pfn)) + if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, &valid_end)) return sprintf(buf, "none\n"); - zone = page_zone(first_page); + zone = page_zone(pfn_to_page(valid_start)); if (zone_idx(zone) == ZONE_MOVABLE - 1) { /*The mem block is the last memoryblock of this zone.*/ - if (end_pfn == zone_end_pfn(zone)) + if (valid_end == zone_end_pfn(zone)) return sprintf(buf, "%s %s\n", zone->name, (zone + 1)->name); } if (zone_idx(zone) == ZONE_MOVABLE) { /*The mem block is the first memoryblock of ZONE_MOVABLE.*/ - if (start_pfn == zone->zone_start_pfn) + if (valid_start == zone->zone_start_pfn) return sprintf(buf, "%s %s\n", zone->name, (zone - 1)->name); } diff --git a/kernel/drivers/base/module.c b/kernel/drivers/base/module.c index db930d3ee..2a215780e 100644 --- a/kernel/drivers/base/module.c +++ b/kernel/drivers/base/module.c @@ -24,10 +24,12 @@ static char *make_driver_name(struct device_driver *drv) static void module_create_drivers_dir(struct module_kobject *mk) { - if (!mk || mk->drivers_dir) - return; + static DEFINE_MUTEX(drivers_dir_mutex); - mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); + mutex_lock(&drivers_dir_mutex); + if (mk && !mk->drivers_dir) + mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); + mutex_unlock(&drivers_dir_mutex); } void module_add_driver(struct module *mod, struct device_driver *drv) diff --git a/kernel/drivers/base/platform.c b/kernel/drivers/base/platform.c index 176b59f5b..ba66330ce 100644 --- a/kernel/drivers/base/platform.c +++ b/kernel/drivers/base/platform.c @@ -96,7 +96,7 @@ int platform_get_irq(struct platform_device *dev, unsigned int num) int ret; ret = of_irq_get(dev->dev.of_node, num); - if (ret >= 0 || ret == -EPROBE_DEFER) + if (ret > 0 || ret == -EPROBE_DEFER) return ret; } @@ -154,7 +154,7 @@ int platform_get_irq_byname(struct platform_device *dev, const char *name) int ret; ret = of_irq_get_byname(dev->dev.of_node, name); - if (ret >= 0 || ret == -EPROBE_DEFER) + if (ret > 0 || ret == -EPROBE_DEFER) return ret; } diff --git a/kernel/drivers/base/power/domain.c b/kernel/drivers/base/power/domain.c index 65f50eccd..a48824dea 100644 --- a/kernel/drivers/base/power/domain.c +++ b/kernel/drivers/base/power/domain.c @@ -1381,7 +1381,7 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, mutex_lock(&genpd->lock); - if (!list_empty(&subdomain->slave_links) || subdomain->device_count) { + if (!list_empty(&subdomain->master_links) || subdomain->device_count) { pr_warn("%s: unable to remove subdomain %s\n", genpd->name, subdomain->name); ret = -EBUSY; diff --git a/kernel/drivers/base/power/main.c b/kernel/drivers/base/power/main.c index 1710c26ba..e9b713675 100644 --- a/kernel/drivers/base/power/main.c +++ b/kernel/drivers/base/power/main.c @@ -1022,6 +1022,8 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a TRACE_DEVICE(dev); TRACE_SUSPEND(0); + dpm_wait_for_children(dev, async); + if (async_error) goto Complete; @@ -1033,8 +1035,6 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a if (dev->power.syscore || dev->power.direct_complete) goto Complete; - dpm_wait_for_children(dev, async); - if (dev->pm_domain) { info = "noirq power domain "; callback = pm_noirq_op(&dev->pm_domain->ops, state); @@ -1169,6 +1169,8 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as __pm_runtime_disable(dev, false); + dpm_wait_for_children(dev, async); + if (async_error) goto Complete; @@ -1180,8 +1182,6 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as if (dev->power.syscore || dev->power.direct_complete) goto Complete; - dpm_wait_for_children(dev, async); - if (dev->pm_domain) { info = "late power domain "; callback = pm_late_early_op(&dev->pm_domain->ops, state); @@ -1262,14 +1262,15 @@ int dpm_suspend_late(pm_message_t state) error = device_suspend_late(dev); mutex_lock(&dpm_list_mtx); + if (!list_empty(&dev->power.entry)) + list_move(&dev->power.entry, &dpm_late_early_list); + if (error) { pm_dev_err(dev, state, " late", error); dpm_save_failed_dev(dev_name(dev)); put_device(dev); break; } - if (!list_empty(&dev->power.entry)) - list_move(&dev->power.entry, &dpm_late_early_list); put_device(dev); if (async_error) diff --git a/kernel/drivers/base/power/opp/core.c b/kernel/drivers/base/power/opp/core.c index b8e76f750..f8580900c 100644 --- a/kernel/drivers/base/power/opp/core.c +++ b/kernel/drivers/base/power/opp/core.c @@ -809,8 +809,14 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev) } opp->u_volt = microvolt[0]; - opp->u_volt_min = microvolt[1]; - opp->u_volt_max = microvolt[2]; + + if (count == 1) { + opp->u_volt_min = opp->u_volt; + opp->u_volt_max = opp->u_volt; + } else { + opp->u_volt_min = microvolt[1]; + opp->u_volt_max = microvolt[2]; + } if (!of_property_read_u32(opp->np, "opp-microamp", &val)) opp->u_amp = val; diff --git a/kernel/drivers/base/power/power.h b/kernel/drivers/base/power/power.h index 998fa6b23..01ac5b1ae 100644 --- a/kernel/drivers/base/power/power.h +++ b/kernel/drivers/base/power/power.h @@ -20,14 +20,22 @@ static inline void pm_runtime_early_init(struct device *dev) extern void pm_runtime_init(struct device *dev); extern void pm_runtime_remove(struct device *dev); +#define WAKE_IRQ_DEDICATED_ALLOCATED BIT(0) +#define WAKE_IRQ_DEDICATED_MANAGED BIT(1) +#define WAKE_IRQ_DEDICATED_MASK (WAKE_IRQ_DEDICATED_ALLOCATED | \ + WAKE_IRQ_DEDICATED_MANAGED) + struct wake_irq { struct device *dev; + unsigned int status; int irq; - bool dedicated_irq:1; }; extern void dev_pm_arm_wake_irq(struct wake_irq *wirq); extern void dev_pm_disarm_wake_irq(struct wake_irq *wirq); +extern void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status); +extern void dev_pm_disable_wake_irq_check(struct device *dev); #ifdef CONFIG_PM_SLEEP @@ -102,6 +110,15 @@ static inline void dev_pm_disarm_wake_irq(struct wake_irq *wirq) { } +static inline void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ +} + +static inline void dev_pm_disable_wake_irq_check(struct device *dev) +{ +} + #endif #ifdef CONFIG_PM_SLEEP diff --git a/kernel/drivers/base/power/runtime.c b/kernel/drivers/base/power/runtime.c index e1a10a03d..3252429f9 100644 --- a/kernel/drivers/base/power/runtime.c +++ b/kernel/drivers/base/power/runtime.c @@ -515,7 +515,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_suspend); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, true); retval = rpm_callback(callback, dev); if (retval) goto fail; @@ -554,7 +554,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) return retval; fail: - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); __update_runtime_status(dev, RPM_ACTIVE); dev->power.deferred_resume = false; wake_up_all(&dev->power.wait_queue); @@ -737,12 +737,12 @@ static int rpm_resume(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_resume); - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); retval = rpm_callback(callback, dev); if (retval) { __update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_cancel_pending(dev); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, false); } else { no_callback: __update_runtime_status(dev, RPM_ACTIVE); @@ -1468,11 +1468,16 @@ int pm_runtime_force_resume(struct device *dev) goto out; } - ret = callback(dev); + ret = pm_runtime_set_active(dev); if (ret) goto out; - pm_runtime_set_active(dev); + ret = callback(dev); + if (ret) { + pm_runtime_set_suspended(dev); + goto out; + } + pm_runtime_mark_last_busy(dev); out: pm_runtime_enable(dev); diff --git a/kernel/drivers/base/power/wakeirq.c b/kernel/drivers/base/power/wakeirq.c index 0d77cd6fd..404d94c6c 100644 --- a/kernel/drivers/base/power/wakeirq.c +++ b/kernel/drivers/base/power/wakeirq.c @@ -110,8 +110,10 @@ void dev_pm_clear_wake_irq(struct device *dev) dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); - if (wirq->dedicated_irq) + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) { free_irq(wirq->irq, wirq); + wirq->status &= ~WAKE_IRQ_DEDICATED_MASK; + } kfree(wirq); } EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq); @@ -179,7 +181,6 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) wirq->dev = dev; wirq->irq = irq; - wirq->dedicated_irq = true; irq_set_status_flags(irq, IRQ_NOAUTOEN); /* @@ -195,6 +196,8 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) if (err) goto err_free_irq; + wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED; + return err; err_free_irq: @@ -210,9 +213,9 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq); * dev_pm_enable_wake_irq - Enable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_suspend() to enable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_resume() to override the PM runtime core managed wake-up + * interrupt handling to enable the wake-up interrupt. * * Note that for runtime_suspend()) the wake-up interrupts * should be unconditionally enabled unlike for suspend() @@ -222,7 +225,7 @@ void dev_pm_enable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) enable_irq(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); @@ -231,19 +234,72 @@ EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); * dev_pm_disable_wake_irq - Disable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_resume() to disable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_suspend() to override the PM runtime core managed wake-up + * interrupt handling to disable the wake-up interrupt. */ void dev_pm_disable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) disable_irq_nosync(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_disable_wake_irq); +/** + * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt + * @dev: Device + * @can_change_status: Can change wake-up interrupt status + * + * Enables wakeirq conditionally. We need to enable wake-up interrupt + * lazily on the first rpm_suspend(). This is needed as the consumer device + * starts in RPM_SUSPENDED state, and the the first pm_runtime_get() would + * otherwise try to disable already disabled wakeirq. The wake-up interrupt + * starts disabled with IRQ_NOAUTOEN set. + * + * Should be only called from rpm_suspend() and rpm_resume() path. + * Caller must hold &dev->power.lock to change wirq->status + */ +void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) { + goto enable; + } else if (can_change_status) { + wirq->status |= WAKE_IRQ_DEDICATED_MANAGED; + goto enable; + } + + return; + +enable: + enable_irq(wirq->irq); +} + +/** + * dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt + * @dev: Device + * + * Disables wake-up interrupt conditionally based on status. + * Should be only called from rpm_suspend() and rpm_resume() path. + */ +void dev_pm_disable_wake_irq_check(struct device *dev) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) + disable_irq_nosync(wirq->irq); +} + /** * dev_pm_arm_wake_irq - Arm device wake-up * @wirq: Device wake-up interrupt diff --git a/kernel/drivers/base/regmap/regmap-spmi.c b/kernel/drivers/base/regmap/regmap-spmi.c index 7e58f6560..4a36e415e 100644 --- a/kernel/drivers/base/regmap/regmap-spmi.c +++ b/kernel/drivers/base/regmap/regmap-spmi.c @@ -142,7 +142,7 @@ static int regmap_spmi_ext_read(void *context, while (val_size) { len = min_t(size_t, val_size, 8); - err = spmi_ext_register_readl(context, addr, val, val_size); + err = spmi_ext_register_readl(context, addr, val, len); if (err) goto err_out; diff --git a/kernel/drivers/bcma/bcma_private.h b/kernel/drivers/bcma/bcma_private.h index 38f156745..71df8f2af 100644 --- a/kernel/drivers/bcma/bcma_private.h +++ b/kernel/drivers/bcma/bcma_private.h @@ -8,8 +8,6 @@ #include #include -#define BCMA_CORE_SIZE 0x1000 - #define bcma_err(bus, fmt, ...) \ pr_err("bus%d: " fmt, (bus)->num, ##__VA_ARGS__) #define bcma_warn(bus, fmt, ...) \ diff --git a/kernel/drivers/block/brd.c b/kernel/drivers/block/brd.c index a5880f4ab..1914c63ca 100644 --- a/kernel/drivers/block/brd.c +++ b/kernel/drivers/block/brd.c @@ -338,7 +338,7 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) if (unlikely(bio->bi_rw & REQ_DISCARD)) { if (sector & ((PAGE_SIZE >> SECTOR_SHIFT) - 1) || - bio->bi_iter.bi_size & PAGE_MASK) + bio->bi_iter.bi_size & ~PAGE_MASK) goto io_error; discard_from_brd(brd, sector, bio->bi_iter.bi_size); goto out; diff --git a/kernel/drivers/block/drbd/drbd_main.c b/kernel/drivers/block/drbd/drbd_main.c index 74d97f4ba..1d58854c4 100644 --- a/kernel/drivers/block/drbd/drbd_main.c +++ b/kernel/drivers/block/drbd/drbd_main.c @@ -1802,7 +1802,7 @@ int drbd_send(struct drbd_connection *connection, struct socket *sock, * do we need to block DRBD_SIG if sock == &meta.socket ?? * otherwise wake_asender() might interrupt some send_*Ack ! */ - rv = kernel_sendmsg(sock, &msg, &iov, 1, size); + rv = kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); if (rv == -EAGAIN) { if (we_should_drop_the_connection(connection, sock)) break; diff --git a/kernel/drivers/block/loop.c b/kernel/drivers/block/loop.c index 423f4ca7d..ab0b2dd3f 100644 --- a/kernel/drivers/block/loop.c +++ b/kernel/drivers/block/loop.c @@ -488,6 +488,12 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); iov_iter_bvec(&iter, ITER_BVEC | rw, bvec, bio_segments(bio), blk_rq_bytes(cmd->rq)); + /* + * This bio may be started from the middle of the 'bvec' + * because of bio splitting, so offset from the bvec must + * be passed to iov iterator + */ + iter.iov_offset = bio->bi_iter.bi_bvec_done; cmd->iocb.ki_pos = pos; cmd->iocb.ki_filp = file; @@ -1651,7 +1657,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, blk_mq_start_request(bd->rq); if (lo->lo_state != Lo_bound) - return -EIO; + return BLK_MQ_RQ_QUEUE_ERROR; if (lo->use_dio && !(cmd->rq->cmd_flags & (REQ_FLUSH | REQ_DISCARD))) diff --git a/kernel/drivers/block/mtip32xx/mtip32xx.c b/kernel/drivers/block/mtip32xx/mtip32xx.c index 3457ac8c0..55d3d1da7 100644 --- a/kernel/drivers/block/mtip32xx/mtip32xx.c +++ b/kernel/drivers/block/mtip32xx/mtip32xx.c @@ -173,7 +173,13 @@ static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) { struct request *rq; + if (mtip_check_surprise_removal(dd->pdev)) + return NULL; + rq = blk_mq_alloc_request(dd->queue, 0, __GFP_RECLAIM, true); + if (IS_ERR(rq)) + return NULL; + return blk_mq_rq_to_pdu(rq); } @@ -233,15 +239,9 @@ static void mtip_async_complete(struct mtip_port *port, "Command tag %d failed due to TFE\n", tag); } - /* Unmap the DMA scatter list entries */ - dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, cmd->direction); - rq = mtip_rq_from_tag(dd, tag); - if (unlikely(cmd->unaligned)) - up(&port->cmd_slot_unal); - - blk_mq_end_request(rq, status ? -EIO : 0); + blk_mq_complete_request(rq, status); } /* @@ -581,6 +581,8 @@ static void mtip_completion(struct mtip_port *port, dev_warn(&port->dd->pdev->dev, "Internal command %d completed with TFE\n", tag); + command->comp_func = NULL; + command->comp_data = NULL; complete(waiting); } @@ -618,8 +620,6 @@ static void mtip_handle_tfe(struct driver_data *dd) port = dd->port; - set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); - if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); @@ -628,7 +628,7 @@ static void mtip_handle_tfe(struct driver_data *dd) cmd->comp_func(port, MTIP_TAG_INTERNAL, cmd, PORT_IRQ_TF_ERR); } - goto handle_tfe_exit; + return; } /* clear the tag accumulator */ @@ -701,7 +701,7 @@ static void mtip_handle_tfe(struct driver_data *dd) fail_reason = "thermal shutdown"; } if (buf[288] == 0xBF) { - set_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); + set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); dev_info(&dd->pdev->dev, "Drive indicates rebuild has failed. Secure erase required.\n"); fail_all_ncq_cmds = 1; @@ -771,11 +771,6 @@ static void mtip_handle_tfe(struct driver_data *dd) } } print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); - -handle_tfe_exit: - /* clear eh_active */ - clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); - wake_up_interruptible(&port->svc_wait); } /* @@ -1007,6 +1002,7 @@ static bool mtip_pause_ncq(struct mtip_port *port, (fis->features == 0x27 || fis->features == 0x72 || fis->features == 0x62 || fis->features == 0x26))) { clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); + clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); /* Com reset after secure erase or lowlevel format */ mtip_restart_port(port); clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); @@ -1021,12 +1017,14 @@ static bool mtip_pause_ncq(struct mtip_port *port, * * @port Pointer to port data structure * @timeout Max duration to wait (ms) + * @atomic gfp_t flag to indicate blockable context or not * * return value * 0 Success * -EBUSY Commands still active */ -static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) +static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout, + gfp_t atomic) { unsigned long to; unsigned int n; @@ -1037,16 +1035,21 @@ static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) to = jiffies + msecs_to_jiffies(timeout); do { if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && - test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { + test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags) && + atomic == GFP_KERNEL) { msleep(20); continue; /* svc thd is actively issuing commands */ } - msleep(100); + if (atomic == GFP_KERNEL) + msleep(100); + else { + cpu_relax(); + udelay(100); + } + if (mtip_check_surprise_removal(port->dd->pdev)) goto err_fault; - if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) - goto err_fault; /* * Ignore s_active bit 0 of array element 0. @@ -1099,6 +1102,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, struct mtip_cmd *int_cmd; struct driver_data *dd = port->dd; int rv = 0; + unsigned long start; /* Make sure the buffer is 8 byte aligned. This is asic specific. */ if (buffer & 0x00000007) { @@ -1107,6 +1111,10 @@ static int mtip_exec_internal_command(struct mtip_port *port, } int_cmd = mtip_get_int_command(dd); + if (!int_cmd) { + dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n"); + return -EFAULT; + } set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); @@ -1119,7 +1127,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, if (fis->command != ATA_CMD_STANDBYNOW1) { /* wait for io to complete if non atomic */ if (mtip_quiesce_io(port, - MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) { + MTIP_QUIESCE_IO_TIMEOUT_MS, atomic) < 0) { dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); mtip_put_int_command(dd, int_cmd); @@ -1162,6 +1170,8 @@ static int mtip_exec_internal_command(struct mtip_port *port, /* Populate the command header */ int_cmd->command_header->byte_count = 0; + start = jiffies; + /* Issue the command to the hardware */ mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); @@ -1170,10 +1180,12 @@ static int mtip_exec_internal_command(struct mtip_port *port, if ((rv = wait_for_completion_interruptible_timeout( &wait, msecs_to_jiffies(timeout))) <= 0) { + if (rv == -ERESTARTSYS) { /* interrupted */ dev_err(&dd->pdev->dev, - "Internal command [%02X] was interrupted after %lu ms\n", - fis->command, timeout); + "Internal command [%02X] was interrupted after %u ms\n", + fis->command, + jiffies_to_msecs(jiffies - start)); rv = -EINTR; goto exec_ic_exit; } else if (rv == 0) /* timeout */ @@ -2897,6 +2909,42 @@ static int mtip_ftl_rebuild_poll(struct driver_data *dd) return -EFAULT; } +static void mtip_softirq_done_fn(struct request *rq) +{ + struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); + struct driver_data *dd = rq->q->queuedata; + + /* Unmap the DMA scatter list entries */ + dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, + cmd->direction); + + if (unlikely(cmd->unaligned)) + up(&dd->port->cmd_slot_unal); + + blk_mq_end_request(rq, rq->errors); +} + +static void mtip_abort_cmd(struct request *req, void *data, + bool reserved) +{ + struct driver_data *dd = data; + + dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag); + + clear_bit(req->tag, dd->port->cmds_to_issue); + req->errors = -EIO; + mtip_softirq_done_fn(req); +} + +static void mtip_queue_cmd(struct request *req, void *data, + bool reserved) +{ + struct driver_data *dd = data; + + set_bit(req->tag, dd->port->cmds_to_issue); + blk_abort_request(req); +} + /* * service thread to issue queued commands * @@ -2909,7 +2957,7 @@ static int mtip_ftl_rebuild_poll(struct driver_data *dd) static int mtip_service_thread(void *data) { struct driver_data *dd = (struct driver_data *)data; - unsigned long slot, slot_start, slot_wrap; + unsigned long slot, slot_start, slot_wrap, to; unsigned int num_cmd_slots = dd->slot_groups * 32; struct mtip_port *port = dd->port; @@ -2924,9 +2972,7 @@ static int mtip_service_thread(void *data) * is in progress nor error handling is active */ wait_event_interruptible(port->svc_wait, (port->flags) && - !(port->flags & MTIP_PF_PAUSE_IO)); - - set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); + (port->flags & MTIP_PF_SVC_THD_WORK)); if (kthread_should_stop() || test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) @@ -2936,6 +2982,8 @@ static int mtip_service_thread(void *data) &dd->dd_flag))) goto st_out; + set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); + restart_eh: /* Demux bits: start with error handling */ if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { @@ -2946,6 +2994,32 @@ restart_eh: if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) goto restart_eh; + if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { + to = jiffies + msecs_to_jiffies(5000); + + do { + mdelay(100); + } while (atomic_read(&dd->irq_workers_active) != 0 && + time_before(jiffies, to)); + + if (atomic_read(&dd->irq_workers_active) != 0) + dev_warn(&dd->pdev->dev, + "Completion workers still active!"); + + spin_lock(dd->queue->queue_lock); + blk_mq_all_tag_busy_iter(*dd->tags.tags, + mtip_queue_cmd, dd); + spin_unlock(dd->queue->queue_lock); + + set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); + + if (mtip_device_reset(dd)) + blk_mq_all_tag_busy_iter(*dd->tags.tags, + mtip_abort_cmd, dd); + + clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); + } + if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { slot = 1; /* used to restrict the loop to one iteration */ @@ -2978,10 +3052,8 @@ restart_eh: } if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { - if (mtip_ftl_rebuild_poll(dd) < 0) - set_bit(MTIP_DDF_REBUILD_FAILED_BIT, - &dd->dd_flag); - clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); + if (mtip_ftl_rebuild_poll(dd) == 0) + clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); } } @@ -3096,7 +3168,7 @@ static int mtip_hw_get_identify(struct driver_data *dd) if (buf[288] == 0xBF) { dev_info(&dd->pdev->dev, "Drive indicates rebuild has failed.\n"); - /* TODO */ + set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); } } @@ -3270,20 +3342,25 @@ out1: return rv; } -static void mtip_standby_drive(struct driver_data *dd) +static int mtip_standby_drive(struct driver_data *dd) { - if (dd->sr) - return; + int rv = 0; + if (dd->sr || !dd->port) + return -ENODEV; /* * Send standby immediate (E0h) to the drive so that it * saves its state. */ if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && - !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) - if (mtip_standby_immediate(dd->port)) + !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && + !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { + rv = mtip_standby_immediate(dd->port); + if (rv) dev_warn(&dd->pdev->dev, "STANDBY IMMEDIATE failed\n"); + } + return rv; } /* @@ -3296,10 +3373,6 @@ static void mtip_standby_drive(struct driver_data *dd) */ static int mtip_hw_exit(struct driver_data *dd) { - /* - * Send standby immediate (E0h) to the drive so that it - * saves its state. - */ if (!dd->sr) { /* de-initialize the port. */ mtip_deinit_port(dd->port); @@ -3341,8 +3414,7 @@ static int mtip_hw_shutdown(struct driver_data *dd) * Send standby immediate (E0h) to the drive so that it * saves its state. */ - if (!dd->sr && dd->port) - mtip_standby_immediate(dd->port); + mtip_standby_drive(dd); return 0; } @@ -3365,7 +3437,7 @@ static int mtip_hw_suspend(struct driver_data *dd) * Send standby immediate (E0h) to the drive * so that it saves its state. */ - if (mtip_standby_immediate(dd->port) != 0) { + if (mtip_standby_drive(dd) != 0) { dev_err(&dd->pdev->dev, "Failed standby-immediate command\n"); return -EFAULT; @@ -3603,6 +3675,28 @@ static int mtip_block_getgeo(struct block_device *dev, return 0; } +static int mtip_block_open(struct block_device *dev, fmode_t mode) +{ + struct driver_data *dd; + + if (dev && dev->bd_disk) { + dd = (struct driver_data *) dev->bd_disk->private_data; + + if (dd) { + if (test_bit(MTIP_DDF_REMOVAL_BIT, + &dd->dd_flag)) { + return -ENODEV; + } + return 0; + } + } + return -ENODEV; +} + +void mtip_block_release(struct gendisk *disk, fmode_t mode) +{ +} + /* * Block device operation function. * @@ -3610,6 +3704,8 @@ static int mtip_block_getgeo(struct block_device *dev, * layer. */ static const struct block_device_operations mtip_block_ops = { + .open = mtip_block_open, + .release = mtip_block_release, .ioctl = mtip_block_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = mtip_block_compat_ioctl, @@ -3671,10 +3767,9 @@ static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) rq_data_dir(rq))) { return -ENODATA; } - if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))) + if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag) || + test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))) return -ENODATA; - if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) - return -ENXIO; } if (rq->cmd_flags & REQ_DISCARD) { @@ -3786,11 +3881,33 @@ static int mtip_init_cmd(void *data, struct request *rq, unsigned int hctx_idx, return 0; } +static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req, + bool reserved) +{ + struct driver_data *dd = req->q->queuedata; + int ret = BLK_EH_RESET_TIMER; + + if (reserved) + goto exit_handler; + + if (test_bit(req->tag, dd->port->cmds_to_issue)) + goto exit_handler; + + if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) + goto exit_handler; + + wake_up_interruptible(&dd->port->svc_wait); +exit_handler: + return ret; +} + static struct blk_mq_ops mtip_mq_ops = { .queue_rq = mtip_queue_rq, .map_queue = blk_mq_map_queue, .init_request = mtip_init_cmd, .exit_request = mtip_free_cmd, + .complete = mtip_softirq_done_fn, + .timeout = mtip_cmd_timeout, }; /* @@ -3857,7 +3974,6 @@ static int mtip_block_initialize(struct driver_data *dd) mtip_hw_debugfs_init(dd); -skip_create_disk: memset(&dd->tags, 0, sizeof(dd->tags)); dd->tags.ops = &mtip_mq_ops; dd->tags.nr_hw_queues = 1; @@ -3867,12 +3983,13 @@ skip_create_disk: dd->tags.numa_node = dd->numa_node; dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; dd->tags.driver_data = dd; + dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; rv = blk_mq_alloc_tag_set(&dd->tags); if (rv) { dev_err(&dd->pdev->dev, "Unable to allocate request queue\n"); - goto block_queue_alloc_init_error; + goto block_queue_alloc_tag_error; } /* Allocate the request queue. */ @@ -3887,6 +4004,7 @@ skip_create_disk: dd->disk->queue = dd->queue; dd->queue->queuedata = dd; +skip_create_disk: /* Initialize the protocol layer. */ wait_for_rebuild = mtip_hw_get_identify(dd); if (wait_for_rebuild < 0) { @@ -3983,8 +4101,9 @@ kthread_run_error: read_capacity_error: init_hw_cmds_error: blk_cleanup_queue(dd->queue); - blk_mq_free_tag_set(&dd->tags); block_queue_alloc_init_error: + blk_mq_free_tag_set(&dd->tags); +block_queue_alloc_tag_error: mtip_hw_debugfs_exit(dd); disk_index_error: spin_lock(&rssd_index_lock); @@ -4001,6 +4120,22 @@ protocol_init_error: return rv; } +static void mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv) +{ + struct driver_data *dd = (struct driver_data *)data; + struct mtip_cmd *cmd; + + if (likely(!reserv)) + blk_mq_complete_request(rq, -ENODEV); + else if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &dd->port->flags)) { + + cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); + if (cmd->comp_func) + cmd->comp_func(dd->port, MTIP_TAG_INTERNAL, + cmd, -ENODEV); + } +} + /* * Block layer deinitialization function. * @@ -4032,12 +4167,23 @@ static int mtip_block_remove(struct driver_data *dd) } } - if (!dd->sr) - mtip_standby_drive(dd); + if (!dd->sr) { + /* + * Explicitly wait here for IOs to quiesce, + * as mtip_standby_drive usually won't wait for IOs. + */ + if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS, + GFP_KERNEL)) + mtip_standby_drive(dd); + } else dev_info(&dd->pdev->dev, "device %s surprise removal\n", dd->disk->disk_name); + blk_mq_freeze_queue_start(dd->queue); + blk_mq_stop_hw_queues(dd->queue); + blk_mq_all_tag_busy_iter(dd->tags.tags[0], mtip_no_dev_cleanup, dd); + /* * Delete our gendisk structure. This also removes the device * from /dev @@ -4047,7 +4193,8 @@ static int mtip_block_remove(struct driver_data *dd) dd->bdev = NULL; } if (dd->disk) { - del_gendisk(dd->disk); + if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) + del_gendisk(dd->disk); if (dd->disk->queue) { blk_cleanup_queue(dd->queue); blk_mq_free_tag_set(&dd->tags); @@ -4088,7 +4235,8 @@ static int mtip_block_shutdown(struct driver_data *dd) dev_info(&dd->pdev->dev, "Shutting down %s ...\n", dd->disk->disk_name); - del_gendisk(dd->disk); + if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) + del_gendisk(dd->disk); if (dd->disk->queue) { blk_cleanup_queue(dd->queue); blk_mq_free_tag_set(&dd->tags); @@ -4433,7 +4581,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) struct driver_data *dd = pci_get_drvdata(pdev); unsigned long flags, to; - set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag); spin_lock_irqsave(&dev_lock, flags); list_del_init(&dd->online_list); @@ -4450,12 +4598,17 @@ static void mtip_pci_remove(struct pci_dev *pdev) } while (atomic_read(&dd->irq_workers_active) != 0 && time_before(jiffies, to)); + if (!dd->sr) + fsync_bdev(dd->bdev); + if (atomic_read(&dd->irq_workers_active) != 0) { dev_warn(&dd->pdev->dev, "Completion workers still active!\n"); } - blk_mq_stop_hw_queues(dd->queue); + blk_set_queue_dying(dd->queue); + set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + /* Clean up the block layer. */ mtip_block_remove(dd); diff --git a/kernel/drivers/block/mtip32xx/mtip32xx.h b/kernel/drivers/block/mtip32xx/mtip32xx.h index 327478400..7617888f7 100644 --- a/kernel/drivers/block/mtip32xx/mtip32xx.h +++ b/kernel/drivers/block/mtip32xx/mtip32xx.h @@ -134,16 +134,24 @@ enum { MTIP_PF_EH_ACTIVE_BIT = 1, /* error handling */ MTIP_PF_SE_ACTIVE_BIT = 2, /* secure erase */ MTIP_PF_DM_ACTIVE_BIT = 3, /* download microcde */ + MTIP_PF_TO_ACTIVE_BIT = 9, /* timeout handling */ MTIP_PF_PAUSE_IO = ((1 << MTIP_PF_IC_ACTIVE_BIT) | (1 << MTIP_PF_EH_ACTIVE_BIT) | (1 << MTIP_PF_SE_ACTIVE_BIT) | - (1 << MTIP_PF_DM_ACTIVE_BIT)), + (1 << MTIP_PF_DM_ACTIVE_BIT) | + (1 << MTIP_PF_TO_ACTIVE_BIT)), MTIP_PF_SVC_THD_ACTIVE_BIT = 4, MTIP_PF_ISSUE_CMDS_BIT = 5, MTIP_PF_REBUILD_BIT = 6, MTIP_PF_SVC_THD_STOP_BIT = 8, + MTIP_PF_SVC_THD_WORK = ((1 << MTIP_PF_EH_ACTIVE_BIT) | + (1 << MTIP_PF_ISSUE_CMDS_BIT) | + (1 << MTIP_PF_REBUILD_BIT) | + (1 << MTIP_PF_SVC_THD_STOP_BIT) | + (1 << MTIP_PF_TO_ACTIVE_BIT)), + /* below are bit numbers in 'dd_flag' defined in driver_data */ MTIP_DDF_SEC_LOCK_BIT = 0, MTIP_DDF_REMOVE_PENDING_BIT = 1, @@ -153,6 +161,7 @@ enum { MTIP_DDF_RESUME_BIT = 6, MTIP_DDF_INIT_DONE_BIT = 7, MTIP_DDF_REBUILD_FAILED_BIT = 8, + MTIP_DDF_REMOVAL_BIT = 9, MTIP_DDF_STOP_IO = ((1 << MTIP_DDF_REMOVE_PENDING_BIT) | (1 << MTIP_DDF_SEC_LOCK_BIT) | diff --git a/kernel/drivers/block/nbd.c b/kernel/drivers/block/nbd.c index 93b3f99b6..8f1ce6d57 100644 --- a/kernel/drivers/block/nbd.c +++ b/kernel/drivers/block/nbd.c @@ -618,8 +618,8 @@ static void nbd_request_handler(struct request_queue *q) req, req->cmd_type); if (unlikely(!nbd->sock)) { - dev_err(disk_to_dev(nbd->disk), - "Attempted send on closed socket\n"); + dev_err_ratelimited(disk_to_dev(nbd->disk), + "Attempted send on closed socket\n"); req->errors++; nbd_end_request(nbd, req); spin_lock_irq(q->queue_lock); diff --git a/kernel/drivers/block/paride/pd.c b/kernel/drivers/block/paride/pd.c index 562b5a4ca..78a39f736 100644 --- a/kernel/drivers/block/paride/pd.c +++ b/kernel/drivers/block/paride/pd.c @@ -126,7 +126,7 @@ */ #include -static bool verbose = 0; +static int verbose = 0; static int major = PD_MAJOR; static char *name = PD_NAME; static int cluster = 64; @@ -161,7 +161,7 @@ enum {D_PRT, D_PRO, D_UNI, D_MOD, D_GEO, D_SBY, D_DLY, D_SLV}; static DEFINE_MUTEX(pd_mutex); static DEFINE_SPINLOCK(pd_lock); -module_param(verbose, bool, 0); +module_param(verbose, int, 0); module_param(major, int, 0); module_param(name, charp, 0); module_param(cluster, int, 0); diff --git a/kernel/drivers/block/paride/pt.c b/kernel/drivers/block/paride/pt.c index 1740d75e8..216a94fed 100644 --- a/kernel/drivers/block/paride/pt.c +++ b/kernel/drivers/block/paride/pt.c @@ -117,7 +117,7 @@ */ -static bool verbose = 0; +static int verbose = 0; static int major = PT_MAJOR; static char *name = PT_NAME; static int disable = 0; @@ -152,7 +152,7 @@ static int (*drives[4])[6] = {&drive0, &drive1, &drive2, &drive3}; #include -module_param(verbose, bool, 0); +module_param(verbose, int, 0); module_param(major, int, 0); module_param(name, charp, 0); module_param_array(drive0, int, NULL, 0); diff --git a/kernel/drivers/block/rbd.c b/kernel/drivers/block/rbd.c index 81ea69fee..fbdddd6f9 100644 --- a/kernel/drivers/block/rbd.c +++ b/kernel/drivers/block/rbd.c @@ -1955,7 +1955,7 @@ static struct ceph_osd_request *rbd_osd_req_create( osdc = &rbd_dev->rbd_client->client->osdc; osd_req = ceph_osdc_alloc_request(osdc, snapc, num_ops, false, - GFP_ATOMIC); + GFP_NOIO); if (!osd_req) return NULL; /* ENOMEM */ @@ -2004,7 +2004,7 @@ rbd_osd_req_create_copyup(struct rbd_obj_request *obj_request) rbd_dev = img_request->rbd_dev; osdc = &rbd_dev->rbd_client->client->osdc; osd_req = ceph_osdc_alloc_request(osdc, snapc, num_osd_ops, - false, GFP_ATOMIC); + false, GFP_NOIO); if (!osd_req) return NULL; /* ENOMEM */ @@ -2506,7 +2506,7 @@ static int rbd_img_request_fill(struct rbd_img_request *img_request, bio_chain_clone_range(&bio_list, &bio_offset, clone_size, - GFP_ATOMIC); + GFP_NOIO); if (!obj_request->bio_list) goto out_unwind; } else if (type == OBJ_REQUEST_PAGES) { diff --git a/kernel/drivers/block/zram/zram_drv.c b/kernel/drivers/block/zram/zram_drv.c index 65e0b375a..4e43cea8f 100644 --- a/kernel/drivers/block/zram/zram_drv.c +++ b/kernel/drivers/block/zram/zram_drv.c @@ -1370,7 +1370,8 @@ static ssize_t hot_remove_store(struct class *class, zram = idr_find(&zram_index_idr, dev_id); if (zram) { ret = zram_remove(zram); - idr_remove(&zram_index_idr, dev_id); + if (!ret) + idr_remove(&zram_index_idr, dev_id); } else { ret = -ENODEV; } @@ -1379,8 +1380,14 @@ static ssize_t hot_remove_store(struct class *class, return ret ? ret : count; } +/* + * NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a + * sense that reading from this file does alter the state of your system -- it + * creates a new un-initialized zram device and returns back this device's + * device_id (or an error code if it fails to create a new device). + */ static struct class_attribute zram_control_class_attrs[] = { - __ATTR_RO(hot_add), + __ATTR(hot_add, 0400, hot_add_show, NULL), __ATTR_WO(hot_remove), __ATTR_NULL, }; diff --git a/kernel/drivers/bluetooth/ath3k.c b/kernel/drivers/bluetooth/ath3k.c index fa893c3ec..0beaa52df 100644 --- a/kernel/drivers/bluetooth/ath3k.c +++ b/kernel/drivers/bluetooth/ath3k.c @@ -82,6 +82,7 @@ static const struct usb_device_id ath3k_table[] = { { USB_DEVICE(0x0489, 0xe05f) }, { USB_DEVICE(0x0489, 0xe076) }, { USB_DEVICE(0x0489, 0xe078) }, + { USB_DEVICE(0x0489, 0xe095) }, { USB_DEVICE(0x04c5, 0x1330) }, { USB_DEVICE(0x04CA, 0x3004) }, { USB_DEVICE(0x04CA, 0x3005) }, @@ -92,6 +93,7 @@ static const struct usb_device_id ath3k_table[] = { { USB_DEVICE(0x04CA, 0x300d) }, { USB_DEVICE(0x04CA, 0x300f) }, { USB_DEVICE(0x04CA, 0x3010) }, + { USB_DEVICE(0x04CA, 0x3014) }, { USB_DEVICE(0x0930, 0x0219) }, { USB_DEVICE(0x0930, 0x021c) }, { USB_DEVICE(0x0930, 0x0220) }, @@ -113,10 +115,12 @@ static const struct usb_device_id ath3k_table[] = { { USB_DEVICE(0x13d3, 0x3362) }, { USB_DEVICE(0x13d3, 0x3375) }, { USB_DEVICE(0x13d3, 0x3393) }, + { USB_DEVICE(0x13d3, 0x3395) }, { USB_DEVICE(0x13d3, 0x3402) }, { USB_DEVICE(0x13d3, 0x3408) }, { USB_DEVICE(0x13d3, 0x3423) }, { USB_DEVICE(0x13d3, 0x3432) }, + { USB_DEVICE(0x13d3, 0x3472) }, { USB_DEVICE(0x13d3, 0x3474) }, /* Atheros AR5BBU12 with sflash firmware */ @@ -144,6 +148,7 @@ static const struct usb_device_id ath3k_blist_tbl[] = { { USB_DEVICE(0x0489, 0xe05f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe076), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe078), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x0489, 0xe095), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04c5, 0x1330), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3004), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3005), .driver_info = BTUSB_ATH3012 }, @@ -154,6 +159,7 @@ static const struct usb_device_id ath3k_blist_tbl[] = { { USB_DEVICE(0x04ca, 0x300d), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x300f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3010), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x04ca, 0x3014), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0219), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x021c), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0220), .driver_info = BTUSB_ATH3012 }, @@ -175,10 +181,12 @@ static const struct usb_device_id ath3k_blist_tbl[] = { { USB_DEVICE(0x13d3, 0x3362), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3375), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3393), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3395), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3402), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3408), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3423), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3432), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 }, /* Atheros AR5BBU22 with sflash firmware */ diff --git a/kernel/drivers/bluetooth/btusb.c b/kernel/drivers/bluetooth/btusb.c index 968897108..c306b483d 100644 --- a/kernel/drivers/bluetooth/btusb.c +++ b/kernel/drivers/bluetooth/btusb.c @@ -196,6 +196,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0489, 0xe05f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe076), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe078), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x0489, 0xe095), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04c5, 0x1330), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3004), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3005), .driver_info = BTUSB_ATH3012 }, @@ -206,6 +207,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x04ca, 0x300d), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x300f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3010), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x04ca, 0x3014), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0219), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x021c), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0220), .driver_info = BTUSB_ATH3012 }, @@ -227,10 +229,12 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x13d3, 0x3362), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3375), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3393), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3395), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3402), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3408), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3423), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3432), .driver_info = BTUSB_ATH3012 }, + { USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 }, /* Atheros AR5BBU12 with sflash firmware */ @@ -2052,12 +2056,13 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) return -EINVAL; } - /* At the moment only the hardware variant iBT 3.0 (LnP/SfP) is - * supported by this firmware loading method. This check has been - * put in place to ensure correct forward compatibility options - * when newer hardware variants come along. + /* At the moment the iBT 3.0 hardware variants 0x0b (LnP/SfP) + * and 0x0c (WsP) are supported by this firmware loading method. + * + * This check has been put in place to ensure correct forward + * compatibility options when newer hardware variants come along. */ - if (ver->hw_variant != 0x0b) { + if (ver->hw_variant != 0x0b && ver->hw_variant != 0x0c) { BT_ERR("%s: Unsupported Intel hardware variant (%u)", hdev->name, ver->hw_variant); kfree_skb(skb); diff --git a/kernel/drivers/bluetooth/hci_intel.c b/kernel/drivers/bluetooth/hci_intel.c index 4a414a5a3..b9065506a 100644 --- a/kernel/drivers/bluetooth/hci_intel.c +++ b/kernel/drivers/bluetooth/hci_intel.c @@ -1234,8 +1234,7 @@ static int intel_probe(struct platform_device *pdev) idev->pdev = pdev; - idev->reset = devm_gpiod_get_optional(&pdev->dev, "reset", - GPIOD_OUT_LOW); + idev->reset = devm_gpiod_get(&pdev->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(idev->reset)) { dev_err(&pdev->dev, "Unable to retrieve gpio\n"); return PTR_ERR(idev->reset); @@ -1247,8 +1246,7 @@ static int intel_probe(struct platform_device *pdev) dev_err(&pdev->dev, "No IRQ, falling back to gpio-irq\n"); - host_wake = devm_gpiod_get_optional(&pdev->dev, "host-wake", - GPIOD_IN); + host_wake = devm_gpiod_get(&pdev->dev, "host-wake", GPIOD_IN); if (IS_ERR(host_wake)) { dev_err(&pdev->dev, "Unable to retrieve IRQ\n"); goto no_irq; diff --git a/kernel/drivers/bluetooth/hci_vhci.c b/kernel/drivers/bluetooth/hci_vhci.c index ed888e302..597b2d16b 100644 --- a/kernel/drivers/bluetooth/hci_vhci.c +++ b/kernel/drivers/bluetooth/hci_vhci.c @@ -50,6 +50,7 @@ struct vhci_data { wait_queue_head_t read_wait; struct sk_buff_head readq; + struct mutex open_mutex; struct delayed_work open_timeout; }; @@ -87,12 +88,15 @@ static int vhci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) return 0; } -static int vhci_create_device(struct vhci_data *data, __u8 opcode) +static int __vhci_create_device(struct vhci_data *data, __u8 opcode) { struct hci_dev *hdev; struct sk_buff *skb; __u8 dev_type; + if (data->hdev) + return -EBADFD; + /* bits 0-1 are dev_type (BR/EDR or AMP) */ dev_type = opcode & 0x03; @@ -151,6 +155,17 @@ static int vhci_create_device(struct vhci_data *data, __u8 opcode) return 0; } +static int vhci_create_device(struct vhci_data *data, __u8 opcode) +{ + int err; + + mutex_lock(&data->open_mutex); + err = __vhci_create_device(data, opcode); + mutex_unlock(&data->open_mutex); + + return err; +} + static inline ssize_t vhci_get_user(struct vhci_data *data, struct iov_iter *from) { @@ -189,11 +204,6 @@ static inline ssize_t vhci_get_user(struct vhci_data *data, break; case HCI_VENDOR_PKT: - if (data->hdev) { - kfree_skb(skb); - return -EBADFD; - } - cancel_delayed_work_sync(&data->open_timeout); opcode = *((__u8 *) skb->data); @@ -320,6 +330,7 @@ static int vhci_open(struct inode *inode, struct file *file) skb_queue_head_init(&data->readq); init_waitqueue_head(&data->read_wait); + mutex_init(&data->open_mutex); INIT_DELAYED_WORK(&data->open_timeout, vhci_open_timeout); file->private_data = data; @@ -333,15 +344,18 @@ static int vhci_open(struct inode *inode, struct file *file) static int vhci_release(struct inode *inode, struct file *file) { struct vhci_data *data = file->private_data; - struct hci_dev *hdev = data->hdev; + struct hci_dev *hdev; cancel_delayed_work_sync(&data->open_timeout); + hdev = data->hdev; + if (hdev) { hci_unregister_dev(hdev); hci_free_dev(hdev); } + skb_queue_purge(&data->readq); file->private_data = NULL; kfree(data); diff --git a/kernel/drivers/bus/arm-ccn.c b/kernel/drivers/bus/arm-ccn.c index 7082c7268..0f54cb7dd 100644 --- a/kernel/drivers/bus/arm-ccn.c +++ b/kernel/drivers/bus/arm-ccn.c @@ -187,6 +187,7 @@ struct arm_ccn { struct arm_ccn_component *xp; struct arm_ccn_dt dt; + int mn_id; }; @@ -326,6 +327,7 @@ struct arm_ccn_pmu_event { static ssize_t arm_ccn_pmu_event_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev)); struct arm_ccn_pmu_event *event = container_of(attr, struct arm_ccn_pmu_event, attr); ssize_t res; @@ -352,6 +354,9 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev, res += snprintf(buf + res, PAGE_SIZE - res, ",cmp_l=?,cmp_h=?,mask=?"); break; + case CCN_TYPE_MN: + res += snprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id); + break; default: res += snprintf(buf + res, PAGE_SIZE - res, ",node=?"); break; @@ -381,9 +386,9 @@ static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj, } static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = { - CCN_EVENT_MN(eobarrier, "dir=0,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE), - CCN_EVENT_MN(ecbarrier, "dir=0,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE), - CCN_EVENT_MN(dvmop, "dir=0,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE), + CCN_EVENT_MN(eobarrier, "dir=1,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE), + CCN_EVENT_MN(ecbarrier, "dir=1,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE), + CCN_EVENT_MN(dvmop, "dir=1,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE), CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY), CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY), CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY), @@ -757,6 +762,12 @@ static int arm_ccn_pmu_event_init(struct perf_event *event) /* Validate node/xp vs topology */ switch (type) { + case CCN_TYPE_MN: + if (node_xp != ccn->mn_id) { + dev_warn(ccn->dev, "Invalid MN ID %d!\n", node_xp); + return -EINVAL; + } + break; case CCN_TYPE_XP: if (node_xp >= ccn->num_xps) { dev_warn(ccn->dev, "Invalid XP ID %d!\n", node_xp); @@ -884,6 +895,10 @@ static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable) struct arm_ccn_component *xp; u32 val, dt_cfg; + /* Nothing to do for cycle counter */ + if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) + return; + if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)]; else @@ -986,7 +1001,7 @@ static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event) /* Comparison values */ writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp)); - writel((cmp_l >> 32) & 0xefffffff, + writel((cmp_l >> 32) & 0x7fffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4); writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp)); writel((cmp_h >> 32) & 0x0fffffff, @@ -994,7 +1009,7 @@ static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event) /* Mask */ writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp)); - writel((mask_l >> 32) & 0xefffffff, + writel((mask_l >> 32) & 0x7fffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4); writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp)); writel((mask_h >> 32) & 0x0fffffff, @@ -1368,6 +1383,8 @@ static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region, switch (type) { case CCN_TYPE_MN: + ccn->mn_id = id; + return 0; case CCN_TYPE_DT: return 0; case CCN_TYPE_XP: diff --git a/kernel/drivers/bus/imx-weim.c b/kernel/drivers/bus/imx-weim.c index e98d15eaa..1827fc4d1 100644 --- a/kernel/drivers/bus/imx-weim.c +++ b/kernel/drivers/bus/imx-weim.c @@ -150,7 +150,7 @@ static int __init weim_parse_dt(struct platform_device *pdev, return ret; } - for_each_child_of_node(pdev->dev.of_node, child) { + for_each_available_child_of_node(pdev->dev.of_node, child) { if (!child->name) continue; diff --git a/kernel/drivers/bus/vexpress-config.c b/kernel/drivers/bus/vexpress-config.c index 6575c0fe6..27ea64fa4 100644 --- a/kernel/drivers/bus/vexpress-config.c +++ b/kernel/drivers/bus/vexpress-config.c @@ -171,6 +171,7 @@ static int vexpress_config_populate(struct device_node *node) { struct device_node *bridge; struct device *parent; + int ret; bridge = of_parse_phandle(node, "arm,vexpress,config-bridge", 0); if (!bridge) @@ -181,7 +182,11 @@ static int vexpress_config_populate(struct device_node *node) if (WARN_ON(!parent)) return -ENODEV; - return of_platform_populate(node, NULL, NULL, parent); + ret = of_platform_populate(node, NULL, NULL, parent); + + put_device(parent); + + return ret; } static int __init vexpress_config_init(void) diff --git a/kernel/drivers/char/hw_random/core.c b/kernel/drivers/char/hw_random/core.c index 6f497aa1b..cf2502057 100644 --- a/kernel/drivers/char/hw_random/core.c +++ b/kernel/drivers/char/hw_random/core.c @@ -84,14 +84,14 @@ static size_t rng_buffer_size(void) static void add_early_randomness(struct hwrng *rng) { - unsigned char bytes[16]; int bytes_read; + size_t size = min_t(size_t, 16, rng_buffer_size()); mutex_lock(&reading_mutex); - bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1); + bytes_read = rng_get_data(rng, rng_buffer, size, 1); mutex_unlock(&reading_mutex); if (bytes_read > 0) - add_device_randomness(bytes, bytes_read); + add_device_randomness(rng_buffer, bytes_read); } static inline void cleanup_rng(struct kref *kref) diff --git a/kernel/drivers/char/hw_random/exynos-rng.c b/kernel/drivers/char/hw_random/exynos-rng.c index 30cf46231..7845a38b6 100644 --- a/kernel/drivers/char/hw_random/exynos-rng.c +++ b/kernel/drivers/char/hw_random/exynos-rng.c @@ -89,6 +89,7 @@ static int exynos_read(struct hwrng *rng, void *buf, struct exynos_rng, rng); u32 *data = buf; int retry = 100; + int ret = 4; pm_runtime_get_sync(exynos_rng->dev); @@ -97,23 +98,27 @@ static int exynos_read(struct hwrng *rng, void *buf, while (!(exynos_rng_readl(exynos_rng, EXYNOS_PRNG_STATUS_OFFSET) & PRNG_DONE) && --retry) cpu_relax(); - if (!retry) - return -ETIMEDOUT; + if (!retry) { + ret = -ETIMEDOUT; + goto out; + } exynos_rng_writel(exynos_rng, PRNG_DONE, EXYNOS_PRNG_STATUS_OFFSET); *data = exynos_rng_readl(exynos_rng, EXYNOS_PRNG_OUT1_OFFSET); +out: pm_runtime_mark_last_busy(exynos_rng->dev); pm_runtime_put_sync_autosuspend(exynos_rng->dev); - return 4; + return ret; } static int exynos_rng_probe(struct platform_device *pdev) { struct exynos_rng *exynos_rng; struct resource *res; + int ret; exynos_rng = devm_kzalloc(&pdev->dev, sizeof(struct exynos_rng), GFP_KERNEL); @@ -141,7 +146,13 @@ static int exynos_rng_probe(struct platform_device *pdev) pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_enable(&pdev->dev); - return devm_hwrng_register(&pdev->dev, &exynos_rng->rng); + ret = devm_hwrng_register(&pdev->dev, &exynos_rng->rng); + if (ret) { + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); + } + + return ret; } #ifdef CONFIG_PM diff --git a/kernel/drivers/char/hw_random/omap-rng.c b/kernel/drivers/char/hw_random/omap-rng.c index 8a1432e8b..f5c26a5f6 100644 --- a/kernel/drivers/char/hw_random/omap-rng.c +++ b/kernel/drivers/char/hw_random/omap-rng.c @@ -384,7 +384,12 @@ static int omap_rng_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(&pdev->dev); + goto err_ioremap; + } ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : get_omap_rng_device_details(priv); @@ -435,8 +440,15 @@ static int __maybe_unused omap_rng_suspend(struct device *dev) static int __maybe_unused omap_rng_resume(struct device *dev) { struct omap_rng_dev *priv = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(dev); + return ret; + } - pm_runtime_get_sync(dev); priv->pdata->init(priv); return 0; diff --git a/kernel/drivers/char/ipmi/ipmi_msghandler.c b/kernel/drivers/char/ipmi/ipmi_msghandler.c index e3536da05..a084a4751 100644 --- a/kernel/drivers/char/ipmi/ipmi_msghandler.c +++ b/kernel/drivers/char/ipmi/ipmi_msghandler.c @@ -3819,6 +3819,7 @@ static void handle_new_recv_msgs(ipmi_smi_t intf) while (!list_empty(&intf->waiting_rcv_msgs)) { smi_msg = list_entry(intf->waiting_rcv_msgs.next, struct ipmi_smi_msg, link); + list_del(&smi_msg->link); if (!run_to_completion) spin_unlock_irqrestore(&intf->waiting_rcv_msgs_lock, flags); @@ -3828,11 +3829,14 @@ static void handle_new_recv_msgs(ipmi_smi_t intf) if (rv > 0) { /* * To preserve message order, quit if we - * can't handle a message. + * can't handle a message. Add the message + * back at the head, this is safe because this + * tasklet is the only thing that pulls the + * messages. */ + list_add(&smi_msg->link, &intf->waiting_rcv_msgs); break; } else { - list_del(&smi_msg->link); if (rv == 0) /* Message handled */ ipmi_free_smi_msg(smi_msg); diff --git a/kernel/drivers/char/random.c b/kernel/drivers/char/random.c index 29abd5fa5..cf69b6b42 100644 --- a/kernel/drivers/char/random.c +++ b/kernel/drivers/char/random.c @@ -722,15 +722,18 @@ retry: } } -static void credit_entropy_bits_safe(struct entropy_store *r, int nbits) +static int credit_entropy_bits_safe(struct entropy_store *r, int nbits) { const int nbits_max = (int)(~0U >> (ENTROPY_SHIFT + 1)); + if (nbits < 0) + return -EINVAL; + /* Cap the value to avoid overflows */ nbits = min(nbits, nbits_max); - nbits = max(nbits, -nbits_max); credit_entropy_bits(r, nbits); + return 0; } /********************************************************************* @@ -941,6 +944,7 @@ void add_interrupt_randomness(int irq, int irq_flags, __u64 ip) /* award one bit for the contents of the fast pool */ credit_entropy_bits(r, credit + 1); } +EXPORT_SYMBOL_GPL(add_interrupt_randomness); #ifdef CONFIG_BLOCK void add_disk_randomness(struct gendisk *disk) @@ -1453,12 +1457,16 @@ random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { + static int maxwarn = 10; int ret; - if (unlikely(nonblocking_pool.initialized == 0)) - printk_once(KERN_NOTICE "random: %s urandom read " - "with %d bits of entropy available\n", - current->comm, nonblocking_pool.entropy_total); + if (unlikely(nonblocking_pool.initialized == 0) && + maxwarn > 0) { + maxwarn--; + printk(KERN_NOTICE "random: %s: uninitialized urandom read " + "(%zd bytes read, %d bits of entropy available)\n", + current->comm, nbytes, nonblocking_pool.entropy_total); + } nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); ret = extract_entropy_user(&nonblocking_pool, buf, nbytes); @@ -1538,8 +1546,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) return -EPERM; if (get_user(ent_count, p)) return -EFAULT; - credit_entropy_bits_safe(&input_pool, ent_count); - return 0; + return credit_entropy_bits_safe(&input_pool, ent_count); case RNDADDENTROPY: if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1553,8 +1560,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) size); if (retval < 0) return retval; - credit_entropy_bits_safe(&input_pool, ent_count); - return 0; + return credit_entropy_bits_safe(&input_pool, ent_count); case RNDZAPENTCNT: case RNDCLEARPOOL: /* @@ -1842,12 +1848,18 @@ void add_hwgenerator_randomness(const char *buffer, size_t count, { struct entropy_store *poolp = &input_pool; - /* Suspend writing if we're above the trickle threshold. - * We'll be woken up again once below random_write_wakeup_thresh, - * or when the calling thread is about to terminate. - */ - wait_event_interruptible(random_write_wait, kthread_should_stop() || + if (unlikely(nonblocking_pool.initialized == 0)) + poolp = &nonblocking_pool; + else { + /* Suspend writing if we're above the trickle + * threshold. We'll be woken up again once below + * random_write_wakeup_thresh, or when the calling + * thread is about to terminate. + */ + wait_event_interruptible(random_write_wait, + kthread_should_stop() || ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); + } mix_pool_bytes(poolp, buffer, count); credit_entropy_bits(poolp, entropy); } diff --git a/kernel/drivers/char/tpm/tpm-chip.c b/kernel/drivers/char/tpm/tpm-chip.c index 45cc39aab..252142524 100644 --- a/kernel/drivers/char/tpm/tpm-chip.c +++ b/kernel/drivers/char/tpm/tpm-chip.c @@ -136,11 +136,13 @@ struct tpm_chip *tpmm_chip_alloc(struct device *dev, chip->cdev.owner = chip->pdev->driver->owner; chip->cdev.kobj.parent = &chip->dev.kobj; + devm_add_action(dev, (void (*)(void *)) put_device, &chip->dev); + return chip; } EXPORT_SYMBOL_GPL(tpmm_chip_alloc); -static int tpm_dev_add_device(struct tpm_chip *chip) +static int tpm_add_char_device(struct tpm_chip *chip) { int rc; @@ -151,7 +153,6 @@ static int tpm_dev_add_device(struct tpm_chip *chip) chip->devname, MAJOR(chip->dev.devt), MINOR(chip->dev.devt), rc); - device_unregister(&chip->dev); return rc; } @@ -162,16 +163,17 @@ static int tpm_dev_add_device(struct tpm_chip *chip) chip->devname, MAJOR(chip->dev.devt), MINOR(chip->dev.devt), rc); + cdev_del(&chip->cdev); return rc; } return rc; } -static void tpm_dev_del_device(struct tpm_chip *chip) +static void tpm_del_char_device(struct tpm_chip *chip) { cdev_del(&chip->cdev); - device_unregister(&chip->dev); + device_del(&chip->dev); } static int tpm1_chip_register(struct tpm_chip *chip) @@ -222,7 +224,7 @@ int tpm_chip_register(struct tpm_chip *chip) tpm_add_ppi(chip); - rc = tpm_dev_add_device(chip); + rc = tpm_add_char_device(chip); if (rc) goto out_err; @@ -274,6 +276,6 @@ void tpm_chip_unregister(struct tpm_chip *chip) sysfs_remove_link(&chip->pdev->kobj, "ppi"); tpm1_chip_unregister(chip); - tpm_dev_del_device(chip); + tpm_del_char_device(chip); } EXPORT_SYMBOL_GPL(tpm_chip_unregister); diff --git a/kernel/drivers/char/tpm/tpm-dev.c b/kernel/drivers/char/tpm/tpm-dev.c index de0337ebd..4f3137d9a 100644 --- a/kernel/drivers/char/tpm/tpm-dev.c +++ b/kernel/drivers/char/tpm/tpm-dev.c @@ -139,7 +139,7 @@ static ssize_t tpm_write(struct file *file, const char __user *buf, /* atomic tpm command send and result receive */ out_size = tpm_transmit(priv->chip, priv->data_buffer, - sizeof(priv->data_buffer)); + sizeof(priv->data_buffer), 0); if (out_size < 0) { mutex_unlock(&priv->buffer_mutex); return out_size; diff --git a/kernel/drivers/char/tpm/tpm-interface.c b/kernel/drivers/char/tpm/tpm-interface.c index c50637db3..17abe52e6 100644 --- a/kernel/drivers/char/tpm/tpm-interface.c +++ b/kernel/drivers/char/tpm/tpm-interface.c @@ -328,8 +328,8 @@ EXPORT_SYMBOL_GPL(tpm_calc_ordinal_duration); /* * Internal kernel interface to transmit TPM commands */ -ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, - size_t bufsiz) +ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz, + unsigned int flags) { ssize_t rc; u32 count, ordinal; @@ -348,7 +348,8 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, return -E2BIG; } - mutex_lock(&chip->tpm_mutex); + if (!(flags & TPM_TRANSMIT_UNLOCKED)) + mutex_lock(&chip->tpm_mutex); rc = chip->ops->send(chip, (u8 *) buf, count); if (rc < 0) { @@ -391,20 +392,21 @@ out_recv: dev_err(chip->pdev, "tpm_transmit: tpm_recv: error %zd\n", rc); out: - mutex_unlock(&chip->tpm_mutex); + if (!(flags & TPM_TRANSMIT_UNLOCKED)) + mutex_unlock(&chip->tpm_mutex); return rc; } #define TPM_DIGEST_SIZE 20 #define TPM_RET_CODE_IDX 6 -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, - int len, const char *desc) +ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, + int len, unsigned int flags, const char *desc) { - struct tpm_output_header *header; + const struct tpm_output_header *header; int err; - len = tpm_transmit(chip, (u8 *) cmd, len); + len = tpm_transmit(chip, (const u8 *)cmd, len, flags); if (len < 0) return len; else if (len < TPM_HEADER_SIZE) @@ -452,7 +454,8 @@ ssize_t tpm_getcap(struct device *dev, __be32 subcap_id, cap_t *cap, tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = subcap_id; } - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, desc); + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, + desc); if (!rc) *cap = tpm_cmd.params.getcap_out.cap; return rc; @@ -468,7 +471,7 @@ void tpm_gen_interrupt(struct tpm_chip *chip) tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to determine the timeouts"); } EXPORT_SYMBOL_GPL(tpm_gen_interrupt); @@ -489,7 +492,7 @@ static int tpm_startup(struct tpm_chip *chip, __be16 startup_type) start_cmd.header.in = tpm_startup_header; start_cmd.params.startup_in.startup_type = startup_type; - return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE, + return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to start the TPM"); } @@ -505,7 +508,8 @@ int tpm_get_timeouts(struct tpm_chip *chip) tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP; tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, NULL); + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, + NULL); if (rc == TPM_ERR_INVALID_POSTINIT) { /* The TPM is not started, we are the first to talk to it. @@ -519,7 +523,7 @@ int tpm_get_timeouts(struct tpm_chip *chip) tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, - NULL); + 0, NULL); } if (rc) { dev_err(chip->pdev, @@ -580,7 +584,7 @@ duration: tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_DURATION; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to determine the durations"); if (rc) return rc; @@ -636,7 +640,7 @@ static int tpm_continue_selftest(struct tpm_chip *chip) struct tpm_cmd_t cmd; cmd.header.in = continue_selftest_header; - rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE, 0, "continue selftest"); return rc; } @@ -656,7 +660,7 @@ int tpm_pcr_read_dev(struct tpm_chip *chip, int pcr_idx, u8 *res_buf) cmd.header.in = pcrread_header; cmd.params.pcrread_in.pcr_idx = cpu_to_be32(pcr_idx); - rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE, 0, "attempting to read a pcr value"); if (rc == 0) @@ -754,7 +758,7 @@ int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash) cmd.header.in = pcrextend_header; cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(pcr_idx); memcpy(cmd.params.pcrextend_in.hash, hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0, "attempting extend a PCR value"); tpm_chip_put(chip); @@ -793,7 +797,7 @@ int tpm_do_selftest(struct tpm_chip *chip) /* Attempt to read a PCR value */ cmd.header.in = pcrread_header; cmd.params.pcrread_in.pcr_idx = cpu_to_be32(0); - rc = tpm_transmit(chip, (u8 *) &cmd, READ_PCR_RESULT_SIZE); + rc = tpm_transmit(chip, (u8 *) &cmd, READ_PCR_RESULT_SIZE, 0); /* Some buggy TPMs will not respond to tpm_tis_ready() for * around 300ms while the self test is ongoing, keep trying * until the self test duration expires. */ @@ -834,7 +838,7 @@ int tpm_send(u32 chip_num, void *cmd, size_t buflen) if (chip == NULL) return -ENODEV; - rc = tpm_transmit_cmd(chip, cmd, buflen, "attempting tpm_cmd"); + rc = tpm_transmit_cmd(chip, cmd, buflen, 0, "attempting tpm_cmd"); tpm_chip_put(chip); return rc; @@ -936,14 +940,15 @@ int tpm_pm_suspend(struct device *dev) cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(tpm_suspend_pcr); memcpy(cmd.params.pcrextend_in.hash, dummy_hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0, "extending dummy pcr before suspend"); } /* now do the actual savestate */ for (try = 0; try < TPM_RETRY; try++) { cmd.header.in = savestate_header; - rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, NULL); + rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, 0, + NULL); /* * If the TPM indicates that it is too busy to respond to @@ -1027,8 +1032,8 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max) tpm_cmd.params.getrandom_in.num_bytes = cpu_to_be32(num_bytes); err = tpm_transmit_cmd(chip, &tpm_cmd, - TPM_GETRANDOM_RESULT_SIZE + num_bytes, - "attempting get random"); + TPM_GETRANDOM_RESULT_SIZE + num_bytes, + 0, "attempting get random"); if (err) break; diff --git a/kernel/drivers/char/tpm/tpm-sysfs.c b/kernel/drivers/char/tpm/tpm-sysfs.c index ee66fd467..f880856aa 100644 --- a/kernel/drivers/char/tpm/tpm-sysfs.c +++ b/kernel/drivers/char/tpm/tpm-sysfs.c @@ -39,7 +39,7 @@ static ssize_t pubek_show(struct device *dev, struct device_attribute *attr, struct tpm_chip *chip = dev_get_drvdata(dev); tpm_cmd.header.in = tpm_readpubek_header; - err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE, + err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE, 0, "attempting to read the PUBEK"); if (err) goto out; diff --git a/kernel/drivers/char/tpm/tpm.h b/kernel/drivers/char/tpm/tpm.h index a4257a329..2216861f8 100644 --- a/kernel/drivers/char/tpm/tpm.h +++ b/kernel/drivers/char/tpm/tpm.h @@ -498,11 +498,15 @@ extern struct class *tpm_class; extern dev_t tpm_devt; extern const struct file_operations tpm_fops; +enum tpm_transmit_flags { + TPM_TRANSMIT_UNLOCKED = BIT(0), +}; + +ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz, + unsigned int flags); +ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, int len, + unsigned int flags, const char *desc); ssize_t tpm_getcap(struct device *, __be32, cap_t *, const char *); -ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, - size_t bufsiz); -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, int len, - const char *desc); extern int tpm_get_timeouts(struct tpm_chip *); extern void tpm_gen_interrupt(struct tpm_chip *); extern int tpm_do_selftest(struct tpm_chip *); diff --git a/kernel/drivers/char/tpm/tpm2-cmd.c b/kernel/drivers/char/tpm/tpm2-cmd.c index c12130485..cb7e4f6b7 100644 --- a/kernel/drivers/char/tpm/tpm2-cmd.c +++ b/kernel/drivers/char/tpm/tpm2-cmd.c @@ -264,7 +264,7 @@ int tpm2_pcr_read(struct tpm_chip *chip, int pcr_idx, u8 *res_buf) sizeof(cmd.params.pcrread_in.pcr_select)); cmd.params.pcrread_in.pcr_select[pcr_idx >> 3] = 1 << (pcr_idx & 0x7); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting to read a pcr value"); if (rc == 0) { buf = cmd.params.pcrread_out.digest; @@ -312,7 +312,7 @@ int tpm2_pcr_extend(struct tpm_chip *chip, int pcr_idx, const u8 *hash) cmd.params.pcrextend_in.hash_alg = cpu_to_be16(TPM2_ALG_SHA1); memcpy(cmd.params.pcrextend_in.digest, hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting extend a PCR value"); return rc; @@ -358,7 +358,7 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *out, size_t max) cmd.header.in = tpm2_getrandom_header; cmd.params.getrandom_in.size = cpu_to_be16(num_bytes); - err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting get random"); if (err) break; @@ -416,12 +416,12 @@ static void tpm2_buf_append_auth(struct tpm_buf *buf, u32 session_handle, } /** - * tpm2_seal_trusted() - seal a trusted key - * @chip_num: A specific chip number for the request or TPM_ANY_NUM - * @options: authentication values and other options + * tpm2_seal_trusted() - seal the payload of a trusted key + * @chip_num: TPM chip to use * @payload: the key data in clear and encrypted form + * @options: authentication values and other options * - * Returns < 0 on error and 0 on success. + * Return: < 0 on error and 0 on success. */ int tpm2_seal_trusted(struct tpm_chip *chip, struct trusted_key_payload *payload, @@ -472,7 +472,7 @@ int tpm2_seal_trusted(struct tpm_chip *chip, goto out; } - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "sealing data"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, 0, "sealing data"); if (rc) goto out; @@ -494,10 +494,18 @@ out: return rc; } -static int tpm2_load(struct tpm_chip *chip, - struct trusted_key_payload *payload, - struct trusted_key_options *options, - u32 *blob_handle) +/** + * tpm2_load_cmd() - execute a TPM2_Load command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static int tpm2_load_cmd(struct tpm_chip *chip, + struct trusted_key_payload *payload, + struct trusted_key_options *options, + u32 *blob_handle, unsigned int flags) { struct tpm_buf buf; unsigned int private_len; @@ -532,7 +540,7 @@ static int tpm2_load(struct tpm_chip *chip, goto out; } - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "loading blob"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "loading blob"); if (!rc) *blob_handle = be32_to_cpup( (__be32 *) &buf.data[TPM_HEADER_SIZE]); @@ -546,7 +554,16 @@ out: return rc; } -static void tpm2_flush_context(struct tpm_chip *chip, u32 handle) +/** + * tpm2_flush_context_cmd() - execute a TPM2_FlushContext command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static void tpm2_flush_context_cmd(struct tpm_chip *chip, u32 handle, + unsigned int flags) { struct tpm_buf buf; int rc; @@ -560,7 +577,8 @@ static void tpm2_flush_context(struct tpm_chip *chip, u32 handle) tpm_buf_append_u32(&buf, handle); - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "flushing context"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, + "flushing context"); if (rc) dev_warn(chip->pdev, "0x%08x was not flushed, rc=%d\n", handle, rc); @@ -568,10 +586,18 @@ static void tpm2_flush_context(struct tpm_chip *chip, u32 handle) tpm_buf_destroy(&buf); } -static int tpm2_unseal(struct tpm_chip *chip, - struct trusted_key_payload *payload, - struct trusted_key_options *options, - u32 blob_handle) +/** + * tpm2_unseal_cmd() - execute a TPM2_Unload command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static int tpm2_unseal_cmd(struct tpm_chip *chip, + struct trusted_key_payload *payload, + struct trusted_key_options *options, + u32 blob_handle, unsigned int flags) { struct tpm_buf buf; u16 data_len; @@ -589,7 +615,7 @@ static int tpm2_unseal(struct tpm_chip *chip, options->blobauth /* hmac */, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "unsealing"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "unsealing"); if (rc > 0) rc = -EPERM; @@ -608,12 +634,12 @@ static int tpm2_unseal(struct tpm_chip *chip, } /** - * tpm_unseal_trusted() - unseal a trusted key - * @chip_num: A specific chip number for the request or TPM_ANY_NUM - * @options: authentication values and other options + * tpm_unseal_trusted() - unseal the payload of a trusted key + * @chip_num: TPM chip to use * @payload: the key data in clear and encrypted form + * @options: authentication values and other options * - * Returns < 0 on error and 0 on success. + * Return: < 0 on error and 0 on success. */ int tpm2_unseal_trusted(struct tpm_chip *chip, struct trusted_key_payload *payload, @@ -622,14 +648,17 @@ int tpm2_unseal_trusted(struct tpm_chip *chip, u32 blob_handle; int rc; - rc = tpm2_load(chip, payload, options, &blob_handle); + mutex_lock(&chip->tpm_mutex); + rc = tpm2_load_cmd(chip, payload, options, &blob_handle, + TPM_TRANSMIT_UNLOCKED); if (rc) - return rc; - - rc = tpm2_unseal(chip, payload, options, blob_handle); - - tpm2_flush_context(chip, blob_handle); + goto out; + rc = tpm2_unseal_cmd(chip, payload, options, blob_handle, + TPM_TRANSMIT_UNLOCKED); + tpm2_flush_context_cmd(chip, blob_handle, TPM_TRANSMIT_UNLOCKED); +out: + mutex_unlock(&chip->tpm_mutex); return rc; } @@ -655,9 +684,9 @@ ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id, u32 *value, cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(property_id); cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), desc); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, desc); if (!rc) - *value = cmd.params.get_tpm_pt_out.value; + *value = be32_to_cpu(cmd.params.get_tpm_pt_out.value); return rc; } @@ -689,7 +718,7 @@ int tpm2_startup(struct tpm_chip *chip, u16 startup_type) cmd.header.in = tpm2_startup_header; cmd.params.startup_in.startup_type = cpu_to_be16(startup_type); - return tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + return tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting to start the TPM"); } EXPORT_SYMBOL_GPL(tpm2_startup); @@ -718,7 +747,7 @@ void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type) cmd.header.in = tpm2_shutdown_header; cmd.params.startup_in.startup_type = cpu_to_be16(shutdown_type); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), "stopping the TPM"); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "stopping the TPM"); /* In places where shutdown command is sent there's no much we can do * except print the error code on a system failure. @@ -784,7 +813,7 @@ static int tpm2_start_selftest(struct tpm_chip *chip, bool full) cmd.header.in = tpm2_selftest_header; cmd.params.selftest_in.full_test = full; - rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE, 0, "continue selftest"); /* At least some prototype chips seem to give RC_TESTING error @@ -836,7 +865,7 @@ int tpm2_do_selftest(struct tpm_chip *chip) cmd.params.pcrread_in.pcr_select[1] = 0x00; cmd.params.pcrread_in.pcr_select[2] = 0x00; - rc = tpm_transmit_cmd(chip, (u8 *) &cmd, sizeof(cmd), NULL); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, NULL); if (rc < 0) break; @@ -885,7 +914,7 @@ int tpm2_probe(struct tpm_chip *chip) cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(0x100); cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1); - rc = tpm_transmit(chip, (const char *) &cmd, sizeof(cmd)); + rc = tpm_transmit(chip, (const u8 *)&cmd, sizeof(cmd), 0); if (rc < 0) return rc; else if (rc < TPM_HEADER_SIZE) diff --git a/kernel/drivers/char/tpm/tpm_crb.c b/kernel/drivers/char/tpm/tpm_crb.c index 4bb9727c1..2b21398c3 100644 --- a/kernel/drivers/char/tpm/tpm_crb.c +++ b/kernel/drivers/char/tpm/tpm_crb.c @@ -149,6 +149,11 @@ static int crb_send(struct tpm_chip *chip, u8 *buf, size_t len) struct crb_priv *priv = chip->vendor.priv; int rc = 0; + /* Zero the cancel register so that the next command will not get + * canceled. + */ + iowrite32(0, &priv->cca->cancel); + if (len > le32_to_cpu(ioread32(&priv->cca->cmd_size))) { dev_err(&chip->dev, "invalid command count value %x %zx\n", @@ -182,8 +187,6 @@ static void crb_cancel(struct tpm_chip *chip) if ((priv->flags & CRB_FL_ACPI_START) && crb_do_acpi_start(chip)) dev_err(&chip->dev, "ACPI Start failed\n"); - - iowrite32(0, &priv->cca->cancel); } static bool crb_req_canceled(struct tpm_chip *chip, u8 status) @@ -310,11 +313,11 @@ static int crb_acpi_remove(struct acpi_device *device) struct device *dev = &device->dev; struct tpm_chip *chip = dev_get_drvdata(dev); - tpm_chip_unregister(chip); - if (chip->flags & TPM_CHIP_FLAG_TPM2) tpm2_shutdown(chip, TPM2_SU_CLEAR); + tpm_chip_unregister(chip); + return 0; } diff --git a/kernel/drivers/char/tpm/tpm_eventlog.c b/kernel/drivers/char/tpm/tpm_eventlog.c index bd72fb042..4e6940acf 100644 --- a/kernel/drivers/char/tpm/tpm_eventlog.c +++ b/kernel/drivers/char/tpm/tpm_eventlog.c @@ -232,7 +232,7 @@ static int tpm_binary_bios_measurements_show(struct seq_file *m, void *v) { struct tcpa_event *event = v; struct tcpa_event temp_event; - char *tempPtr; + char *temp_ptr; int i; memcpy(&temp_event, event, sizeof(struct tcpa_event)); @@ -242,10 +242,16 @@ static int tpm_binary_bios_measurements_show(struct seq_file *m, void *v) temp_event.event_type = do_endian_conversion(event->event_type); temp_event.event_size = do_endian_conversion(event->event_size); - tempPtr = (char *)&temp_event; + temp_ptr = (char *) &temp_event; - for (i = 0; i < sizeof(struct tcpa_event) + temp_event.event_size; i++) - seq_putc(m, tempPtr[i]); + for (i = 0; i < (sizeof(struct tcpa_event) - 1) ; i++) + seq_putc(m, temp_ptr[i]); + + temp_ptr = (char *) v; + + for (i = (sizeof(struct tcpa_event) - 1); + i < (sizeof(struct tcpa_event) + temp_event.event_size); i++) + seq_putc(m, temp_ptr[i]); return 0; diff --git a/kernel/drivers/char/tpm/xen-tpmfront.c b/kernel/drivers/char/tpm/xen-tpmfront.c index 3111f2778..849f2e29c 100644 --- a/kernel/drivers/char/tpm/xen-tpmfront.c +++ b/kernel/drivers/char/tpm/xen-tpmfront.c @@ -305,7 +305,6 @@ static int tpmfront_probe(struct xenbus_device *dev, rv = setup_ring(dev, priv); if (rv) { chip = dev_get_drvdata(&dev->dev); - tpm_chip_unregister(chip); ring_free(priv); return rv; } diff --git a/kernel/drivers/char/virtio_console.c b/kernel/drivers/char/virtio_console.c index d2406fe25..090183f81 100644 --- a/kernel/drivers/char/virtio_console.c +++ b/kernel/drivers/char/virtio_console.c @@ -1533,19 +1533,29 @@ static void remove_port_data(struct port *port) spin_lock_irq(&port->inbuf_lock); /* Remove unused data this port might have received. */ discard_port_data(port); + spin_unlock_irq(&port->inbuf_lock); /* Remove buffers we queued up for the Host to send us data in. */ - while ((buf = virtqueue_detach_unused_buf(port->in_vq))) - free_buf(buf, true); - spin_unlock_irq(&port->inbuf_lock); + do { + spin_lock_irq(&port->inbuf_lock); + buf = virtqueue_detach_unused_buf(port->in_vq); + spin_unlock_irq(&port->inbuf_lock); + if (buf) + free_buf(buf, true); + } while (buf); spin_lock_irq(&port->outvq_lock); reclaim_consumed_buffers(port); + spin_unlock_irq(&port->outvq_lock); /* Free pending buffers from the out-queue. */ - while ((buf = virtqueue_detach_unused_buf(port->out_vq))) - free_buf(buf, true); - spin_unlock_irq(&port->outvq_lock); + do { + spin_lock_irq(&port->outvq_lock); + buf = virtqueue_detach_unused_buf(port->out_vq); + spin_unlock_irq(&port->outvq_lock); + if (buf) + free_buf(buf, true); + } while (buf); } /* diff --git a/kernel/drivers/clk/at91/clk-h32mx.c b/kernel/drivers/clk/at91/clk-h32mx.c index 819f5842f..8e20c8a76 100644 --- a/kernel/drivers/clk/at91/clk-h32mx.c +++ b/kernel/drivers/clk/at91/clk-h32mx.c @@ -114,7 +114,7 @@ static void __init of_sama5d4_clk_h32mx_setup(struct device_node *np) h32mxclk->regmap = regmap; clk = clk_register(NULL, &h32mxclk->hw); - if (!clk) { + if (IS_ERR(clk)) { kfree(h32mxclk); return; } diff --git a/kernel/drivers/clk/bcm/clk-bcm2835.c b/kernel/drivers/clk/bcm/clk-bcm2835.c index 39bf58202..35ab89fe9 100644 --- a/kernel/drivers/clk/bcm/clk-bcm2835.c +++ b/kernel/drivers/clk/bcm/clk-bcm2835.c @@ -890,8 +890,14 @@ static void bcm2835_pll_off(struct clk_hw *hw) struct bcm2835_cprman *cprman = pll->cprman; const struct bcm2835_pll_data *data = pll->data; - cprman_write(cprman, data->cm_ctrl_reg, CM_PLL_ANARST); - cprman_write(cprman, data->a2w_ctrl_reg, A2W_PLL_CTRL_PWRDN); + spin_lock(&cprman->regs_lock); + cprman_write(cprman, data->cm_ctrl_reg, + cprman_read(cprman, data->cm_ctrl_reg) | + CM_PLL_ANARST); + cprman_write(cprman, data->a2w_ctrl_reg, + cprman_read(cprman, data->a2w_ctrl_reg) | + A2W_PLL_CTRL_PWRDN); + spin_unlock(&cprman->regs_lock); } static int bcm2835_pll_on(struct clk_hw *hw) @@ -901,6 +907,10 @@ static int bcm2835_pll_on(struct clk_hw *hw) const struct bcm2835_pll_data *data = pll->data; ktime_t timeout; + cprman_write(cprman, data->a2w_ctrl_reg, + cprman_read(cprman, data->a2w_ctrl_reg) & + ~A2W_PLL_CTRL_PWRDN); + /* Take the PLL out of reset. */ cprman_write(cprman, data->cm_ctrl_reg, cprman_read(cprman, data->cm_ctrl_reg) & ~CM_PLL_ANARST); @@ -1068,10 +1078,14 @@ static void bcm2835_pll_divider_off(struct clk_hw *hw) struct bcm2835_cprman *cprman = divider->cprman; const struct bcm2835_pll_divider_data *data = divider->data; + spin_lock(&cprman->regs_lock); cprman_write(cprman, data->cm_reg, (cprman_read(cprman, data->cm_reg) & ~data->load_mask) | data->hold_mask); - cprman_write(cprman, data->a2w_reg, A2W_PLL_CHANNEL_DISABLE); + cprman_write(cprman, data->a2w_reg, + cprman_read(cprman, data->a2w_reg) | + A2W_PLL_CHANNEL_DISABLE); + spin_unlock(&cprman->regs_lock); } static int bcm2835_pll_divider_on(struct clk_hw *hw) @@ -1080,12 +1094,14 @@ static int bcm2835_pll_divider_on(struct clk_hw *hw) struct bcm2835_cprman *cprman = divider->cprman; const struct bcm2835_pll_divider_data *data = divider->data; + spin_lock(&cprman->regs_lock); cprman_write(cprman, data->a2w_reg, cprman_read(cprman, data->a2w_reg) & ~A2W_PLL_CHANNEL_DISABLE); cprman_write(cprman, data->cm_reg, cprman_read(cprman, data->cm_reg) & ~data->hold_mask); + spin_unlock(&cprman->regs_lock); return 0; } @@ -1097,13 +1113,15 @@ static int bcm2835_pll_divider_set_rate(struct clk_hw *hw, struct bcm2835_pll_divider *divider = bcm2835_pll_divider_from_hw(hw); struct bcm2835_cprman *cprman = divider->cprman; const struct bcm2835_pll_divider_data *data = divider->data; - u32 cm; - int ret; + u32 cm, div, max_div = 1 << A2W_PLL_DIV_BITS; - ret = clk_divider_ops.set_rate(hw, rate, parent_rate); - if (ret) - return ret; + div = DIV_ROUND_UP_ULL(parent_rate, rate); + + div = min(div, max_div); + if (div == max_div) + div = 0; + cprman_write(cprman, data->a2w_reg, div); cm = cprman_read(cprman, data->cm_reg); cprman_write(cprman, data->cm_reg, cm | data->load_mask); cprman_write(cprman, data->cm_reg, cm & ~data->load_mask); @@ -1165,8 +1183,9 @@ static u32 bcm2835_clock_choose_div(struct clk_hw *hw, div &= ~unused_frac_mask; } - /* Clamp to the limits. */ - div = max(div, unused_frac_mask + 1); + /* clamp to min divider of 1 */ + div = max_t(u32, div, 1 << CM_DIV_FRAC_BITS); + /* clamp to the highest possible fractional divider */ div = min_t(u32, div, GENMASK(data->int_bits + CM_DIV_FRAC_BITS - 1, CM_DIV_FRAC_BITS - data->frac_bits)); diff --git a/kernel/drivers/clk/clk-divider.c b/kernel/drivers/clk/clk-divider.c index 3ace102a2..ac9582de6 100644 --- a/kernel/drivers/clk/clk-divider.c +++ b/kernel/drivers/clk/clk-divider.c @@ -354,7 +354,7 @@ static long clk_divider_round_rate(struct clk_hw *hw, unsigned long rate, /* if read only, just return current value */ if (divider->flags & CLK_DIVIDER_READ_ONLY) { - bestdiv = readl(divider->reg) >> divider->shift; + bestdiv = clk_readl(divider->reg) >> divider->shift; bestdiv &= div_mask(divider->width); bestdiv = _get_div(divider->table, bestdiv, divider->flags, divider->width); @@ -422,6 +422,12 @@ const struct clk_ops clk_divider_ops = { }; EXPORT_SYMBOL_GPL(clk_divider_ops); +const struct clk_ops clk_divider_ro_ops = { + .recalc_rate = clk_divider_recalc_rate, + .round_rate = clk_divider_round_rate, +}; +EXPORT_SYMBOL_GPL(clk_divider_ro_ops); + static struct clk *_register_divider(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 shift, u8 width, @@ -445,7 +451,10 @@ static struct clk *_register_divider(struct device *dev, const char *name, return ERR_PTR(-ENOMEM); init.name = name; - init.ops = &clk_divider_ops; + if (clk_divider_flags & CLK_DIVIDER_READ_ONLY) + init.ops = &clk_divider_ro_ops; + else + init.ops = &clk_divider_ops; init.flags = flags | CLK_IS_BASIC; init.parent_names = (parent_name ? &parent_name: NULL); init.num_parents = (parent_name ? 1 : 0); diff --git a/kernel/drivers/clk/clk-qoriq.c b/kernel/drivers/clk/clk-qoriq.c index 7bc1c4527..a5070f9cb 100644 --- a/kernel/drivers/clk/clk-qoriq.c +++ b/kernel/drivers/clk/clk-qoriq.c @@ -700,6 +700,7 @@ static struct clk * __init create_mux_common(struct clockgen *cg, struct mux_hwclock *hwc, const struct clk_ops *ops, unsigned long min_rate, + unsigned long max_rate, unsigned long pct80_rate, const char *fmt, int idx) { @@ -728,6 +729,8 @@ static struct clk * __init create_mux_common(struct clockgen *cg, continue; if (rate < min_rate) continue; + if (rate > max_rate) + continue; parent_names[j] = div->name; hwc->parent_to_clksel[j] = i; @@ -759,14 +762,18 @@ static struct clk * __init create_one_cmux(struct clockgen *cg, int idx) struct mux_hwclock *hwc; const struct clockgen_pll_div *div; unsigned long plat_rate, min_rate; - u64 pct80_rate; + u64 max_rate, pct80_rate; u32 clksel; hwc = kzalloc(sizeof(*hwc), GFP_KERNEL); if (!hwc) return NULL; - hwc->reg = cg->regs + 0x20 * idx; + if (cg->info.flags & CG_VER3) + hwc->reg = cg->regs + 0x70000 + 0x20 * idx; + else + hwc->reg = cg->regs + 0x20 * idx; + hwc->info = cg->info.cmux_groups[cg->info.cmux_to_group[idx]]; /* @@ -783,8 +790,8 @@ static struct clk * __init create_one_cmux(struct clockgen *cg, int idx) return NULL; } - pct80_rate = clk_get_rate(div->clk); - pct80_rate *= 8; + max_rate = clk_get_rate(div->clk); + pct80_rate = max_rate * 8; do_div(pct80_rate, 10); plat_rate = clk_get_rate(cg->pll[PLATFORM_PLL].div[PLL_DIV1].clk); @@ -794,7 +801,7 @@ static struct clk * __init create_one_cmux(struct clockgen *cg, int idx) else min_rate = plat_rate / 2; - return create_mux_common(cg, hwc, &cmux_ops, min_rate, + return create_mux_common(cg, hwc, &cmux_ops, min_rate, max_rate, pct80_rate, "cg-cmux%d", idx); } @@ -809,7 +816,7 @@ static struct clk * __init create_one_hwaccel(struct clockgen *cg, int idx) hwc->reg = cg->regs + 0x20 * idx + 0x10; hwc->info = cg->info.hwaccel[idx]; - return create_mux_common(cg, hwc, &hwaccel_ops, 0, 0, + return create_mux_common(cg, hwc, &hwaccel_ops, 0, ULONG_MAX, 0, "cg-hwaccel%d", idx); } diff --git a/kernel/drivers/clk/clk-wm831x.c b/kernel/drivers/clk/clk-wm831x.c index 43f9d1525..763aed2de 100644 --- a/kernel/drivers/clk/clk-wm831x.c +++ b/kernel/drivers/clk/clk-wm831x.c @@ -247,7 +247,7 @@ static int wm831x_clkout_is_prepared(struct clk_hw *hw) if (ret < 0) { dev_err(wm831x->dev, "Unable to read CLOCK_CONTROL_1: %d\n", ret); - return true; + return false; } return (ret & WM831X_CLKOUT_ENA) != 0; diff --git a/kernel/drivers/clk/clk-xgene.c b/kernel/drivers/clk/clk-xgene.c index 27c0da29e..b134a8b15 100644 --- a/kernel/drivers/clk/clk-xgene.c +++ b/kernel/drivers/clk/clk-xgene.c @@ -351,7 +351,8 @@ static int xgene_clk_set_rate(struct clk_hw *hw, unsigned long rate, /* Set new divider */ data = xgene_clk_read(pclk->param.divider_reg + pclk->param.reg_divider_offset); - data &= ~((1 << pclk->param.reg_divider_width) - 1); + data &= ~(((1 << pclk->param.reg_divider_width) - 1) + << pclk->param.reg_divider_shift); data |= divider; xgene_clk_write(data, pclk->param.divider_reg + pclk->param.reg_divider_offset); diff --git a/kernel/drivers/clk/imx/clk-imx31.c b/kernel/drivers/clk/imx/clk-imx31.c index 6a964144a..6a49ba2b9 100644 --- a/kernel/drivers/clk/imx/clk-imx31.c +++ b/kernel/drivers/clk/imx/clk-imx31.c @@ -157,10 +157,8 @@ static void __init _mx31_clocks_init(unsigned long fref) } } -int __init mx31_clocks_init(void) +int __init mx31_clocks_init(unsigned long fref) { - u32 fref = 26000000; /* default */ - _mx31_clocks_init(fref); clk_register_clkdev(clk[gpt_gate], "per", "imx-gpt.0"); diff --git a/kernel/drivers/clk/imx/clk-imx35.c b/kernel/drivers/clk/imx/clk-imx35.c index a71d24cb4..d302ed3b8 100644 --- a/kernel/drivers/clk/imx/clk-imx35.c +++ b/kernel/drivers/clk/imx/clk-imx35.c @@ -66,7 +66,7 @@ static const char *std_sel[] = {"ppll", "arm"}; static const char *ipg_per_sel[] = {"ahb_per_div", "arm_per_div"}; enum mx35_clks { - ckih, ckil, mpll, ppll, mpll_075, arm, hsp, hsp_div, hsp_sel, ahb, ipg, + ckih, mpll, ppll, mpll_075, arm, hsp, hsp_div, hsp_sel, ahb, ipg, arm_per_div, ahb_per_div, ipg_per, uart_sel, uart_div, esdhc_sel, esdhc1_div, esdhc2_div, esdhc3_div, spdif_sel, spdif_div_pre, spdif_div_post, ssi_sel, ssi1_div_pre, ssi1_div_post, ssi2_div_pre, @@ -79,7 +79,7 @@ enum mx35_clks { rtc_gate, rtic_gate, scc_gate, sdma_gate, spba_gate, spdif_gate, ssi1_gate, ssi2_gate, uart1_gate, uart2_gate, uart3_gate, usbotg_gate, wdog_gate, max_gate, admux_gate, csi_gate, csi_div, csi_sel, iim_gate, - gpu2d_gate, clk_max + gpu2d_gate, ckil, clk_max }; static struct clk *clk[clk_max]; @@ -115,7 +115,7 @@ static void __init _mx35_clocks_init(void) } clk[ckih] = imx_clk_fixed("ckih", 24000000); - clk[ckil] = imx_clk_fixed("ckih", 32768); + clk[ckil] = imx_clk_fixed("ckil", 32768); clk[mpll] = imx_clk_pllv1(IMX_PLLV1_IMX35, "mpll", "ckih", base + MX35_CCM_MPCTL); clk[ppll] = imx_clk_pllv1(IMX_PLLV1_IMX35, "ppll", "ckih", base + MX35_CCM_PPCTL); diff --git a/kernel/drivers/clk/imx/clk-imx6q.c b/kernel/drivers/clk/imx/clk-imx6q.c index c1935081d..aab64205d 100644 --- a/kernel/drivers/clk/imx/clk-imx6q.c +++ b/kernel/drivers/clk/imx/clk-imx6q.c @@ -550,6 +550,24 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) if (IS_ENABLED(CONFIG_PCI_IMX6)) clk_set_parent(clk[IMX6QDL_CLK_LVDS1_SEL], clk[IMX6QDL_CLK_SATA_REF_100M]); + /* + * Initialize the GPU clock muxes, so that the maximum specified clock + * rates for the respective SoC are not exceeded. + */ + if (clk_on_imx6dl()) { + clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], + clk[IMX6QDL_CLK_PLL2_PFD1_594M]); + clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], + clk[IMX6QDL_CLK_PLL2_PFD1_594M]); + } else if (clk_on_imx6q()) { + clk_set_parent(clk[IMX6QDL_CLK_GPU3D_CORE_SEL], + clk[IMX6QDL_CLK_MMDC_CH0_AXI]); + clk_set_parent(clk[IMX6QDL_CLK_GPU3D_SHADER_SEL], + clk[IMX6QDL_CLK_PLL2_PFD1_594M]); + clk_set_parent(clk[IMX6QDL_CLK_GPU2D_CORE_SEL], + clk[IMX6QDL_CLK_PLL3_USB_OTG]); + } + imx_register_uart_clocks(uart_clks); } CLK_OF_DECLARE(imx6q, "fsl,imx6q-ccm", imx6q_clocks_init); diff --git a/kernel/drivers/clk/meson/clkc.c b/kernel/drivers/clk/meson/clkc.c index c83ae1367..d920d410b 100644 --- a/kernel/drivers/clk/meson/clkc.c +++ b/kernel/drivers/clk/meson/clkc.c @@ -198,7 +198,7 @@ meson_clk_register_fixed_rate(const struct clk_conf *clk_conf, } void __init meson_clk_register_clks(const struct clk_conf *clk_confs, - size_t nr_confs, + unsigned int nr_confs, void __iomem *clk_base) { unsigned int i; diff --git a/kernel/drivers/clk/mmp/clk-of-mmp2.c b/kernel/drivers/clk/mmp/clk-of-mmp2.c index 251533d87..f261b1d29 100644 --- a/kernel/drivers/clk/mmp/clk-of-mmp2.c +++ b/kernel/drivers/clk/mmp/clk-of-mmp2.c @@ -313,7 +313,7 @@ static void __init mmp2_clk_init(struct device_node *np) } pxa_unit->apmu_base = of_iomap(np, 1); - if (!pxa_unit->mpmu_base) { + if (!pxa_unit->apmu_base) { pr_err("failed to map apmu registers\n"); return; } diff --git a/kernel/drivers/clk/mmp/clk-of-pxa168.c b/kernel/drivers/clk/mmp/clk-of-pxa168.c index 64eaf4141..427f4bb08 100644 --- a/kernel/drivers/clk/mmp/clk-of-pxa168.c +++ b/kernel/drivers/clk/mmp/clk-of-pxa168.c @@ -262,7 +262,7 @@ static void __init pxa168_clk_init(struct device_node *np) } pxa_unit->apmu_base = of_iomap(np, 1); - if (!pxa_unit->mpmu_base) { + if (!pxa_unit->apmu_base) { pr_err("failed to map apmu registers\n"); return; } diff --git a/kernel/drivers/clk/mmp/clk-of-pxa910.c b/kernel/drivers/clk/mmp/clk-of-pxa910.c index 13d617332..cdf5ba566 100644 --- a/kernel/drivers/clk/mmp/clk-of-pxa910.c +++ b/kernel/drivers/clk/mmp/clk-of-pxa910.c @@ -282,7 +282,7 @@ static void __init pxa910_clk_init(struct device_node *np) } pxa_unit->apmu_base = of_iomap(np, 1); - if (!pxa_unit->mpmu_base) { + if (!pxa_unit->apmu_base) { pr_err("failed to map apmu registers\n"); return; } @@ -294,7 +294,7 @@ static void __init pxa910_clk_init(struct device_node *np) } pxa_unit->apbcp_base = of_iomap(np, 3); - if (!pxa_unit->mpmu_base) { + if (!pxa_unit->apbcp_base) { pr_err("failed to map apbcp registers\n"); return; } diff --git a/kernel/drivers/clk/nxp/clk-lpc18xx-ccu.c b/kernel/drivers/clk/nxp/clk-lpc18xx-ccu.c index 13aabbb3a..558da8955 100644 --- a/kernel/drivers/clk/nxp/clk-lpc18xx-ccu.c +++ b/kernel/drivers/clk/nxp/clk-lpc18xx-ccu.c @@ -222,7 +222,7 @@ static void lpc18xx_ccu_register_branch_gate_div(struct lpc18xx_clk_branch *bran div->width = 1; div_hw = &div->hw; - div_ops = &clk_divider_ops; + div_ops = &clk_divider_ro_ops; } branch->gate.reg = branch->offset + reg_base; diff --git a/kernel/drivers/clk/qcom/gcc-msm8916.c b/kernel/drivers/clk/qcom/gcc-msm8916.c index d0a0313d6..2e7f03d50 100644 --- a/kernel/drivers/clk/qcom/gcc-msm8916.c +++ b/kernel/drivers/clk/qcom/gcc-msm8916.c @@ -2346,6 +2346,7 @@ static struct clk_branch gcc_crypto_ahb_clk = { "pcnoc_bfdcd_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -2381,6 +2382,7 @@ static struct clk_branch gcc_crypto_clk = { "crypto_clk_src", }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, diff --git a/kernel/drivers/clk/qcom/gcc-msm8960.c b/kernel/drivers/clk/qcom/gcc-msm8960.c index 66c18bc97..bdc4b2d07 100644 --- a/kernel/drivers/clk/qcom/gcc-msm8960.c +++ b/kernel/drivers/clk/qcom/gcc-msm8960.c @@ -2753,7 +2753,7 @@ static struct clk_rcg ce3_src = { }, .freq_tbl = clk_tbl_ce3, .clkr = { - .enable_reg = 0x2c08, + .enable_reg = 0x36c0, .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "ce3_src", @@ -2769,7 +2769,7 @@ static struct clk_branch ce3_core_clk = { .halt_reg = 0x2fdc, .halt_bit = 5, .clkr = { - .enable_reg = 0x36c4, + .enable_reg = 0x36cc, .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "ce3_core_clk", diff --git a/kernel/drivers/clk/rockchip/clk-mmc-phase.c b/kernel/drivers/clk/rockchip/clk-mmc-phase.c index 268564482..33c20c6b4 100644 --- a/kernel/drivers/clk/rockchip/clk-mmc-phase.c +++ b/kernel/drivers/clk/rockchip/clk-mmc-phase.c @@ -153,6 +153,7 @@ struct clk *rockchip_clk_register_mmc(const char *name, return NULL; init.name = name; + init.flags = 0; init.num_parents = num_parents; init.parent_names = parent_names; init.ops = &rockchip_mmc_clk_ops; diff --git a/kernel/drivers/clk/rockchip/clk-rk3188.c b/kernel/drivers/clk/rockchip/clk-rk3188.c index abb476087..fe728f8dc 100644 --- a/kernel/drivers/clk/rockchip/clk-rk3188.c +++ b/kernel/drivers/clk/rockchip/clk-rk3188.c @@ -718,6 +718,7 @@ static const char *const rk3188_critical_clocks[] __initconst = { "hclk_peri", "pclk_cpu", "pclk_peri", + "hclk_cpubus" }; static void __init rk3188_common_clk_init(struct device_node *np) diff --git a/kernel/drivers/clk/rockchip/clk-rk3368.c b/kernel/drivers/clk/rockchip/clk-rk3368.c index 7e6b783e6..1b148694b 100644 --- a/kernel/drivers/clk/rockchip/clk-rk3368.c +++ b/kernel/drivers/clk/rockchip/clk-rk3368.c @@ -165,7 +165,7 @@ static const struct rockchip_cpuclk_reg_data rk3368_cpuclkb_data = { .core_reg = RK3368_CLKSEL_CON(0), .div_core_shift = 0, .div_core_mask = 0x1f, - .mux_core_shift = 15, + .mux_core_shift = 7, }; static const struct rockchip_cpuclk_reg_data rk3368_cpuclkl_data = { @@ -218,29 +218,29 @@ static const struct rockchip_cpuclk_reg_data rk3368_cpuclkl_data = { } static struct rockchip_cpuclk_rate_table rk3368_cpuclkb_rates[] __initdata = { - RK3368_CPUCLKB_RATE(1512000000, 2, 6, 6), - RK3368_CPUCLKB_RATE(1488000000, 2, 5, 5), - RK3368_CPUCLKB_RATE(1416000000, 2, 5, 5), - RK3368_CPUCLKB_RATE(1200000000, 2, 4, 4), - RK3368_CPUCLKB_RATE(1008000000, 2, 4, 4), - RK3368_CPUCLKB_RATE( 816000000, 2, 3, 3), - RK3368_CPUCLKB_RATE( 696000000, 2, 3, 3), - RK3368_CPUCLKB_RATE( 600000000, 2, 2, 2), - RK3368_CPUCLKB_RATE( 408000000, 2, 2, 2), - RK3368_CPUCLKB_RATE( 312000000, 2, 2, 2), + RK3368_CPUCLKB_RATE(1512000000, 1, 5, 5), + RK3368_CPUCLKB_RATE(1488000000, 1, 4, 4), + RK3368_CPUCLKB_RATE(1416000000, 1, 4, 4), + RK3368_CPUCLKB_RATE(1200000000, 1, 3, 3), + RK3368_CPUCLKB_RATE(1008000000, 1, 3, 3), + RK3368_CPUCLKB_RATE( 816000000, 1, 2, 2), + RK3368_CPUCLKB_RATE( 696000000, 1, 2, 2), + RK3368_CPUCLKB_RATE( 600000000, 1, 1, 1), + RK3368_CPUCLKB_RATE( 408000000, 1, 1, 1), + RK3368_CPUCLKB_RATE( 312000000, 1, 1, 1), }; static struct rockchip_cpuclk_rate_table rk3368_cpuclkl_rates[] __initdata = { - RK3368_CPUCLKL_RATE(1512000000, 2, 7, 7), - RK3368_CPUCLKL_RATE(1488000000, 2, 6, 6), - RK3368_CPUCLKL_RATE(1416000000, 2, 6, 6), - RK3368_CPUCLKL_RATE(1200000000, 2, 5, 5), - RK3368_CPUCLKL_RATE(1008000000, 2, 5, 5), - RK3368_CPUCLKL_RATE( 816000000, 2, 4, 4), - RK3368_CPUCLKL_RATE( 696000000, 2, 3, 3), - RK3368_CPUCLKL_RATE( 600000000, 2, 3, 3), - RK3368_CPUCLKL_RATE( 408000000, 2, 2, 2), - RK3368_CPUCLKL_RATE( 312000000, 2, 2, 2), + RK3368_CPUCLKL_RATE(1512000000, 1, 6, 6), + RK3368_CPUCLKL_RATE(1488000000, 1, 5, 5), + RK3368_CPUCLKL_RATE(1416000000, 1, 5, 5), + RK3368_CPUCLKL_RATE(1200000000, 1, 4, 4), + RK3368_CPUCLKL_RATE(1008000000, 1, 4, 4), + RK3368_CPUCLKL_RATE( 816000000, 1, 3, 3), + RK3368_CPUCLKL_RATE( 696000000, 1, 2, 2), + RK3368_CPUCLKL_RATE( 600000000, 1, 2, 2), + RK3368_CPUCLKL_RATE( 408000000, 1, 1, 1), + RK3368_CPUCLKL_RATE( 312000000, 1, 1, 1), }; static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { @@ -384,10 +384,10 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { * Clock-Architecture Diagram 3 */ - COMPOSITE(0, "aclk_vepu", mux_pll_src_cpll_gpll_usb_p, 0, + COMPOSITE(0, "aclk_vepu", mux_pll_src_cpll_gpll_npll_usb_p, 0, RK3368_CLKSEL_CON(15), 6, 2, MFLAGS, 0, 5, DFLAGS, RK3368_CLKGATE_CON(4), 6, GFLAGS), - COMPOSITE(0, "aclk_vdpu", mux_pll_src_cpll_gpll_usb_p, 0, + COMPOSITE(0, "aclk_vdpu", mux_pll_src_cpll_gpll_npll_usb_p, 0, RK3368_CLKSEL_CON(15), 14, 2, MFLAGS, 8, 5, DFLAGS, RK3368_CLKGATE_CON(4), 7, GFLAGS), @@ -442,7 +442,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { GATE(SCLK_HDMI_HDCP, "sclk_hdmi_hdcp", "xin24m", 0, RK3368_CLKGATE_CON(4), 13, GFLAGS), GATE(SCLK_HDMI_CEC, "sclk_hdmi_cec", "xin32k", 0, - RK3368_CLKGATE_CON(5), 12, GFLAGS), + RK3368_CLKGATE_CON(4), 12, GFLAGS), COMPOSITE_NODIV(0, "vip_src", mux_pll_src_cpll_gpll_p, 0, RK3368_CLKSEL_CON(21), 15, 1, MFLAGS, diff --git a/kernel/drivers/clk/rockchip/clk.c b/kernel/drivers/clk/rockchip/clk.c index be6c7fd83..9b6c8188e 100644 --- a/kernel/drivers/clk/rockchip/clk.c +++ b/kernel/drivers/clk/rockchip/clk.c @@ -70,7 +70,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, if (gate_offset >= 0) { gate = kzalloc(sizeof(*gate), GFP_KERNEL); if (!gate) - return ERR_PTR(-ENOMEM); + goto err_gate; gate->flags = gate_flags; gate->reg = base + gate_offset; @@ -82,7 +82,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, if (div_width > 0) { div = kzalloc(sizeof(*div), GFP_KERNEL); if (!div) - return ERR_PTR(-ENOMEM); + goto err_div; div->flags = div_flags; div->reg = base + muxdiv_offset; @@ -90,7 +90,9 @@ static struct clk *rockchip_clk_register_branch(const char *name, div->width = div_width; div->lock = lock; div->table = div_table; - div_ops = &clk_divider_ops; + div_ops = (div_flags & CLK_DIVIDER_READ_ONLY) + ? &clk_divider_ro_ops + : &clk_divider_ops; } clk = clk_register_composite(NULL, name, parent_names, num_parents, @@ -100,6 +102,11 @@ static struct clk *rockchip_clk_register_branch(const char *name, flags); return clk; +err_div: + kfree(gate); +err_gate: + kfree(mux); + return ERR_PTR(-ENOMEM); } static struct clk *rockchip_clk_register_frac_branch(const char *name, diff --git a/kernel/drivers/clk/ti/clk-3xxx.c b/kernel/drivers/clk/ti/clk-3xxx.c index 8831e1a05..11d8aa3ec 100644 --- a/kernel/drivers/clk/ti/clk-3xxx.c +++ b/kernel/drivers/clk/ti/clk-3xxx.c @@ -22,13 +22,6 @@ #include "clock.h" -/* - * DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks - * that are sourced by DPLL5, and both of these require this clock - * to be at 120 MHz for proper operation. - */ -#define DPLL5_FREQ_FOR_USBHOST 120000000 - #define OMAP3430ES2_ST_DSS_IDLE_SHIFT 1 #define OMAP3430ES2_ST_HSOTGUSB_IDLE_SHIFT 5 #define OMAP3430ES2_ST_SSI_IDLE_SHIFT 8 @@ -546,14 +539,21 @@ void __init omap3_clk_lock_dpll5(void) struct clk *dpll5_clk; struct clk *dpll5_m2_clk; + /* + * Errata sprz319f advisory 2.1 documents a USB host clock drift issue + * that can be worked around using specially crafted dpll5 settings + * with a dpll5_m2 divider set to 8. Set the dpll5 rate to 8x the USB + * host clock rate, its .set_rate handler() will detect that frequency + * and use the errata settings. + */ dpll5_clk = clk_get(NULL, "dpll5_ck"); - clk_set_rate(dpll5_clk, DPLL5_FREQ_FOR_USBHOST); + clk_set_rate(dpll5_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST * 8); clk_prepare_enable(dpll5_clk); - /* Program dpll5_m2_clk divider for no division */ + /* Program dpll5_m2_clk divider */ dpll5_m2_clk = clk_get(NULL, "dpll5_m2_ck"); clk_prepare_enable(dpll5_m2_clk); - clk_set_rate(dpll5_m2_clk, DPLL5_FREQ_FOR_USBHOST); + clk_set_rate(dpll5_m2_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST); clk_disable_unprepare(dpll5_m2_clk); clk_disable_unprepare(dpll5_clk); diff --git a/kernel/drivers/clk/ti/clock.h b/kernel/drivers/clk/ti/clock.h index 90f3f472a..13c37f48d 100644 --- a/kernel/drivers/clk/ti/clock.h +++ b/kernel/drivers/clk/ti/clock.h @@ -257,11 +257,20 @@ long omap2_dpll_round_rate(struct clk_hw *hw, unsigned long target_rate, unsigned long omap3_clkoutx2_recalc(struct clk_hw *hw, unsigned long parent_rate); +/* + * OMAP3_DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks + * that are sourced by DPLL5, and both of these require this clock + * to be at 120 MHz for proper operation. + */ +#define OMAP3_DPLL5_FREQ_FOR_USBHOST 120000000 + unsigned long omap3_dpll_recalc(struct clk_hw *hw, unsigned long parent_rate); int omap3_dpll4_set_rate(struct clk_hw *clk, unsigned long rate, unsigned long parent_rate); int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index); +int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate); void omap3_clk_lock_dpll5(void); unsigned long omap4_dpll_regm4xen_recalc(struct clk_hw *hw, diff --git a/kernel/drivers/clk/ti/dpll.c b/kernel/drivers/clk/ti/dpll.c index 5519b386e..f9a5089dd 100644 --- a/kernel/drivers/clk/ti/dpll.c +++ b/kernel/drivers/clk/ti/dpll.c @@ -114,6 +114,18 @@ static const struct clk_ops omap3_dpll_ck_ops = { .round_rate = &omap2_dpll_round_rate, }; +static const struct clk_ops omap3_dpll5_ck_ops = { + .enable = &omap3_noncore_dpll_enable, + .disable = &omap3_noncore_dpll_disable, + .get_parent = &omap2_init_dpll_parent, + .recalc_rate = &omap3_dpll_recalc, + .set_rate = &omap3_dpll5_set_rate, + .set_parent = &omap3_noncore_dpll_set_parent, + .set_rate_and_parent = &omap3_noncore_dpll_set_rate_and_parent, + .determine_rate = &omap3_noncore_dpll_determine_rate, + .round_rate = &omap2_dpll_round_rate, +}; + static const struct clk_ops omap3_dpll_per_ck_ops = { .enable = &omap3_noncore_dpll_enable, .disable = &omap3_noncore_dpll_disable, @@ -461,7 +473,12 @@ static void __init of_ti_omap3_dpll_setup(struct device_node *node) .modes = (1 << DPLL_LOW_POWER_BYPASS) | (1 << DPLL_LOCKED), }; - of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd); + if ((of_machine_is_compatible("ti,omap3630") || + of_machine_is_compatible("ti,omap36xx")) && + !strcmp(node->name, "dpll5_ck")) + of_ti_dpll_setup(node, &omap3_dpll5_ck_ops, &dd); + else + of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd); } CLK_OF_DECLARE(ti_omap3_dpll_clock, "ti,omap3-dpll-clock", of_ti_omap3_dpll_setup); diff --git a/kernel/drivers/clk/ti/dpll3xxx.c b/kernel/drivers/clk/ti/dpll3xxx.c index f4dec00fb..0e9119fae 100644 --- a/kernel/drivers/clk/ti/dpll3xxx.c +++ b/kernel/drivers/clk/ti/dpll3xxx.c @@ -815,3 +815,70 @@ int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, return omap3_noncore_dpll_set_rate_and_parent(hw, rate, parent_rate, index); } + +/* Apply DM3730 errata sprz319 advisory 2.1. */ +static bool omap3_dpll5_apply_errata(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct omap3_dpll5_settings { + unsigned int rate, m, n; + }; + + static const struct omap3_dpll5_settings precomputed[] = { + /* + * From DM3730 errata advisory 2.1, table 35 and 36. + * The N value is increased by 1 compared to the tables as the + * errata lists register values while last_rounded_field is the + * real divider value. + */ + { 12000000, 80, 0 + 1 }, + { 13000000, 443, 5 + 1 }, + { 19200000, 50, 0 + 1 }, + { 26000000, 443, 11 + 1 }, + { 38400000, 25, 0 + 1 } + }; + + const struct omap3_dpll5_settings *d; + struct clk_hw_omap *clk = to_clk_hw_omap(hw); + struct dpll_data *dd; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(precomputed); ++i) { + if (parent_rate == precomputed[i].rate) + break; + } + + if (i == ARRAY_SIZE(precomputed)) + return false; + + d = &precomputed[i]; + + /* Update the M, N and rounded rate values and program the DPLL. */ + dd = clk->dpll_data; + dd->last_rounded_m = d->m; + dd->last_rounded_n = d->n; + dd->last_rounded_rate = div_u64((u64)parent_rate * d->m, d->n); + omap3_noncore_dpll_program(clk, 0); + + return true; +} + +/** + * omap3_dpll5_set_rate - set rate for omap3 dpll5 + * @hw: clock to change + * @rate: target rate for clock + * @parent_rate: rate of the parent clock + * + * Set rate for the DPLL5 clock. Apply the sprz319 advisory 2.1 on OMAP36xx if + * the DPLL is used for USB host (detected through the requested rate). + */ +int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + if (rate == OMAP3_DPLL5_FREQ_FOR_USBHOST * 8) { + if (omap3_dpll5_apply_errata(hw, parent_rate)) + return 0; + } + + return omap3_noncore_dpll_set_rate(hw, rate, parent_rate); +} diff --git a/kernel/drivers/clk/versatile/clk-sp810.c b/kernel/drivers/clk/versatile/clk-sp810.c index a1cdef6b0..897c36c17 100644 --- a/kernel/drivers/clk/versatile/clk-sp810.c +++ b/kernel/drivers/clk/versatile/clk-sp810.c @@ -92,6 +92,7 @@ static void __init clk_sp810_of_setup(struct device_node *node) int num = ARRAY_SIZE(parent_names); char name[12]; struct clk_init_data init; + static int instance; int i; bool deprecated; @@ -118,7 +119,7 @@ static void __init clk_sp810_of_setup(struct device_node *node) deprecated = !of_find_property(node, "assigned-clock-parents", NULL); for (i = 0; i < ARRAY_SIZE(sp810->timerclken); i++) { - snprintf(name, ARRAY_SIZE(name), "timerclken%d", i); + snprintf(name, sizeof(name), "sp810_%d_%d", instance, i); sp810->timerclken[i].sp810 = sp810; sp810->timerclken[i].channel = i; @@ -139,5 +140,6 @@ static void __init clk_sp810_of_setup(struct device_node *node) } of_clk_add_provider(node, clk_sp810_timerclken_of_get, sp810); + instance++; } CLK_OF_DECLARE(sp810, "arm,sp810", clk_sp810_of_setup); diff --git a/kernel/drivers/clocksource/exynos_mct.c b/kernel/drivers/clocksource/exynos_mct.c index ff44082a0..47f8aafe3 100644 --- a/kernel/drivers/clocksource/exynos_mct.c +++ b/kernel/drivers/clocksource/exynos_mct.c @@ -482,6 +482,7 @@ static void exynos4_local_timer_stop(struct mct_clock_event_device *mevt) if (mct_int_type == MCT_INT_SPI) { if (evt->irq != -1) disable_irq_nosync(evt->irq); + exynos4_mct_write(0x1, mevt->base + MCT_L_INT_CSTAT_OFFSET); } else { disable_percpu_irq(mct_irqs[MCT_L0_IRQ]); } diff --git a/kernel/drivers/clocksource/sun4i_timer.c b/kernel/drivers/clocksource/sun4i_timer.c index 6f3719d73..e84877a2c 100644 --- a/kernel/drivers/clocksource/sun4i_timer.c +++ b/kernel/drivers/clocksource/sun4i_timer.c @@ -123,12 +123,16 @@ static struct clock_event_device sun4i_clockevent = { .set_next_event = sun4i_clkevt_next_event, }; +static void sun4i_timer_clear_interrupt(void) +{ + writel(TIMER_IRQ_EN(0), timer_base + TIMER_IRQ_ST_REG); +} static irqreturn_t sun4i_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = (struct clock_event_device *)dev_id; - writel(0x1, timer_base + TIMER_IRQ_ST_REG); + sun4i_timer_clear_interrupt(); evt->event_handler(evt); return IRQ_HANDLED; @@ -193,6 +197,9 @@ static void __init sun4i_timer_init(struct device_node *node) /* Make sure timer is stopped before playing with interrupts */ sun4i_clkevt_time_stop(0); + /* clear timer0 interrupt */ + sun4i_timer_clear_interrupt(); + sun4i_clockevent.cpumask = cpu_possible_mask; sun4i_clockevent.irq = irq; diff --git a/kernel/drivers/cpufreq/cpufreq_userspace.c b/kernel/drivers/cpufreq/cpufreq_userspace.c index 4dbf1db16..9cc8abd3d 100644 --- a/kernel/drivers/cpufreq/cpufreq_userspace.c +++ b/kernel/drivers/cpufreq/cpufreq_userspace.c @@ -17,6 +17,7 @@ #include #include #include +#include static DEFINE_PER_CPU(unsigned int, cpu_is_managed); static DEFINE_MUTEX(userspace_mutex); @@ -31,6 +32,7 @@ static DEFINE_MUTEX(userspace_mutex); static int cpufreq_set(struct cpufreq_policy *policy, unsigned int freq) { int ret = -EINVAL; + unsigned int *setspeed = policy->governor_data; pr_debug("cpufreq_set for cpu %u, freq %u kHz\n", policy->cpu, freq); @@ -38,6 +40,8 @@ static int cpufreq_set(struct cpufreq_policy *policy, unsigned int freq) if (!per_cpu(cpu_is_managed, policy->cpu)) goto err; + *setspeed = freq; + ret = __cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L); err: mutex_unlock(&userspace_mutex); @@ -49,19 +53,45 @@ static ssize_t show_speed(struct cpufreq_policy *policy, char *buf) return sprintf(buf, "%u\n", policy->cur); } +static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy) +{ + unsigned int *setspeed; + + setspeed = kzalloc(sizeof(*setspeed), GFP_KERNEL); + if (!setspeed) + return -ENOMEM; + + policy->governor_data = setspeed; + return 0; +} + static int cpufreq_governor_userspace(struct cpufreq_policy *policy, unsigned int event) { + unsigned int *setspeed = policy->governor_data; unsigned int cpu = policy->cpu; int rc = 0; + if (event == CPUFREQ_GOV_POLICY_INIT) + return cpufreq_userspace_policy_init(policy); + + if (!setspeed) + return -EINVAL; + switch (event) { + case CPUFREQ_GOV_POLICY_EXIT: + mutex_lock(&userspace_mutex); + policy->governor_data = NULL; + kfree(setspeed); + mutex_unlock(&userspace_mutex); + break; case CPUFREQ_GOV_START: BUG_ON(!policy->cur); pr_debug("started managing cpu %u\n", cpu); mutex_lock(&userspace_mutex); per_cpu(cpu_is_managed, cpu) = 1; + *setspeed = policy->cur; mutex_unlock(&userspace_mutex); break; case CPUFREQ_GOV_STOP: @@ -69,20 +99,23 @@ static int cpufreq_governor_userspace(struct cpufreq_policy *policy, mutex_lock(&userspace_mutex); per_cpu(cpu_is_managed, cpu) = 0; + *setspeed = 0; mutex_unlock(&userspace_mutex); break; case CPUFREQ_GOV_LIMITS: mutex_lock(&userspace_mutex); - pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz\n", - cpu, policy->min, policy->max, - policy->cur); + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", + cpu, policy->min, policy->max, policy->cur, *setspeed); - if (policy->max < policy->cur) + if (policy->max < *setspeed) __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); - else if (policy->min > policy->cur) + else if (policy->min > *setspeed) __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + else + __cpufreq_driver_target(policy, *setspeed, + CPUFREQ_RELATION_L); mutex_unlock(&userspace_mutex); break; } diff --git a/kernel/drivers/cpufreq/intel_pstate.c b/kernel/drivers/cpufreq/intel_pstate.c index 98fb88213..7ff8b15a3 100644 --- a/kernel/drivers/cpufreq/intel_pstate.c +++ b/kernel/drivers/cpufreq/intel_pstate.c @@ -285,14 +285,14 @@ static void intel_pstate_hwp_set(void) int min, hw_min, max, hw_max, cpu, range, adj_range; u64 value, cap; - rdmsrl(MSR_HWP_CAPABILITIES, cap); - hw_min = HWP_LOWEST_PERF(cap); - hw_max = HWP_HIGHEST_PERF(cap); - range = hw_max - hw_min; - get_online_cpus(); for_each_online_cpu(cpu) { + rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); + hw_min = HWP_LOWEST_PERF(cap); + hw_max = HWP_HIGHEST_PERF(cap); + range = hw_max - hw_min; + rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); adj_range = limits->min_perf_pct * range / 100; min = hw_min + adj_range; @@ -662,11 +662,16 @@ static int core_get_max_pstate(void) if (err) goto skip_tar; - tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl; + tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x3); err = rdmsrl_safe(tdp_msr, &tdp_ratio); if (err) goto skip_tar; + /* For level 1 and 2, bits[23:16] contain the ratio */ + if (tdp_ctrl) + tdp_ratio >>= 16; + + tdp_ratio &= 0xff; /* ratios are only 8 bits long */ if (tdp_ratio - 1 == tar) { max_pstate = tar; pr_debug("max_pstate=TAC %x\n", max_pstate); diff --git a/kernel/drivers/cpufreq/powernv-cpufreq.c b/kernel/drivers/cpufreq/powernv-cpufreq.c index cb501386e..c4b0ef659 100644 --- a/kernel/drivers/cpufreq/powernv-cpufreq.c +++ b/kernel/drivers/cpufreq/powernv-cpufreq.c @@ -373,8 +373,14 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, if (unlikely(rebooting) && new_index != get_nominal_index()) return 0; - if (!throttled) + if (!throttled) { + /* we don't want to be preempted while + * checking if the CPU frequency has been throttled + */ + preempt_disable(); powernv_cpufreq_throttle_check(NULL); + preempt_enable(); + } freq_data.pstate_id = powernv_freqs[new_index].driver_data; diff --git a/kernel/drivers/cpuidle/cpuidle-arm.c b/kernel/drivers/cpuidle/cpuidle-arm.c index 545069d5f..1855b9ee8 100644 --- a/kernel/drivers/cpuidle/cpuidle-arm.c +++ b/kernel/drivers/cpuidle/cpuidle-arm.c @@ -50,7 +50,7 @@ static int arm_enter_idle_state(struct cpuidle_device *dev, * call the CPU ops suspend protocol with idle index as a * parameter. */ - arm_cpuidle_suspend(idx); + ret = arm_cpuidle_suspend(idx); cpu_pm_exit(); } @@ -135,6 +135,7 @@ static int __init arm_idle_init(void) dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) { pr_err("Failed to allocate cpuidle device\n"); + ret = -ENOMEM; goto out_fail; } dev->cpu = cpu; diff --git a/kernel/drivers/cpuidle/cpuidle.c b/kernel/drivers/cpuidle/cpuidle.c index 17a6dc0e2..d40b2c077 100644 --- a/kernel/drivers/cpuidle/cpuidle.c +++ b/kernel/drivers/cpuidle/cpuidle.c @@ -214,7 +214,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, tick_broadcast_exit(); } - if (!cpuidle_state_is_coupled(drv, entered_state)) + if (!cpuidle_state_is_coupled(drv, index)) local_irq_enable(); diff = ktime_to_us(ktime_sub(time_end, time_start)); @@ -433,6 +433,8 @@ static void __cpuidle_unregister_device(struct cpuidle_device *dev) list_del(&dev->device_list); per_cpu(cpuidle_devices, dev->cpu) = NULL; module_put(drv->owner); + + dev->registered = 0; } static void __cpuidle_device_init(struct cpuidle_device *dev) diff --git a/kernel/drivers/crypto/atmel-aes.c b/kernel/drivers/crypto/atmel-aes.c index fb16d812c..1dffb13e5 100644 --- a/kernel/drivers/crypto/atmel-aes.c +++ b/kernel/drivers/crypto/atmel-aes.c @@ -1396,9 +1396,9 @@ static int atmel_aes_probe(struct platform_device *pdev) } aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res); - if (!aes_dd->io_base) { + if (IS_ERR(aes_dd->io_base)) { dev_err(dev, "can't ioremap\n"); - err = -ENOMEM; + err = PTR_ERR(aes_dd->io_base); goto res_err; } diff --git a/kernel/drivers/crypto/atmel-sha.c b/kernel/drivers/crypto/atmel-sha.c index 3178f84d2..0dadb6332 100644 --- a/kernel/drivers/crypto/atmel-sha.c +++ b/kernel/drivers/crypto/atmel-sha.c @@ -1405,9 +1405,9 @@ static int atmel_sha_probe(struct platform_device *pdev) } sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res); - if (!sha_dd->io_base) { + if (IS_ERR(sha_dd->io_base)) { dev_err(dev, "can't ioremap\n"); - err = -ENOMEM; + err = PTR_ERR(sha_dd->io_base); goto res_err; } diff --git a/kernel/drivers/crypto/atmel-tdes.c b/kernel/drivers/crypto/atmel-tdes.c index 2c7a628d0..bf467d7be 100644 --- a/kernel/drivers/crypto/atmel-tdes.c +++ b/kernel/drivers/crypto/atmel-tdes.c @@ -1417,9 +1417,9 @@ static int atmel_tdes_probe(struct platform_device *pdev) } tdes_dd->io_base = devm_ioremap_resource(&pdev->dev, tdes_res); - if (!tdes_dd->io_base) { + if (IS_ERR(tdes_dd->io_base)) { dev_err(dev, "can't ioremap\n"); - err = -ENOMEM; + err = PTR_ERR(tdes_dd->io_base); goto res_err; } diff --git a/kernel/drivers/crypto/caam/caamalg.c b/kernel/drivers/crypto/caam/caamalg.c index ea8189f4b..f3307fc38 100644 --- a/kernel/drivers/crypto/caam/caamalg.c +++ b/kernel/drivers/crypto/caam/caamalg.c @@ -441,6 +441,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; + if (!ctx->authsize) + return 0; + /* NULL encryption / decryption */ if (!ctx->enckeylen) return aead_null_set_sh_desc(aead); @@ -553,7 +556,10 @@ skip_enc: /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + if (alg->caam.geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); @@ -562,6 +568,14 @@ skip_enc: append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); + if (alg->caam.geniv) { + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); + } + /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | @@ -614,7 +628,7 @@ skip_enc: keys_fit_inline = true; /* aead_givencrypt shared descriptor */ - desc = ctx->sh_desc_givenc; + desc = ctx->sh_desc_enc; /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); @@ -645,13 +659,13 @@ copy_iv: append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* ivsize + cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - /* Read and write assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + /* ivsize + cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + /* Skip assoc data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); @@ -688,7 +702,9 @@ copy_iv: /* Will read cryptlen */ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); /* Write ICV */ append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | @@ -697,7 +713,7 @@ copy_iv: ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; } @@ -2147,7 +2163,7 @@ static void init_authenc_job(struct aead_request *req, init_aead_job(req, edesc, all_contig, encrypt); - if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) + if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | @@ -2534,20 +2550,6 @@ static int aead_decrypt(struct aead_request *req) return ret; } -static int aead_givdecrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - unsigned int ivsize = crypto_aead_ivsize(aead); - - if (req->cryptlen < ivsize) - return -EINVAL; - - req->cryptlen -= ivsize; - req->assoclen += ivsize; - - return aead_decrypt(req); -} - /* * allocate and map the ablkcipher extended descriptor for ablkcipher */ @@ -3207,7 +3209,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, }, @@ -3253,7 +3255,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -3299,7 +3301,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, }, @@ -3345,7 +3347,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -3391,7 +3393,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, }, @@ -3437,7 +3439,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, @@ -3483,7 +3485,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, }, @@ -3531,7 +3533,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -3579,7 +3581,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, }, @@ -3627,7 +3629,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -3675,7 +3677,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, }, @@ -3723,7 +3725,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, @@ -3769,7 +3771,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, }, @@ -3815,7 +3817,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -3861,7 +3863,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, }, @@ -3907,7 +3909,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -3953,7 +3955,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, }, @@ -3999,7 +4001,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, @@ -4048,7 +4050,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = MD5_DIGEST_SIZE, }, @@ -4099,7 +4101,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -4150,7 +4152,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, }, @@ -4201,7 +4203,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -4252,7 +4254,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, }, @@ -4303,7 +4305,7 @@ static struct caam_aead_alg driver_aeads[] = { .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_givdecrypt, + .decrypt = aead_decrypt, .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, @@ -4542,6 +4544,15 @@ static int __init caam_algapi_init(void) if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) continue; + /* + * Check support for AES modes not available + * on LP devices. + */ + if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) + if ((alg->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_XTS) + continue; + t_alg = caam_alg_alloc(alg); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); diff --git a/kernel/drivers/crypto/caam/caamhash.c b/kernel/drivers/crypto/caam/caamhash.c index 49106ea42..99d5e11db 100644 --- a/kernel/drivers/crypto/caam/caamhash.c +++ b/kernel/drivers/crypto/caam/caamhash.c @@ -1873,6 +1873,7 @@ caam_hash_alloc(struct caam_hash_template *template, template->name); snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", template->driver_name); + t_alg->ahash_alg.setkey = NULL; } alg->cra_module = THIS_MODULE; alg->cra_init = caam_hash_cra_init; diff --git a/kernel/drivers/crypto/caam/jr.c b/kernel/drivers/crypto/caam/jr.c index f7e0d8d4c..8f50a02ff 100644 --- a/kernel/drivers/crypto/caam/jr.c +++ b/kernel/drivers/crypto/caam/jr.c @@ -248,7 +248,7 @@ static void caam_jr_dequeue(unsigned long devarg) struct device *caam_jr_alloc(void) { struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; - struct device *dev = NULL; + struct device *dev = ERR_PTR(-ENODEV); int min_tfm_cnt = INT_MAX; int tfm_cnt; diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c index d89f20c04..60fc0fa26 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -220,6 +220,42 @@ static int ccp_aes_cmac_digest(struct ahash_request *req) return ccp_aes_cmac_finup(req); } +static int ccp_aes_cmac_export(struct ahash_request *req, void *out) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_aes_cmac_exp_ctx state; + + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); + + state.null_msg = rctx->null_msg; + memcpy(state.iv, rctx->iv, sizeof(state.iv)); + state.buf_count = rctx->buf_count; + memcpy(state.buf, rctx->buf, sizeof(state.buf)); + + /* 'out' may not be aligned so memcpy from local variable */ + memcpy(out, &state, sizeof(state)); + + return 0; +} + +static int ccp_aes_cmac_import(struct ahash_request *req, const void *in) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_aes_cmac_exp_ctx state; + + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); + + memset(rctx, 0, sizeof(*rctx)); + rctx->null_msg = state.null_msg; + memcpy(rctx->iv, state.iv, sizeof(rctx->iv)); + rctx->buf_count = state.buf_count; + memcpy(rctx->buf, state.buf, sizeof(rctx->buf)); + + return 0; +} + static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int key_len) { @@ -352,10 +388,13 @@ int ccp_register_aes_cmac_algs(struct list_head *head) alg->final = ccp_aes_cmac_final; alg->finup = ccp_aes_cmac_finup; alg->digest = ccp_aes_cmac_digest; + alg->export = ccp_aes_cmac_export; + alg->import = ccp_aes_cmac_import; alg->setkey = ccp_aes_cmac_setkey; halg = &alg->halg; halg->digestsize = AES_BLOCK_SIZE; + halg->statesize = sizeof(struct ccp_aes_cmac_exp_ctx); base = &halg->base; snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)"); diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/kernel/drivers/crypto/ccp/ccp-crypto-aes-xts.c index 52c7395cb..0d0d4529e 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-aes-xts.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -122,6 +122,7 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); unsigned int unit; + u32 unit_size; int ret; if (!ctx->u.aes.key_len) @@ -133,11 +134,17 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, if (!req->info) return -EINVAL; - for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) - if (!(req->nbytes & (unit_size_map[unit].size - 1))) - break; + unit_size = CCP_XTS_AES_UNIT_SIZE__LAST; + if (req->nbytes <= unit_size_map[0].size) { + for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) { + if (!(req->nbytes & (unit_size_map[unit].size - 1))) { + unit_size = unit_size_map[unit].value; + break; + } + } + } - if ((unit_size_map[unit].value == CCP_XTS_AES_UNIT_SIZE__LAST) || + if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) || (ctx->u.aes.key_len != AES_KEYSIZE_128)) { /* Use the fallback to process the request for any * unsupported unit sizes or key sizes @@ -158,7 +165,7 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; - rctx->cmd.u.xts.unit_size = unit_size_map[unit].value; + rctx->cmd.u.xts.unit_size = unit_size; rctx->cmd.u.xts.key = &ctx->u.aes.key_sg; rctx->cmd.u.xts.key_len = ctx->u.aes.key_len; rctx->cmd.u.xts.iv = &rctx->iv_sg; diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-sha.c b/kernel/drivers/crypto/ccp/ccp-crypto-sha.c index d14b3f28e..ab9945f2c 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-sha.c @@ -207,6 +207,46 @@ static int ccp_sha_digest(struct ahash_request *req) return ccp_sha_finup(req); } +static int ccp_sha_export(struct ahash_request *req, void *out) +{ + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_sha_exp_ctx state; + + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); + + state.type = rctx->type; + state.msg_bits = rctx->msg_bits; + state.first = rctx->first; + memcpy(state.ctx, rctx->ctx, sizeof(state.ctx)); + state.buf_count = rctx->buf_count; + memcpy(state.buf, rctx->buf, sizeof(state.buf)); + + /* 'out' may not be aligned so memcpy from local variable */ + memcpy(out, &state, sizeof(state)); + + return 0; +} + +static int ccp_sha_import(struct ahash_request *req, const void *in) +{ + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_sha_exp_ctx state; + + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); + + memset(rctx, 0, sizeof(*rctx)); + rctx->type = state.type; + rctx->msg_bits = state.msg_bits; + rctx->first = state.first; + memcpy(rctx->ctx, state.ctx, sizeof(rctx->ctx)); + rctx->buf_count = state.buf_count; + memcpy(rctx->buf, state.buf, sizeof(rctx->buf)); + + return 0; +} + static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int key_len) { @@ -403,9 +443,12 @@ static int ccp_register_sha_alg(struct list_head *head, alg->final = ccp_sha_final; alg->finup = ccp_sha_finup; alg->digest = ccp_sha_digest; + alg->export = ccp_sha_export; + alg->import = ccp_sha_import; halg = &alg->halg; halg->digestsize = def->digest_size; + halg->statesize = sizeof(struct ccp_sha_exp_ctx); base = &halg->base; snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); diff --git a/kernel/drivers/crypto/ccp/ccp-crypto.h b/kernel/drivers/crypto/ccp/ccp-crypto.h index 76a96f0f4..a326ec20b 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto.h +++ b/kernel/drivers/crypto/ccp/ccp-crypto.h @@ -129,6 +129,15 @@ struct ccp_aes_cmac_req_ctx { struct ccp_cmd cmd; }; +struct ccp_aes_cmac_exp_ctx { + unsigned int null_msg; + + u8 iv[AES_BLOCK_SIZE]; + + unsigned int buf_count; + u8 buf[AES_BLOCK_SIZE]; +}; + /***** SHA related defines *****/ #define MAX_SHA_CONTEXT_SIZE SHA256_DIGEST_SIZE #define MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE @@ -171,6 +180,19 @@ struct ccp_sha_req_ctx { struct ccp_cmd cmd; }; +struct ccp_sha_exp_ctx { + enum ccp_sha_type type; + + u64 msg_bits; + + unsigned int first; + + u8 ctx[MAX_SHA_CONTEXT_SIZE]; + + unsigned int buf_count; + u8 buf[MAX_SHA_BLOCK_SIZE]; +}; + /***** Common Context Structure *****/ struct ccp_ctx { int (*complete)(struct crypto_async_request *req, int ret); diff --git a/kernel/drivers/crypto/marvell/cesa.c b/kernel/drivers/crypto/marvell/cesa.c index c0656e7f3..80239ae69 100644 --- a/kernel/drivers/crypto/marvell/cesa.c +++ b/kernel/drivers/crypto/marvell/cesa.c @@ -420,7 +420,7 @@ static int mv_cesa_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); cesa->regs = devm_ioremap_resource(dev, res); if (IS_ERR(cesa->regs)) - return -ENOMEM; + return PTR_ERR(cesa->regs); ret = mv_cesa_dev_dma_init(cesa); if (ret) diff --git a/kernel/drivers/crypto/nx/nx-842-powernv.c b/kernel/drivers/crypto/nx/nx-842-powernv.c index 9ef51fafd..6e105e87b 100644 --- a/kernel/drivers/crypto/nx/nx-842-powernv.c +++ b/kernel/drivers/crypto/nx/nx-842-powernv.c @@ -442,6 +442,14 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, (unsigned int)ccw, (unsigned int)be32_to_cpu(crb->ccw)); + /* + * NX842 coprocessor sets 3rd bit in CR register with XER[S0]. + * XER[S0] is the integer summary overflow bit which is nothing + * to do NX. Since this bit can be set with other return values, + * mask this bit. + */ + ret &= ~ICSWX_XERS0; + switch (ret) { case ICSWX_INITIATED: ret = wait_for_csb(wmem, csb); @@ -454,10 +462,6 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, pr_err_ratelimited("ICSWX rejected\n"); ret = -EPROTO; break; - default: - pr_err_ratelimited("Invalid ICSWX return code %x\n", ret); - ret = -EPROTO; - break; } if (!ret) diff --git a/kernel/drivers/crypto/nx/nx.c b/kernel/drivers/crypto/nx/nx.c index 0794f1cc0..42f0f229f 100644 --- a/kernel/drivers/crypto/nx/nx.c +++ b/kernel/drivers/crypto/nx/nx.c @@ -392,7 +392,7 @@ static void nx_of_update_msc(struct device *dev, ((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) && i < msc->triplets; i++) { - if (msc->fc > NX_MAX_FC || msc->mode > NX_MAX_MODE) { + if (msc->fc >= NX_MAX_FC || msc->mode >= NX_MAX_MODE) { dev_err(dev, "unknown function code/mode " "combo: %d/%d (ignored)\n", msc->fc, msc->mode); diff --git a/kernel/drivers/crypto/qat/qat_common/Makefile b/kernel/drivers/crypto/qat/qat_common/Makefile index 9e9e196c6..45b5adaaf 100644 --- a/kernel/drivers/crypto/qat/qat_common/Makefile +++ b/kernel/drivers/crypto/qat/qat_common/Makefile @@ -2,6 +2,7 @@ $(obj)/qat_rsapubkey-asn1.o: $(obj)/qat_rsapubkey-asn1.c \ $(obj)/qat_rsapubkey-asn1.h $(obj)/qat_rsaprivkey-asn1.o: $(obj)/qat_rsaprivkey-asn1.c \ $(obj)/qat_rsaprivkey-asn1.h +$(obj)/qat_asym_algs.o: $(obj)/qat_rsapubkey-asn1.h $(obj)/qat_rsaprivkey-asn1.h clean-files += qat_rsapubkey-asn1.c qat_rsapubkey-asn1.h clean-files += qat_rsaprivkey-asn1.c qat_rsapvivkey-asn1.h diff --git a/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h b/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h index 3f76bd495..aa1dbeaa9 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -227,6 +227,8 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, uint32_t vf_mask); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, uint32_t vf_mask); +int adf_init_pf_wq(void); +void adf_exit_pf_wq(void); #else static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) { @@ -236,5 +238,14 @@ static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) { } + +static inline int adf_init_pf_wq(void) +{ + return 0; +} + +static inline void adf_exit_pf_wq(void) +{ +} #endif #endif diff --git a/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 473d36d91..e7480f373 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -469,12 +469,17 @@ static int __init adf_register_ctl_device_driver(void) if (adf_init_aer()) goto err_aer; + if (adf_init_pf_wq()) + goto err_pf_wq; + if (qat_crypto_register()) goto err_crypto_register; return 0; err_crypto_register: + adf_exit_pf_wq(); +err_pf_wq: adf_exit_aer(); err_aer: adf_chr_drv_destroy(); @@ -487,6 +492,7 @@ static void __exit adf_unregister_ctl_device_driver(void) { adf_chr_drv_destroy(); adf_exit_aer(); + adf_exit_pf_wq(); qat_crypto_unregister(); adf_clean_vf_map(false); mutex_destroy(&adf_ctl_lock); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_sriov.c b/kernel/drivers/crypto/qat/qat_common/adf_sriov.c index 1117a8b58..38a0415e7 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_sriov.c @@ -119,11 +119,6 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) int i; u32 reg; - /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); - if (!pf2vf_resp_wq) - return -ENOMEM; - for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; i++, vf_info++) { /* This ptr will be populated when VFs will be created */ @@ -216,11 +211,6 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) kfree(accel_dev->pf.vf_info); accel_dev->pf.vf_info = NULL; - - if (pf2vf_resp_wq) { - destroy_workqueue(pf2vf_resp_wq); - pf2vf_resp_wq = NULL; - } } EXPORT_SYMBOL_GPL(adf_disable_sriov); @@ -304,3 +294,19 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) return numvfs; } EXPORT_SYMBOL_GPL(adf_sriov_configure); + +int __init adf_init_pf_wq(void) +{ + /* Workqueue for PF2VF responses */ + pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + + return !pf2vf_resp_wq ? -ENOMEM : 0; +} + +void adf_exit_pf_wq(void) +{ + if (pf2vf_resp_wq) { + destroy_workqueue(pf2vf_resp_wq); + pf2vf_resp_wq = NULL; + } +} diff --git a/kernel/drivers/crypto/qat/qat_common/qat_algs.c b/kernel/drivers/crypto/qat/qat_common/qat_algs.c index 59e4c3af1..367b6661e 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_algs.c +++ b/kernel/drivers/crypto/qat/qat_common/qat_algs.c @@ -1262,8 +1262,8 @@ static struct crypto_alg qat_algs[] = { { .setkey = qat_alg_ablkcipher_xts_setkey, .decrypt = qat_alg_ablkcipher_decrypt, .encrypt = qat_alg_ablkcipher_encrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, }, }, diff --git a/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index a19ee127e..e72fea737 100644 --- a/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -35,6 +35,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ + unsigned long flags; if (areq->nbytes == 0) return 0; @@ -49,7 +50,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) return -EINVAL; } - spin_lock_bh(&ss->slock); + spin_lock_irqsave(&ss->slock, flags); for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); @@ -117,7 +118,7 @@ release_ss: sg_miter_stop(&mi); sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); + spin_unlock_irqrestore(&ss->slock, flags); return err; } @@ -149,6 +150,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ + unsigned long flags; if (areq->nbytes == 0) return 0; @@ -181,7 +183,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) if (no_chunk == 1) return sun4i_ss_opti_poll(areq); - spin_lock_bh(&ss->slock); + spin_lock_irqsave(&ss->slock, flags); for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); @@ -308,7 +310,7 @@ release_ss: sg_miter_stop(&mi); sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); + spin_unlock_irqrestore(&ss->slock, flags); return err; } diff --git a/kernel/drivers/crypto/talitos.c b/kernel/drivers/crypto/talitos.c index b6f9f42e2..9a8a18aaf 100644 --- a/kernel/drivers/crypto/talitos.c +++ b/kernel/drivers/crypto/talitos.c @@ -63,6 +63,14 @@ static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, ptr->eptr = upper_32_bits(dma_addr); } +static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, + struct talitos_ptr *src_ptr, bool is_sec1) +{ + dst_ptr->ptr = src_ptr->ptr; + if (!is_sec1) + dst_ptr->eptr = src_ptr->eptr; +} + static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, bool is_sec1) { @@ -827,6 +835,16 @@ struct talitos_ahash_req_ctx { struct scatterlist *psrc; }; +struct talitos_export_state { + u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; + u8 buf[HASH_MAX_BLOCK_SIZE]; + unsigned int swinit; + unsigned int first; + unsigned int last; + unsigned int to_hash_later; + unsigned int nbuf; +}; + static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { @@ -1083,21 +1101,20 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); - /* hmac data */ desc->ptr[1].len = cpu_to_be16(areq->assoclen); if (sg_count > 1 && (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, areq->assoclen, &edesc->link_tbl[tbl_off])) > 1) { - tbl_off += ret; - to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * sizeof(struct talitos_ptr), 0); desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + + tbl_off += ret; } else { to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); desc->ptr[1].j_extent = 0; @@ -1126,11 +1143,13 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) sg_link_tbl_len += authsize; - if (sg_count > 1 && - (ret = sg_to_link_tbl_offset(areq->src, sg_count, areq->assoclen, - sg_link_tbl_len, - &edesc->link_tbl[tbl_off])) > 1) { - tbl_off += ret; + if (sg_count == 1) { + to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) + + areq->assoclen, 0); + } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count, + areq->assoclen, sg_link_tbl_len, + &edesc->link_tbl[tbl_off])) > + 1) { desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + tbl_off * @@ -1138,8 +1157,10 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - } else - to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0); + tbl_off += ret; + } else { + copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0); + } /* cipher out */ desc->ptr[5].len = cpu_to_be16(cryptlen); @@ -1151,11 +1172,13 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->icv_ool = false; - if (sg_count > 1 && - (sg_count = sg_to_link_tbl_offset(areq->dst, sg_count, + if (sg_count == 1) { + to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) + + areq->assoclen, 0); + } else if ((sg_count = + sg_to_link_tbl_offset(areq->dst, sg_count, areq->assoclen, cryptlen, - &edesc->link_tbl[tbl_off])) > - 1) { + &edesc->link_tbl[tbl_off])) > 1) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + @@ -1178,8 +1201,9 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->dma_len, DMA_BIDIRECTIONAL); edesc->icv_ool = true; - } else - to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0); + } else { + copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0); + } /* iv out */ map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, @@ -1940,6 +1964,46 @@ static int ahash_digest(struct ahash_request *areq) return ahash_process_req(areq, areq->nbytes); } +static int ahash_export(struct ahash_request *areq, void *out) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_export_state *export = out; + + memcpy(export->hw_context, req_ctx->hw_context, + req_ctx->hw_context_size); + memcpy(export->buf, req_ctx->buf, req_ctx->nbuf); + export->swinit = req_ctx->swinit; + export->first = req_ctx->first; + export->last = req_ctx->last; + export->to_hash_later = req_ctx->to_hash_later; + export->nbuf = req_ctx->nbuf; + + return 0; +} + +static int ahash_import(struct ahash_request *areq, const void *in) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + const struct talitos_export_state *export = in; + + memset(req_ctx, 0, sizeof(*req_ctx)); + req_ctx->hw_context_size = + (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + memcpy(req_ctx->hw_context, export->hw_context, + req_ctx->hw_context_size); + memcpy(req_ctx->buf, export->buf, export->nbuf); + req_ctx->swinit = export->swinit; + req_ctx->first = export->first; + req_ctx->last = export->last; + req_ctx->to_hash_later = export->to_hash_later; + req_ctx->nbuf = export->nbuf; + + return 0; +} + struct keyhash_result { struct completion completion; int err; @@ -2334,6 +2398,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "md5", .cra_driver_name = "md5-talitos", @@ -2349,6 +2414,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha1", .cra_driver_name = "sha1-talitos", @@ -2364,6 +2430,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha224", .cra_driver_name = "sha224-talitos", @@ -2379,6 +2446,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha256", .cra_driver_name = "sha256-talitos", @@ -2394,6 +2462,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha384", .cra_driver_name = "sha384-talitos", @@ -2409,6 +2478,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha512", .cra_driver_name = "sha512-talitos", @@ -2424,6 +2494,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(md5)", .cra_driver_name = "hmac-md5-talitos", @@ -2439,6 +2510,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha1)", .cra_driver_name = "hmac-sha1-talitos", @@ -2454,6 +2526,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "hmac-sha224-talitos", @@ -2469,6 +2542,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "hmac-sha256-talitos", @@ -2484,6 +2558,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "hmac-sha384-talitos", @@ -2499,6 +2574,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "hmac-sha512-talitos", @@ -2519,21 +2595,11 @@ struct talitos_crypto_alg { struct talitos_alg_template algt; }; -static int talitos_cra_init(struct crypto_tfm *tfm) +static int talitos_init_common(struct talitos_ctx *ctx, + struct talitos_crypto_alg *talitos_alg) { - struct crypto_alg *alg = tfm->__crt_alg; - struct talitos_crypto_alg *talitos_alg; - struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); struct talitos_private *priv; - if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) - talitos_alg = container_of(__crypto_ahash_alg(alg), - struct talitos_crypto_alg, - algt.alg.hash); - else - talitos_alg = container_of(alg, struct talitos_crypto_alg, - algt.alg.crypto); - /* update context with ptr to dev */ ctx->dev = talitos_alg->dev; @@ -2551,10 +2617,33 @@ static int talitos_cra_init(struct crypto_tfm *tfm) return 0; } +static int talitos_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct talitos_crypto_alg *talitos_alg; + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + + if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) + talitos_alg = container_of(__crypto_ahash_alg(alg), + struct talitos_crypto_alg, + algt.alg.hash); + else + talitos_alg = container_of(alg, struct talitos_crypto_alg, + algt.alg.crypto); + + return talitos_init_common(ctx, talitos_alg); +} + static int talitos_cra_init_aead(struct crypto_aead *tfm) { - talitos_cra_init(crypto_aead_tfm(tfm)); - return 0; + struct aead_alg *alg = crypto_aead_alg(tfm); + struct talitos_crypto_alg *talitos_alg; + struct talitos_ctx *ctx = crypto_aead_ctx(tfm); + + talitos_alg = container_of(alg, struct talitos_crypto_alg, + algt.alg.aead); + + return talitos_init_common(ctx, talitos_alg); } static int talitos_cra_init_ahash(struct crypto_tfm *tfm) @@ -2677,6 +2766,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.hash.finup = ahash_finup; t_alg->algt.alg.hash.digest = ahash_digest; t_alg->algt.alg.hash.setkey = ahash_setkey; + t_alg->algt.alg.hash.import = ahash_import; + t_alg->algt.alg.hash.export = ahash_export; if (!(priv->features & TALITOS_FTR_HMAC_OK) && !strncmp(alg->cra_name, "hmac", 4)) { diff --git a/kernel/drivers/crypto/ux500/cryp/cryp_core.c b/kernel/drivers/crypto/ux500/cryp/cryp_core.c index 4c243c1ff..790f7cadc 100644 --- a/kernel/drivers/crypto/ux500/cryp/cryp_core.c +++ b/kernel/drivers/crypto/ux500/cryp/cryp_core.c @@ -1440,9 +1440,9 @@ static int ux500_cryp_probe(struct platform_device *pdev) device_data->phybase = res->start; device_data->base = devm_ioremap_resource(dev, res); - if (!device_data->base) { + if (IS_ERR(device_data->base)) { dev_err(dev, "[%s]: ioremap failed!", __func__); - ret = -ENOMEM; + ret = PTR_ERR(device_data->base); goto out; } diff --git a/kernel/drivers/crypto/ux500/hash/hash_core.c b/kernel/drivers/crypto/ux500/hash/hash_core.c index f47d11204..cd4398498 100644 --- a/kernel/drivers/crypto/ux500/hash/hash_core.c +++ b/kernel/drivers/crypto/ux500/hash/hash_core.c @@ -797,7 +797,7 @@ static int hash_process_data(struct hash_device_data *device_data, &device_data->state); memmove(req_ctx->state.buffer, device_data->state.buffer, - HASH_BLOCK_SIZE / sizeof(u32)); + HASH_BLOCK_SIZE); if (ret) { dev_err(device_data->dev, "%s: hash_resume_state() failed!\n", @@ -848,7 +848,7 @@ static int hash_process_data(struct hash_device_data *device_data, memmove(device_data->state.buffer, req_ctx->state.buffer, - HASH_BLOCK_SIZE / sizeof(u32)); + HASH_BLOCK_SIZE); if (ret) { dev_err(device_data->dev, "%s: hash_save_state() failed!\n", __func__); @@ -1675,9 +1675,9 @@ static int ux500_hash_probe(struct platform_device *pdev) device_data->phybase = res->start; device_data->base = devm_ioremap_resource(dev, res); - if (!device_data->base) { + if (IS_ERR(device_data->base)) { dev_err(dev, "%s: ioremap() failed!\n", __func__); - ret = -ENOMEM; + ret = PTR_ERR(device_data->base); goto out; } spin_lock_init(&device_data->ctx_lock); diff --git a/kernel/drivers/crypto/vmx/aes_cbc.c b/kernel/drivers/crypto/vmx/aes_cbc.c index 0b8fe2ec5..3f8bb9a40 100644 --- a/kernel/drivers/crypto/vmx/aes_cbc.c +++ b/kernel/drivers/crypto/vmx/aes_cbc.c @@ -182,7 +182,7 @@ struct crypto_alg p8_aes_cbc_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "p8_aes_cbc", .cra_module = THIS_MODULE, - .cra_priority = 1000, + .cra_priority = 2000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, @@ -191,7 +191,7 @@ struct crypto_alg p8_aes_cbc_alg = { .cra_init = p8_aes_cbc_init, .cra_exit = p8_aes_cbc_exit, .cra_blkcipher = { - .ivsize = 0, + .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_cbc_setkey, diff --git a/kernel/drivers/crypto/vmx/aes_ctr.c b/kernel/drivers/crypto/vmx/aes_ctr.c index ee1306cd8..72f138985 100644 --- a/kernel/drivers/crypto/vmx/aes_ctr.c +++ b/kernel/drivers/crypto/vmx/aes_ctr.c @@ -166,7 +166,7 @@ struct crypto_alg p8_aes_ctr_alg = { .cra_name = "ctr(aes)", .cra_driver_name = "p8_aes_ctr", .cra_module = THIS_MODULE, - .cra_priority = 1000, + .cra_priority = 2000, .cra_type = &crypto_blkcipher_type, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, .cra_alignmask = 0, @@ -175,7 +175,7 @@ struct crypto_alg p8_aes_ctr_alg = { .cra_init = p8_aes_ctr_init, .cra_exit = p8_aes_ctr_exit, .cra_blkcipher = { - .ivsize = 0, + .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_ctr_setkey, diff --git a/kernel/drivers/crypto/vmx/ghash.c b/kernel/drivers/crypto/vmx/ghash.c index 2183a2e77..9cb3a0b71 100644 --- a/kernel/drivers/crypto/vmx/ghash.c +++ b/kernel/drivers/crypto/vmx/ghash.c @@ -26,16 +26,13 @@ #include #include #include +#include #include #include #include #define IN_INTERRUPT in_interrupt() -#define GHASH_BLOCK_SIZE (16) -#define GHASH_DIGEST_SIZE (16) -#define GHASH_KEY_LEN (16) - void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], @@ -55,16 +52,11 @@ struct p8_ghash_desc_ctx { static int p8_ghash_init_tfm(struct crypto_tfm *tfm) { - const char *alg; + const char *alg = "ghash-generic"; struct crypto_shash *fallback; struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm); struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - fallback = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { printk(KERN_ERR @@ -78,10 +70,18 @@ static int p8_ghash_init_tfm(struct crypto_tfm *tfm) crypto_shash_set_flags(fallback, crypto_shash_get_flags((struct crypto_shash *) tfm)); - ctx->fallback = fallback; - shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx) - + crypto_shash_descsize(fallback); + /* Check if the descsize defined in the algorithm is still enough. */ + if (shash_tfm->descsize < sizeof(struct p8_ghash_desc_ctx) + + crypto_shash_descsize(fallback)) { + printk(KERN_ERR + "Desc size of the fallback implementation (%s) does not match the expected value: %lu vs %u\n", + alg, + shash_tfm->descsize - sizeof(struct p8_ghash_desc_ctx), + crypto_shash_descsize(fallback)); + return -EINVAL; + } + ctx->fallback = fallback; return 0; } @@ -113,7 +113,7 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, { struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); - if (keylen != GHASH_KEY_LEN) + if (keylen != GHASH_BLOCK_SIZE) return -EINVAL; preempt_disable(); @@ -215,7 +215,8 @@ struct shash_alg p8_ghash_alg = { .update = p8_ghash_update, .final = p8_ghash_final, .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx), + .descsize = sizeof(struct p8_ghash_desc_ctx) + + sizeof(struct ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "p8_ghash", diff --git a/kernel/drivers/crypto/vmx/ppc-xlate.pl b/kernel/drivers/crypto/vmx/ppc-xlate.pl index b9997335f..b18e67d0e 100644 --- a/kernel/drivers/crypto/vmx/ppc-xlate.pl +++ b/kernel/drivers/crypto/vmx/ppc-xlate.pl @@ -139,6 +139,26 @@ my $vmr = sub { " vor $vx,$vy,$vy"; }; +# Some ABIs specify vrsave, special-purpose register #256, as reserved +# for system use. +my $no_vrsave = ($flavour =~ /linux-ppc64le/); +my $mtspr = sub { + my ($f,$idx,$ra) = @_; + if ($idx == 256 && $no_vrsave) { + " or $ra,$ra,$ra"; + } else { + " mtspr $idx,$ra"; + } +}; +my $mfspr = sub { + my ($f,$rd,$idx) = @_; + if ($idx == 256 && $no_vrsave) { + " li $rd,-1"; + } else { + " mfspr $rd,$idx"; + } +}; + # PowerISA 2.06 stuff sub vsxmem_op { my ($f, $vrt, $ra, $rb, $op) = @_; diff --git a/kernel/drivers/dma/at_xdmac.c b/kernel/drivers/dma/at_xdmac.c index 02f9aa4eb..66c073fc8 100644 --- a/kernel/drivers/dma/at_xdmac.c +++ b/kernel/drivers/dma/at_xdmac.c @@ -242,7 +242,7 @@ struct at_xdmac_lld { u32 mbr_dus; /* Destination Microblock Stride Register */ }; - +/* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */ struct at_xdmac_desc { struct at_xdmac_lld lld; enum dma_transfer_direction direction; @@ -253,7 +253,7 @@ struct at_xdmac_desc { unsigned int xfer_size; struct list_head descs_list; struct list_head xfer_node; -}; +} __aligned(sizeof(u64)); static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb) { @@ -864,8 +864,12 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, * access. Hopefully we can access DDR through both ports (at least on * SAMA5D4x), so we can use the same interface for source and dest, * that solves the fact we don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DIF(0) + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_MEM_TRAN; @@ -1042,8 +1046,12 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * access DDR through both ports (at least on SAMA5D4x), so we can use * the same interface for source and dest, that solves the fact we * don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DAM_INCREMENTED_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) @@ -1144,8 +1152,12 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, * access. Hopefully we can access DDR through both ports (at least on * SAMA5D4x), so we can use the same interface for source and dest, * that solves the fact we don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DAM_UBS_AM + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DAM_UBS_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) @@ -1183,8 +1195,8 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, desc->lld.mbr_cfg = chan_cc; dev_dbg(chan2dev(chan), - "%s: lld: mbr_da=%pad, mbr_ds=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", - __func__, &desc->lld.mbr_da, &desc->lld.mbr_ds, desc->lld.mbr_ubc, + "%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", + __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc, desc->lld.mbr_cfg); return desc; @@ -1388,6 +1400,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, u32 cur_nda, check_nda, cur_ubc, mask, value; u8 dwidth = 0; unsigned long flags; + bool initd; ret = dma_cookie_status(chan, cookie, txstate); if (ret == DMA_COMPLETE) @@ -1412,7 +1425,16 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, residue = desc->xfer_size; /* * Flush FIFO: only relevant when the transfer is source peripheral - * synchronized. + * synchronized. Flush is needed before reading CUBC because data in + * the FIFO are not reported by CUBC. Reporting a residue of the + * transfer length while we have data in FIFO can cause issue. + * Usecase: atmel USART has a timeout which means I have received + * characters but there is no more character received for a while. On + * timeout, it requests the residue. If the data are in the DMA FIFO, + * we will return a residue of the transfer length. It means no data + * received. If an application is waiting for these data, it will hang + * since we won't have another USART timeout without receiving new + * data. */ mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC; value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM; @@ -1423,34 +1445,43 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, } /* - * When processing the residue, we need to read two registers but we - * can't do it in an atomic way. AT_XDMAC_CNDA is used to find where - * we stand in the descriptor list and AT_XDMAC_CUBC is used - * to know how many data are remaining for the current descriptor. - * Since the dma channel is not paused to not loose data, between the - * AT_XDMAC_CNDA and AT_XDMAC_CUBC read, we may have change of - * descriptor. - * For that reason, after reading AT_XDMAC_CUBC, we check if we are - * still using the same descriptor by reading a second time - * AT_XDMAC_CNDA. If AT_XDMAC_CNDA has changed, it means we have to - * read again AT_XDMAC_CUBC. + * The easiest way to compute the residue should be to pause the DMA + * but doing this can lead to miss some data as some devices don't + * have FIFO. + * We need to read several registers because: + * - DMA is running therefore a descriptor change is possible while + * reading these registers + * - When the block transfer is done, the value of the CUBC register + * is set to its initial value until the fetch of the next descriptor. + * This value will corrupt the residue calculation so we have to skip + * it. + * + * INITD -------- ------------ + * |____________________| + * _______________________ _______________ + * NDA @desc2 \/ @desc3 + * _______________________/\_______________ + * __________ ___________ _______________ + * CUBC 0 \/ MAX desc1 \/ MAX desc2 + * __________/\___________/\_______________ + * + * Since descriptors are aligned on 64 bits, we can assume that + * the update of NDA and CUBC is atomic. * Memory barriers are used to ensure the read order of the registers. - * A max number of retries is set because unlikely it can never ends if - * we are transferring a lot of data with small buffers. + * A max number of retries is set because unlikely it could never ends. */ - cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; - rmb(); - cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC); for (retry = 0; retry < AT_XDMAC_RESIDUE_MAX_RETRIES; retry++) { - rmb(); check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; - - if (likely(cur_nda == check_nda)) - break; - - cur_nda = check_nda; + rmb(); + initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD); rmb(); cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC); + rmb(); + cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc; + rmb(); + + if ((check_nda == cur_nda) && initd) + break; } if (unlikely(retry >= AT_XDMAC_RESIDUE_MAX_RETRIES)) { @@ -1458,6 +1489,19 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, goto spin_unlock; } + /* + * Flush FIFO: only relevant when the transfer is source peripheral + * synchronized. Another flush is needed here because CUBC is updated + * when the controller sends the data write command. It can lead to + * report data that are not written in the memory or the device. The + * FIFO flush ensures that data are really written. + */ + if ((desc->lld.mbr_cfg & mask) == value) { + at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask); + while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS)) + cpu_relax(); + } + /* * Remove size of all microblocks already transferred and the current * one. Then add the remaining size to transfer of the current @@ -2023,7 +2067,7 @@ err_dma_unregister: err_clk_disable: clk_disable_unprepare(atxdmac->clk); err_free_irq: - free_irq(atxdmac->irq, atxdmac->dma.dev); + free_irq(atxdmac->irq, atxdmac); return ret; } @@ -2039,7 +2083,7 @@ static int at_xdmac_remove(struct platform_device *pdev) synchronize_irq(atxdmac->irq); - free_irq(atxdmac->irq, atxdmac->dma.dev); + free_irq(atxdmac->irq, atxdmac); for (i = 0; i < atxdmac->dma.chancnt; i++) { struct at_xdmac_chan *atchan = &atxdmac->chan[i]; diff --git a/kernel/drivers/dma/dw/core.c b/kernel/drivers/dma/dw/core.c index 4f099ea29..c66133b5e 100644 --- a/kernel/drivers/dma/dw/core.c +++ b/kernel/drivers/dma/dw/core.c @@ -130,26 +130,14 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc) static void dwc_initialize(struct dw_dma_chan *dwc) { struct dw_dma *dw = to_dw_dma(dwc->chan.device); - struct dw_dma_slave *dws = dwc->chan.private; u32 cfghi = DWC_CFGH_FIFO_MODE; u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority); if (dwc->initialized == true) return; - if (dws) { - /* - * We need controller-specific data to set up slave - * transfers. - */ - BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev); - - cfghi |= DWC_CFGH_DST_PER(dws->dst_id); - cfghi |= DWC_CFGH_SRC_PER(dws->src_id); - } else { - cfghi |= DWC_CFGH_DST_PER(dwc->dst_id); - cfghi |= DWC_CFGH_SRC_PER(dwc->src_id); - } + cfghi |= DWC_CFGH_DST_PER(dwc->dst_id); + cfghi |= DWC_CFGH_SRC_PER(dwc->src_id); channel_writel(dwc, CFG_LO, cfglo); channel_writel(dwc, CFG_HI, cfghi); @@ -936,7 +924,7 @@ bool dw_dma_filter(struct dma_chan *chan, void *param) struct dw_dma_chan *dwc = to_dw_dma_chan(chan); struct dw_dma_slave *dws = param; - if (!dws || dws->dma_dev != chan->device->dev) + if (dws->dma_dev != chan->device->dev) return false; /* We have to copy data since dws can be temporary storage */ @@ -1160,6 +1148,14 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan) * doesn't mean what you think it means), and status writeback. */ + /* + * We need controller-specific data to set up slave transfers. + */ + if (chan->private && !dw_dma_filter(chan, chan->private)) { + dev_warn(chan2dev(chan), "Wrong controller-specific data\n"); + return -EINVAL; + } + /* Enable controller here if needed */ if (!dw->in_use) dw_dma_on(dw); @@ -1221,6 +1217,14 @@ static void dwc_free_chan_resources(struct dma_chan *chan) spin_lock_irqsave(&dwc->lock, flags); list_splice_init(&dwc->free_list, &list); dwc->descs_allocated = 0; + + /* Clear custom channel configuration */ + dwc->src_id = 0; + dwc->dst_id = 0; + + dwc->src_master = 0; + dwc->dst_master = 0; + dwc->initialized = false; /* Disable interrupts */ diff --git a/kernel/drivers/dma/hsu/hsu.c b/kernel/drivers/dma/hsu/hsu.c index 823ad728a..efc02b98e 100644 --- a/kernel/drivers/dma/hsu/hsu.c +++ b/kernel/drivers/dma/hsu/hsu.c @@ -135,7 +135,7 @@ static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc) sr = hsu_chan_readl(hsuc, HSU_CH_SR); spin_unlock_irqrestore(&hsuc->vchan.lock, flags); - return sr; + return sr & ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY); } irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) diff --git a/kernel/drivers/dma/hsu/hsu.h b/kernel/drivers/dma/hsu/hsu.h index f06579c6d..26da2865b 100644 --- a/kernel/drivers/dma/hsu/hsu.h +++ b/kernel/drivers/dma/hsu/hsu.h @@ -41,6 +41,9 @@ #define HSU_CH_SR_DESCTO(x) BIT(8 + (x)) #define HSU_CH_SR_DESCTO_ANY (BIT(11) | BIT(10) | BIT(9) | BIT(8)) #define HSU_CH_SR_CHE BIT(15) +#define HSU_CH_SR_DESCE(x) BIT(16 + (x)) +#define HSU_CH_SR_DESCE_ANY (BIT(19) | BIT(18) | BIT(17) | BIT(16)) +#define HSU_CH_SR_CDESC_ANY (BIT(31) | BIT(30)) /* Bits in HSU_CH_CR */ #define HSU_CH_CR_CHA BIT(0) diff --git a/kernel/drivers/dma/ipu/ipu_irq.c b/kernel/drivers/dma/ipu/ipu_irq.c index 2bf37e68a..dd184b50e 100644 --- a/kernel/drivers/dma/ipu/ipu_irq.c +++ b/kernel/drivers/dma/ipu/ipu_irq.c @@ -286,22 +286,21 @@ static void ipu_irq_handler(struct irq_desc *desc) raw_spin_unlock(&bank_lock); while ((line = ffs(status))) { struct ipu_irq_map *map; - unsigned int irq = NO_IRQ; + unsigned int irq; line--; status &= ~(1UL << line); raw_spin_lock(&bank_lock); map = src2map(32 * i + line); - if (map) - irq = map->irq; - raw_spin_unlock(&bank_lock); - if (!map) { + raw_spin_unlock(&bank_lock); pr_err("IPU: Interrupt on unmapped source %u bank %d\n", line, i); continue; } + irq = map->irq; + raw_spin_unlock(&bank_lock); generic_handle_irq(irq); } } diff --git a/kernel/drivers/dma/pl330.c b/kernel/drivers/dma/pl330.c index 17ee758b4..8250950aa 100644 --- a/kernel/drivers/dma/pl330.c +++ b/kernel/drivers/dma/pl330.c @@ -445,6 +445,9 @@ struct dma_pl330_chan { /* for cyclic capability */ bool cyclic; + + /* for runtime pm tracking */ + bool active; }; struct pl330_dmac { @@ -1994,6 +1997,7 @@ static void pl330_tasklet(unsigned long data) _stop(pch->thread); spin_unlock(&pch->thread->dmac->lock); power_down = true; + pch->active = false; } else { /* Make sure the PL330 Channel thread is active */ spin_lock(&pch->thread->dmac->lock); @@ -2015,6 +2019,7 @@ static void pl330_tasklet(unsigned long data) desc->status = PREP; list_move_tail(&desc->node, &pch->work_list); if (power_down) { + pch->active = true; spin_lock(&pch->thread->dmac->lock); _start(pch->thread); spin_unlock(&pch->thread->dmac->lock); @@ -2129,6 +2134,7 @@ static int pl330_terminate_all(struct dma_chan *chan) unsigned long flags; struct pl330_dmac *pl330 = pch->dmac; LIST_HEAD(list); + bool power_down = false; pm_runtime_get_sync(pl330->ddma.dev); spin_lock_irqsave(&pch->lock, flags); @@ -2139,6 +2145,8 @@ static int pl330_terminate_all(struct dma_chan *chan) pch->thread->req[0].desc = NULL; pch->thread->req[1].desc = NULL; pch->thread->req_running = -1; + power_down = pch->active; + pch->active = false; /* Mark all desc done */ list_for_each_entry(desc, &pch->submitted_list, node) { @@ -2156,6 +2164,8 @@ static int pl330_terminate_all(struct dma_chan *chan) list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); pm_runtime_mark_last_busy(pl330->ddma.dev); + if (power_down) + pm_runtime_put_autosuspend(pl330->ddma.dev); pm_runtime_put_autosuspend(pl330->ddma.dev); return 0; @@ -2302,6 +2312,7 @@ static void pl330_issue_pending(struct dma_chan *chan) * updated on work_list emptiness status. */ WARN_ON(list_empty(&pch->submitted_list)); + pch->active = true; pm_runtime_get_sync(pch->dmac->ddma.dev); } list_splice_tail_init(&pch->submitted_list, &pch->work_list); diff --git a/kernel/drivers/dma/pxa_dma.c b/kernel/drivers/dma/pxa_dma.c index a59061e42..55f5d33f6 100644 --- a/kernel/drivers/dma/pxa_dma.c +++ b/kernel/drivers/dma/pxa_dma.c @@ -122,6 +122,7 @@ struct pxad_chan { struct pxad_device { struct dma_device slave; int nr_chans; + int nr_requestors; void __iomem *base; struct pxad_phy *phys; spinlock_t phy_lock; /* Phy association */ @@ -473,7 +474,7 @@ static void pxad_free_phy(struct pxad_chan *chan) return; /* clear the channel mapping in DRCMR */ - if (chan->drcmr <= DRCMR_CHLNUM) { + if (chan->drcmr <= pdev->nr_requestors) { reg = pxad_drcmr(chan->drcmr); writel_relaxed(0, chan->phy->base + reg); } @@ -509,6 +510,7 @@ static bool is_running_chan_misaligned(struct pxad_chan *chan) static void phy_enable(struct pxad_phy *phy, bool misaligned) { + struct pxad_device *pdev; u32 reg, dalgn; if (!phy->vchan) @@ -518,7 +520,8 @@ static void phy_enable(struct pxad_phy *phy, bool misaligned) "%s(); phy=%p(%d) misaligned=%d\n", __func__, phy, phy->idx, misaligned); - if (phy->vchan->drcmr <= DRCMR_CHLNUM) { + pdev = to_pxad_dev(phy->vchan->vc.chan.device); + if (phy->vchan->drcmr <= pdev->nr_requestors) { reg = pxad_drcmr(phy->vchan->drcmr); writel_relaxed(DRCMR_MAPVLD | phy->idx, phy->base + reg); } @@ -914,6 +917,7 @@ static void pxad_get_config(struct pxad_chan *chan, { u32 maxburst = 0, dev_addr = 0; enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; + struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device); *dcmd = 0; if (dir == DMA_DEV_TO_MEM) { @@ -922,7 +926,7 @@ static void pxad_get_config(struct pxad_chan *chan, dev_addr = chan->cfg.src_addr; *dev_src = dev_addr; *dcmd |= PXA_DCMD_INCTRGADDR; - if (chan->drcmr <= DRCMR_CHLNUM) + if (chan->drcmr <= pdev->nr_requestors) *dcmd |= PXA_DCMD_FLOWSRC; } if (dir == DMA_MEM_TO_DEV) { @@ -931,7 +935,7 @@ static void pxad_get_config(struct pxad_chan *chan, dev_addr = chan->cfg.dst_addr; *dev_dst = dev_addr; *dcmd |= PXA_DCMD_INCSRCADDR; - if (chan->drcmr <= DRCMR_CHLNUM) + if (chan->drcmr <= pdev->nr_requestors) *dcmd |= PXA_DCMD_FLOWTRG; } if (dir == DMA_MEM_TO_MEM) @@ -1341,13 +1345,15 @@ static struct dma_chan *pxad_dma_xlate(struct of_phandle_args *dma_spec, static int pxad_init_dmadev(struct platform_device *op, struct pxad_device *pdev, - unsigned int nr_phy_chans) + unsigned int nr_phy_chans, + unsigned int nr_requestors) { int ret; unsigned int i; struct pxad_chan *c; pdev->nr_chans = nr_phy_chans; + pdev->nr_requestors = nr_requestors; INIT_LIST_HEAD(&pdev->slave.channels); pdev->slave.device_alloc_chan_resources = pxad_alloc_chan_resources; pdev->slave.device_free_chan_resources = pxad_free_chan_resources; @@ -1382,7 +1388,7 @@ static int pxad_probe(struct platform_device *op) const struct of_device_id *of_id; struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); struct resource *iores; - int ret, dma_channels = 0; + int ret, dma_channels = 0, nb_requestors = 0; const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES; @@ -1399,13 +1405,23 @@ static int pxad_probe(struct platform_device *op) return PTR_ERR(pdev->base); of_id = of_match_device(pxad_dt_ids, &op->dev); - if (of_id) + if (of_id) { of_property_read_u32(op->dev.of_node, "#dma-channels", &dma_channels); - else if (pdata && pdata->dma_channels) + ret = of_property_read_u32(op->dev.of_node, "#dma-requests", + &nb_requestors); + if (ret) { + dev_warn(pdev->slave.dev, + "#dma-requests set to default 32 as missing in OF: %d", + ret); + nb_requestors = 32; + }; + } else if (pdata && pdata->dma_channels) { dma_channels = pdata->dma_channels; - else + nb_requestors = pdata->nb_requestors; + } else { dma_channels = 32; /* default 32 channel */ + } dma_cap_set(DMA_SLAVE, pdev->slave.cap_mask); dma_cap_set(DMA_MEMCPY, pdev->slave.cap_mask); @@ -1422,7 +1438,7 @@ static int pxad_probe(struct platform_device *op) pdev->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; pdev->slave.dev = &op->dev; - ret = pxad_init_dmadev(op, pdev, dma_channels); + ret = pxad_init_dmadev(op, pdev, dma_channels, nb_requestors); if (ret) { dev_err(pdev->slave.dev, "unable to register\n"); return ret; @@ -1441,7 +1457,8 @@ static int pxad_probe(struct platform_device *op) platform_set_drvdata(op, pdev); pxad_init_debugfs(pdev); - dev_info(pdev->slave.dev, "initialized %d channels\n", dma_channels); + dev_info(pdev->slave.dev, "initialized %d channels on %d requestors\n", + dma_channels, nb_requestors); return 0; } diff --git a/kernel/drivers/dma/sh/usb-dmac.c b/kernel/drivers/dma/sh/usb-dmac.c index f1bcc2a16..b1bc945f0 100644 --- a/kernel/drivers/dma/sh/usb-dmac.c +++ b/kernel/drivers/dma/sh/usb-dmac.c @@ -600,27 +600,30 @@ static irqreturn_t usb_dmac_isr_channel(int irq, void *dev) { struct usb_dmac_chan *chan = dev; irqreturn_t ret = IRQ_NONE; - u32 mask = USB_DMACHCR_TE; - u32 check_bits = USB_DMACHCR_TE | USB_DMACHCR_SP; + u32 mask = 0; u32 chcr; + bool xfer_end = false; spin_lock(&chan->vc.lock); chcr = usb_dmac_chan_read(chan, USB_DMACHCR); - if (chcr & check_bits) - mask |= USB_DMACHCR_DE | check_bits; + if (chcr & (USB_DMACHCR_TE | USB_DMACHCR_SP)) { + mask |= USB_DMACHCR_DE | USB_DMACHCR_TE | USB_DMACHCR_SP; + if (chcr & USB_DMACHCR_DE) + xfer_end = true; + ret |= IRQ_HANDLED; + } if (chcr & USB_DMACHCR_NULL) { /* An interruption of TE will happen after we set FTE */ mask |= USB_DMACHCR_NULL; chcr |= USB_DMACHCR_FTE; ret |= IRQ_HANDLED; } - usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask); + if (mask) + usb_dmac_chan_write(chan, USB_DMACHCR, chcr & ~mask); - if (chcr & check_bits) { + if (xfer_end) usb_dmac_isr_transfer_end(chan); - ret |= IRQ_HANDLED; - } spin_unlock(&chan->vc.lock); diff --git a/kernel/drivers/edac/amd64_edac.c b/kernel/drivers/edac/amd64_edac.c index 9eee13ef8..d87a47547 100644 --- a/kernel/drivers/edac/amd64_edac.c +++ b/kernel/drivers/edac/amd64_edac.c @@ -1452,7 +1452,7 @@ static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, u64 chan_off; u64 dram_base = get_dram_base(pvt, range); u64 hole_off = f10_dhar_offset(pvt); - u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; + u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16; if (hi_rng) { /* diff --git a/kernel/drivers/edac/edac_mc.c b/kernel/drivers/edac/edac_mc.c index 1b2c2187b..dc68394da 100644 --- a/kernel/drivers/edac/edac_mc.c +++ b/kernel/drivers/edac/edac_mc.c @@ -966,7 +966,7 @@ static void edac_inc_ue_error(struct mem_ctl_info *mci, mci->ue_mc += count; if (!enable_per_layer_report) { - mci->ce_noinfo_count += count; + mci->ue_noinfo_count += count; return; } diff --git a/kernel/drivers/edac/edac_mc_sysfs.c b/kernel/drivers/edac/edac_mc_sysfs.c index 58aed67b7..3c8f19f5a 100644 --- a/kernel/drivers/edac/edac_mc_sysfs.c +++ b/kernel/drivers/edac/edac_mc_sysfs.c @@ -313,7 +313,6 @@ static struct device_type csrow_attr_type = { * possible dynamic channel DIMM Label attribute files * */ - DEVICE_CHANNEL(ch0_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 0); DEVICE_CHANNEL(ch1_dimm_label, S_IRUGO | S_IWUSR, @@ -326,6 +325,10 @@ DEVICE_CHANNEL(ch4_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 4); DEVICE_CHANNEL(ch5_dimm_label, S_IRUGO | S_IWUSR, channel_dimm_label_show, channel_dimm_label_store, 5); +DEVICE_CHANNEL(ch6_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 6); +DEVICE_CHANNEL(ch7_dimm_label, S_IRUGO | S_IWUSR, + channel_dimm_label_show, channel_dimm_label_store, 7); /* Total possible dynamic DIMM Label attribute file table */ static struct attribute *dynamic_csrow_dimm_attr[] = { @@ -335,6 +338,8 @@ static struct attribute *dynamic_csrow_dimm_attr[] = { &dev_attr_legacy_ch3_dimm_label.attr.attr, &dev_attr_legacy_ch4_dimm_label.attr.attr, &dev_attr_legacy_ch5_dimm_label.attr.attr, + &dev_attr_legacy_ch6_dimm_label.attr.attr, + &dev_attr_legacy_ch7_dimm_label.attr.attr, NULL }; @@ -351,6 +356,10 @@ DEVICE_CHANNEL(ch4_ce_count, S_IRUGO, channel_ce_count_show, NULL, 4); DEVICE_CHANNEL(ch5_ce_count, S_IRUGO, channel_ce_count_show, NULL, 5); +DEVICE_CHANNEL(ch6_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 6); +DEVICE_CHANNEL(ch7_ce_count, S_IRUGO, + channel_ce_count_show, NULL, 7); /* Total possible dynamic ce_count attribute file table */ static struct attribute *dynamic_csrow_ce_count_attr[] = { @@ -360,6 +369,8 @@ static struct attribute *dynamic_csrow_ce_count_attr[] = { &dev_attr_legacy_ch3_ce_count.attr.attr, &dev_attr_legacy_ch4_ce_count.attr.attr, &dev_attr_legacy_ch5_ce_count.attr.attr, + &dev_attr_legacy_ch6_ce_count.attr.attr, + &dev_attr_legacy_ch7_ce_count.attr.attr, NULL }; @@ -371,9 +382,16 @@ static umode_t csrow_dev_is_visible(struct kobject *kobj, if (idx >= csrow->nr_channels) return 0; + + if (idx >= ARRAY_SIZE(dynamic_csrow_ce_count_attr) - 1) { + WARN_ONCE(1, "idx: %d\n", idx); + return 0; + } + /* Only expose populated DIMMs */ if (!csrow->channels[idx]->dimm->nr_pages) return 0; + return attr->mode; } diff --git a/kernel/drivers/edac/i7core_edac.c b/kernel/drivers/edac/i7core_edac.c index 01087a38d..792bdae2b 100644 --- a/kernel/drivers/edac/i7core_edac.c +++ b/kernel/drivers/edac/i7core_edac.c @@ -1866,7 +1866,7 @@ static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val, i7_dev = get_i7core_dev(mce->socketid); if (!i7_dev) - return NOTIFY_BAD; + return NOTIFY_DONE; mci = i7_dev->mci; pvt = mci->pvt_info; diff --git a/kernel/drivers/edac/sb_edac.c b/kernel/drivers/edac/sb_edac.c index 429309c62..ca64b174f 100644 --- a/kernel/drivers/edac/sb_edac.c +++ b/kernel/drivers/edac/sb_edac.c @@ -218,8 +218,11 @@ static const u32 rir_offset[MAX_RIR_RANGES][MAX_RIR_WAY] = { { 0x1a0, 0x1a4, 0x1a8, 0x1ac, 0x1b0, 0x1b4, 0x1b8, 0x1bc }, }; -#define RIR_RNK_TGT(reg) GET_BITFIELD(reg, 16, 19) -#define RIR_OFFSET(reg) GET_BITFIELD(reg, 2, 14) +#define RIR_RNK_TGT(type, reg) (((type) == BROADWELL) ? \ + GET_BITFIELD(reg, 20, 23) : GET_BITFIELD(reg, 16, 19)) + +#define RIR_OFFSET(type, reg) (((type) == HASWELL || (type) == BROADWELL) ? \ + GET_BITFIELD(reg, 2, 15) : GET_BITFIELD(reg, 2, 14)) /* Device 16, functions 2-7 */ @@ -1117,8 +1120,8 @@ static void get_memory_layout(const struct mem_ctl_info *mci) edac_dbg(0, "TAD#%d: up to %u.%03u GB (0x%016Lx), socket interleave %d, memory interleave %d, TGT: %d, %d, %d, %d, reg=0x%08x\n", n_tads, gb, (mb*1000)/1024, ((u64)tmp_mb) << 20L, - (u32)TAD_SOCK(reg), - (u32)TAD_CH(reg), + (u32)(1 << TAD_SOCK(reg)), + (u32)TAD_CH(reg) + 1, (u32)TAD_TGT0(reg), (u32)TAD_TGT1(reg), (u32)TAD_TGT2(reg), @@ -1175,14 +1178,14 @@ static void get_memory_layout(const struct mem_ctl_info *mci) pci_read_config_dword(pvt->pci_tad[i], rir_offset[j][k], ®); - tmp_mb = RIR_OFFSET(reg) << 6; + tmp_mb = RIR_OFFSET(pvt->info.type, reg) << 6; gb = div_u64_rem(tmp_mb, 1024, &mb); edac_dbg(0, "CH#%d RIR#%d INTL#%d, offset %u.%03u GB (0x%016Lx), tgt: %d, reg=0x%08x\n", i, j, k, gb, (mb*1000)/1024, ((u64)tmp_mb) << 20L, - (u32)RIR_RNK_TGT(reg), + (u32)RIR_RNK_TGT(pvt->info.type, reg), reg); } } @@ -1396,7 +1399,7 @@ static int get_memory_error_data(struct mem_ctl_info *mci, } ch_way = TAD_CH(reg) + 1; - sck_way = TAD_SOCK(reg) + 1; + sck_way = TAD_SOCK(reg); if (ch_way == 3) idx = addr >> 6; @@ -1435,7 +1438,7 @@ static int get_memory_error_data(struct mem_ctl_info *mci, switch(ch_way) { case 2: case 4: - sck_xch = 1 << sck_way * (ch_way >> 1); + sck_xch = (1 << sck_way) * (ch_way >> 1); break; default: sprintf(msg, "Invalid mirror set. Can't decode addr"); @@ -1453,7 +1456,7 @@ static int get_memory_error_data(struct mem_ctl_info *mci, n_tads, addr, limit, - (u32)TAD_SOCK(reg), + sck_way, ch_way, offset, idx, @@ -1468,18 +1471,12 @@ static int get_memory_error_data(struct mem_ctl_info *mci, offset, addr); return -EINVAL; } - addr -= offset; - /* Store the low bits [0:6] of the addr */ - ch_addr = addr & 0x7f; - /* Remove socket wayness and remove 6 bits */ - addr >>= 6; - addr = div_u64(addr, sck_xch); -#if 0 - /* Divide by channel way */ - addr = addr / ch_way; -#endif - /* Recover the last 6 bits */ - ch_addr |= addr << 6; + + ch_addr = addr - offset; + ch_addr >>= (6 + shiftup); + ch_addr /= sck_xch; + ch_addr <<= (6 + shiftup); + ch_addr |= addr & ((1 << (6 + shiftup)) - 1); /* * Step 3) Decode rank @@ -1518,7 +1515,7 @@ static int get_memory_error_data(struct mem_ctl_info *mci, pci_read_config_dword(pvt->pci_tad[ch_add + base_ch], rir_offset[n_rir][idx], ®); - *rank = RIR_RNK_TGT(reg); + *rank = RIR_RNK_TGT(pvt->info.type, reg); edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n", n_rir, @@ -2260,7 +2257,7 @@ static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, mci = get_mci_for_node_id(mce->socketid); if (!mci) - return NOTIFY_BAD; + return NOTIFY_DONE; pvt = mci->pvt_info; /* diff --git a/kernel/drivers/extcon/extcon-max77843.c b/kernel/drivers/extcon/extcon-max77843.c index 9f9ea3343..b6cb30d20 100644 --- a/kernel/drivers/extcon/extcon-max77843.c +++ b/kernel/drivers/extcon/extcon-max77843.c @@ -803,7 +803,7 @@ static int max77843_muic_probe(struct platform_device *pdev) /* Clear IRQ bits before request IRQs */ ret = regmap_bulk_read(max77843->regmap_muic, MAX77843_MUIC_REG_INT1, info->status, - MAX77843_MUIC_IRQ_NUM); + MAX77843_MUIC_STATUS_NUM); if (ret) { dev_err(&pdev->dev, "Failed to Clear IRQ bits\n"); goto err_muic_irq; diff --git a/kernel/drivers/firewire/net.c b/kernel/drivers/firewire/net.c index f4ea80d60..b9d2f76a0 100644 --- a/kernel/drivers/firewire/net.c +++ b/kernel/drivers/firewire/net.c @@ -73,13 +73,13 @@ struct rfc2734_header { #define fwnet_get_hdr_lf(h) (((h)->w0 & 0xc0000000) >> 30) #define fwnet_get_hdr_ether_type(h) (((h)->w0 & 0x0000ffff)) -#define fwnet_get_hdr_dg_size(h) (((h)->w0 & 0x0fff0000) >> 16) +#define fwnet_get_hdr_dg_size(h) ((((h)->w0 & 0x0fff0000) >> 16) + 1) #define fwnet_get_hdr_fg_off(h) (((h)->w0 & 0x00000fff)) #define fwnet_get_hdr_dgl(h) (((h)->w1 & 0xffff0000) >> 16) -#define fwnet_set_hdr_lf(lf) ((lf) << 30) +#define fwnet_set_hdr_lf(lf) ((lf) << 30) #define fwnet_set_hdr_ether_type(et) (et) -#define fwnet_set_hdr_dg_size(dgs) ((dgs) << 16) +#define fwnet_set_hdr_dg_size(dgs) (((dgs) - 1) << 16) #define fwnet_set_hdr_fg_off(fgo) (fgo) #define fwnet_set_hdr_dgl(dgl) ((dgl) << 16) @@ -578,6 +578,9 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, int retval; u16 ether_type; + if (len <= RFC2374_UNFRAG_HDR_SIZE) + return 0; + hdr.w0 = be32_to_cpu(buf[0]); lf = fwnet_get_hdr_lf(&hdr); if (lf == RFC2374_HDR_UNFRAG) { @@ -602,7 +605,12 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, return fwnet_finish_incoming_packet(net, skb, source_node_id, is_broadcast, ether_type); } + /* A datagram fragment has been received, now the fun begins. */ + + if (len <= RFC2374_FRAG_HDR_SIZE) + return 0; + hdr.w1 = ntohl(buf[1]); buf += 2; len -= RFC2374_FRAG_HDR_SIZE; @@ -614,7 +622,10 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, fg_off = fwnet_get_hdr_fg_off(&hdr); } datagram_label = fwnet_get_hdr_dgl(&hdr); - dg_size = fwnet_get_hdr_dg_size(&hdr); /* ??? + 1 */ + dg_size = fwnet_get_hdr_dg_size(&hdr); + + if (fg_off + len > dg_size) + return 0; spin_lock_irqsave(&dev->lock, flags); @@ -722,6 +733,22 @@ static void fwnet_receive_packet(struct fw_card *card, struct fw_request *r, fw_send_response(card, r, rcode); } +static int gasp_source_id(__be32 *p) +{ + return be32_to_cpu(p[0]) >> 16; +} + +static u32 gasp_specifier_id(__be32 *p) +{ + return (be32_to_cpu(p[0]) & 0xffff) << 8 | + (be32_to_cpu(p[1]) & 0xff000000) >> 24; +} + +static u32 gasp_version(__be32 *p) +{ + return be32_to_cpu(p[1]) & 0xffffff; +} + static void fwnet_receive_broadcast(struct fw_iso_context *context, u32 cycle, size_t header_length, void *header, void *data) { @@ -731,9 +758,6 @@ static void fwnet_receive_broadcast(struct fw_iso_context *context, __be32 *buf_ptr; int retval; u32 length; - u16 source_node_id; - u32 specifier_id; - u32 ver; unsigned long offset; unsigned long flags; @@ -750,22 +774,17 @@ static void fwnet_receive_broadcast(struct fw_iso_context *context, spin_unlock_irqrestore(&dev->lock, flags); - specifier_id = (be32_to_cpu(buf_ptr[0]) & 0xffff) << 8 - | (be32_to_cpu(buf_ptr[1]) & 0xff000000) >> 24; - ver = be32_to_cpu(buf_ptr[1]) & 0xffffff; - source_node_id = be32_to_cpu(buf_ptr[0]) >> 16; - - if (specifier_id == IANA_SPECIFIER_ID && - (ver == RFC2734_SW_VERSION + if (length > IEEE1394_GASP_HDR_SIZE && + gasp_specifier_id(buf_ptr) == IANA_SPECIFIER_ID && + (gasp_version(buf_ptr) == RFC2734_SW_VERSION #if IS_ENABLED(CONFIG_IPV6) - || ver == RFC3146_SW_VERSION + || gasp_version(buf_ptr) == RFC3146_SW_VERSION #endif - )) { - buf_ptr += 2; - length -= IEEE1394_GASP_HDR_SIZE; - fwnet_incoming_packet(dev, buf_ptr, length, source_node_id, + )) + fwnet_incoming_packet(dev, buf_ptr + 2, + length - IEEE1394_GASP_HDR_SIZE, + gasp_source_id(buf_ptr), context->card->generation, true); - } packet.payload_length = dev->rcv_buffer_size; packet.interrupt = 1; diff --git a/kernel/drivers/firmware/efi/efi.c b/kernel/drivers/firmware/efi/efi.c index 027ca2121..3b52677f4 100644 --- a/kernel/drivers/firmware/efi/efi.c +++ b/kernel/drivers/firmware/efi/efi.c @@ -180,6 +180,7 @@ static int generic_ops_register(void) { generic_ops.get_variable = efi.get_variable; generic_ops.set_variable = efi.set_variable; + generic_ops.set_variable_nonblocking = efi.set_variable_nonblocking; generic_ops.get_next_variable = efi.get_next_variable; generic_ops.query_variable_store = efi_query_variable_store; diff --git a/kernel/drivers/firmware/efi/vars.c b/kernel/drivers/firmware/efi/vars.c index 7f2ea21c7..6f182fd91 100644 --- a/kernel/drivers/firmware/efi/vars.c +++ b/kernel/drivers/firmware/efi/vars.c @@ -202,29 +202,44 @@ static const struct variable_validate variable_validate[] = { { NULL_GUID, "", NULL }, }; +/* + * Check if @var_name matches the pattern given in @match_name. + * + * @var_name: an array of @len non-NUL characters. + * @match_name: a NUL-terminated pattern string, optionally ending in "*". A + * final "*" character matches any trailing characters @var_name, + * including the case when there are none left in @var_name. + * @match: on output, the number of non-wildcard characters in @match_name + * that @var_name matches, regardless of the return value. + * @return: whether @var_name fully matches @match_name. + */ static bool variable_matches(const char *var_name, size_t len, const char *match_name, int *match) { for (*match = 0; ; (*match)++) { char c = match_name[*match]; - char u = var_name[*match]; - /* Wildcard in the matching name means we've matched */ - if (c == '*') + switch (c) { + case '*': + /* Wildcard in @match_name means we've matched. */ return true; - /* Case sensitive match */ - if (!c && *match == len) - return true; + case '\0': + /* @match_name has ended. Has @var_name too? */ + return (*match == len); - if (c != u) + default: + /* + * We've reached a non-wildcard char in @match_name. + * Continue only if there's an identical character in + * @var_name. + */ + if (*match < len && c == var_name[*match]) + continue; return false; - - if (!c) - return true; + } } - return true; } bool diff --git a/kernel/drivers/gpio/Kconfig b/kernel/drivers/gpio/Kconfig index b18bea08f..469dc378a 100644 --- a/kernel/drivers/gpio/Kconfig +++ b/kernel/drivers/gpio/Kconfig @@ -50,6 +50,7 @@ config GPIO_DEVRES config OF_GPIO def_bool y depends on OF + depends on HAS_IOMEM config GPIO_ACPI def_bool y diff --git a/kernel/drivers/gpio/gpio-bcm-kona.c b/kernel/drivers/gpio/gpio-bcm-kona.c index 33a1f9779..4ea71d505 100644 --- a/kernel/drivers/gpio/gpio-bcm-kona.c +++ b/kernel/drivers/gpio/gpio-bcm-kona.c @@ -551,11 +551,11 @@ static void bcm_kona_gpio_reset(struct bcm_kona_gpio *kona_gpio) /* disable interrupts and clear status */ for (i = 0; i < kona_gpio->num_bank; i++) { /* Unlock the entire bank first */ - bcm_kona_gpio_write_lock_regs(kona_gpio, i, UNLOCK_CODE); + bcm_kona_gpio_write_lock_regs(reg_base, i, UNLOCK_CODE); writel(0xffffffff, reg_base + GPIO_INT_MASK(i)); writel(0xffffffff, reg_base + GPIO_INT_STATUS(i)); /* Now re-lock the bank */ - bcm_kona_gpio_write_lock_regs(kona_gpio, i, LOCK_CODE); + bcm_kona_gpio_write_lock_regs(reg_base, i, LOCK_CODE); } } diff --git a/kernel/drivers/gpio/gpio-intel-mid.c b/kernel/drivers/gpio/gpio-intel-mid.c index 70097472b..c50e930d9 100644 --- a/kernel/drivers/gpio/gpio-intel-mid.c +++ b/kernel/drivers/gpio/gpio-intel-mid.c @@ -17,7 +17,6 @@ * Moorestown platform Langwell chip. * Medfield platform Penwell chip. * Clovertrail platform Cloverview chip. - * Merrifield platform Tangier chip. */ #include @@ -64,10 +63,6 @@ enum GPIO_REG { /* intel_mid gpio driver data */ struct intel_mid_gpio_ddata { u16 ngpio; /* number of gpio pins */ - u32 gplr_offset; /* offset of first GPLR register from base */ - u32 flis_base; /* base address of FLIS registers */ - u32 flis_len; /* length of FLIS registers */ - u32 (*get_flis_offset)(int gpio); u32 chip_irq_type; /* chip interrupt type */ }; @@ -257,15 +252,6 @@ static const struct intel_mid_gpio_ddata gpio_cloverview_core = { .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, }; -static const struct intel_mid_gpio_ddata gpio_tangier = { - .ngpio = 192, - .gplr_offset = 4, - .flis_base = 0xff0c0000, - .flis_len = 0x8000, - .get_flis_offset = NULL, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - static const struct pci_device_id intel_gpio_ids[] = { { /* Lincroft */ @@ -292,11 +278,6 @@ static const struct pci_device_id intel_gpio_ids[] = { PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08f7), .driver_data = (kernel_ulong_t)&gpio_cloverview_core, }, - { - /* Tangier */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1199), - .driver_data = (kernel_ulong_t)&gpio_tangier, - }, { 0 } }; MODULE_DEVICE_TABLE(pci, intel_gpio_ids); diff --git a/kernel/drivers/gpio/gpio-mpc8xxx.c b/kernel/drivers/gpio/gpio-mpc8xxx.c index 48ef36834..9e02cb6af 100644 --- a/kernel/drivers/gpio/gpio-mpc8xxx.c +++ b/kernel/drivers/gpio/gpio-mpc8xxx.c @@ -329,7 +329,7 @@ static int mpc8xxx_gpio_irq_map(struct irq_domain *h, unsigned int irq, irq_hw_number_t hwirq) { irq_set_chip_data(irq, h->host_data); - irq_set_chip_and_handler(irq, &mpc8xxx_irq_chip, handle_level_irq); + irq_set_chip_and_handler(irq, &mpc8xxx_irq_chip, handle_edge_irq); return 0; } diff --git a/kernel/drivers/gpio/gpio-pca953x.c b/kernel/drivers/gpio/gpio-pca953x.c index 2d4892cc7..c844d7ecc 100644 --- a/kernel/drivers/gpio/gpio-pca953x.c +++ b/kernel/drivers/gpio/gpio-pca953x.c @@ -86,7 +86,7 @@ MODULE_DEVICE_TABLE(acpi, pca953x_acpi_ids); #define MAX_BANK 5 #define BANK_SZ 8 -#define NBANK(chip) (chip->gpio_chip.ngpio / BANK_SZ) +#define NBANK(chip) DIV_ROUND_UP(chip->gpio_chip.ngpio, BANK_SZ) struct pca953x_chip { unsigned gpio_start; diff --git a/kernel/drivers/gpio/gpio-sa1100.c b/kernel/drivers/gpio/gpio-sa1100.c index 990fa9023..3b6bce051 100644 --- a/kernel/drivers/gpio/gpio-sa1100.c +++ b/kernel/drivers/gpio/gpio-sa1100.c @@ -155,7 +155,7 @@ static int sa1100_gpio_irqdomain_map(struct irq_domain *d, { irq_set_chip_and_handler(irq, &sa1100_gpio_irq_chip, handle_edge_irq); - irq_set_noprobe(irq); + irq_set_probe(irq); return 0; } diff --git a/kernel/drivers/gpio/gpiolib-legacy.c b/kernel/drivers/gpio/gpiolib-legacy.c index 3a5c7011a..8b830996f 100644 --- a/kernel/drivers/gpio/gpiolib-legacy.c +++ b/kernel/drivers/gpio/gpiolib-legacy.c @@ -28,6 +28,10 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) if (!desc && gpio_is_valid(gpio)) return -EPROBE_DEFER; + err = gpiod_request(desc, label); + if (err) + return err; + if (flags & GPIOF_OPEN_DRAIN) set_bit(FLAG_OPEN_DRAIN, &desc->flags); @@ -37,10 +41,6 @@ int gpio_request_one(unsigned gpio, unsigned long flags, const char *label) if (flags & GPIOF_ACTIVE_LOW) set_bit(FLAG_ACTIVE_LOW, &desc->flags); - err = gpiod_request(desc, label); - if (err) - return err; - if (flags & GPIOF_DIR_IN) err = gpiod_direction_input(desc); else diff --git a/kernel/drivers/gpio/gpiolib.c b/kernel/drivers/gpio/gpiolib.c index 4e4c3083a..06d345b08 100644 --- a/kernel/drivers/gpio/gpiolib.c +++ b/kernel/drivers/gpio/gpiolib.c @@ -927,14 +927,6 @@ static int __gpiod_request(struct gpio_desc *desc, const char *label) spin_lock_irqsave(&gpio_lock, flags); } done: - if (status < 0) { - /* Clear flags that might have been set by the caller before - * requesting the GPIO. - */ - clear_bit(FLAG_ACTIVE_LOW, &desc->flags); - clear_bit(FLAG_OPEN_DRAIN, &desc->flags); - clear_bit(FLAG_OPEN_SOURCE, &desc->flags); - } spin_unlock_irqrestore(&gpio_lock, flags); return status; } @@ -2062,28 +2054,13 @@ struct gpio_desc *__must_check gpiod_get_optional(struct device *dev, } EXPORT_SYMBOL_GPL(gpiod_get_optional); -/** - * gpiod_parse_flags - helper function to parse GPIO lookup flags - * @desc: gpio to be setup - * @lflags: gpio_lookup_flags - returned from of_find_gpio() or - * of_get_gpio_hog() - * - * Set the GPIO descriptor flags based on the given GPIO lookup flags. - */ -static void gpiod_parse_flags(struct gpio_desc *desc, unsigned long lflags) -{ - if (lflags & GPIO_ACTIVE_LOW) - set_bit(FLAG_ACTIVE_LOW, &desc->flags); - if (lflags & GPIO_OPEN_DRAIN) - set_bit(FLAG_OPEN_DRAIN, &desc->flags); - if (lflags & GPIO_OPEN_SOURCE) - set_bit(FLAG_OPEN_SOURCE, &desc->flags); -} /** * gpiod_configure_flags - helper function to configure a given GPIO * @desc: gpio whose value will be assigned * @con_id: function within the GPIO consumer + * @lflags: gpio_lookup_flags - returned from of_find_gpio() or + * of_get_gpio_hog() * @dflags: gpiod_flags - optional GPIO initialization flags * * Return 0 on success, -ENOENT if no GPIO has been assigned to the @@ -2091,10 +2068,17 @@ static void gpiod_parse_flags(struct gpio_desc *desc, unsigned long lflags) * occurred while trying to acquire the GPIO. */ static int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, - enum gpiod_flags dflags) + unsigned long lflags, enum gpiod_flags dflags) { int status; + if (lflags & GPIO_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIO_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIO_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + /* No particular flag request, return here... */ if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) { pr_debug("no flags found for %s\n", con_id); @@ -2161,13 +2145,11 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev, return desc; } - gpiod_parse_flags(desc, lookupflags); - status = gpiod_request(desc, con_id); if (status < 0) return ERR_PTR(status); - status = gpiod_configure_flags(desc, con_id, flags); + status = gpiod_configure_flags(desc, con_id, lookupflags, flags); if (status < 0) { dev_dbg(dev, "setup of GPIO %s failed\n", con_id); gpiod_put(desc); @@ -2223,6 +2205,10 @@ struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, if (IS_ERR(desc)) return desc; + ret = gpiod_request(desc, NULL); + if (ret) + return ERR_PTR(ret); + if (active_low) set_bit(FLAG_ACTIVE_LOW, &desc->flags); @@ -2233,10 +2219,6 @@ struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, set_bit(FLAG_OPEN_SOURCE, &desc->flags); } - ret = gpiod_request(desc, NULL); - if (ret) - return ERR_PTR(ret); - return desc; } EXPORT_SYMBOL_GPL(fwnode_get_named_gpiod); @@ -2289,8 +2271,6 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, chip = gpiod_to_chip(desc); hwnum = gpio_chip_hwgpio(desc); - gpiod_parse_flags(desc, lflags); - local_desc = gpiochip_request_own_desc(chip, hwnum, name); if (IS_ERR(local_desc)) { pr_err("requesting hog GPIO %s (chip %s, offset %d) failed\n", @@ -2298,7 +2278,7 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, return PTR_ERR(local_desc); } - status = gpiod_configure_flags(desc, name, dflags); + status = gpiod_configure_flags(desc, name, lflags, dflags); if (status < 0) { pr_err("setup of hog GPIO %s (chip %s, offset %d) failed\n", name, chip->label, hwnum); diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.h index bb1099c54..e8e962f7b 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -532,6 +532,7 @@ struct amdgpu_bo { u64 metadata_flags; void *metadata; u32 metadata_size; + unsigned prime_shared_count; /* list of all virtual address to which this bo * is associated to */ @@ -710,9 +711,9 @@ int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev); void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev); int amdgpu_gart_init(struct amdgpu_device *adev); void amdgpu_gart_fini(struct amdgpu_device *adev); -void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset, +void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, int pages); -int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset, +int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, int pages, struct page **pagelist, dma_addr_t *dma_addr, uint32_t flags); @@ -1673,6 +1674,7 @@ struct amdgpu_uvd { struct amdgpu_bo *vcpu_bo; void *cpu_addr; uint64_t gpu_addr; + unsigned fw_version; atomic_t handles[AMDGPU_MAX_UVD_HANDLES]; struct drm_file *filp[AMDGPU_MAX_UVD_HANDLES]; struct delayed_work idle_work; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 9416e0f5c..51a9942cd 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -331,6 +331,19 @@ bool amdgpu_atombios_get_connector_info_from_object_table(struct amdgpu_device * (le16_to_cpu(path->usConnObjectId) & OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT; + /* Skip TV/CV support */ + if ((le16_to_cpu(path->usDeviceTag) == + ATOM_DEVICE_TV1_SUPPORT) || + (le16_to_cpu(path->usDeviceTag) == + ATOM_DEVICE_CV_SUPPORT)) + continue; + + if (con_obj_id >= ARRAY_SIZE(object_connector_convert)) { + DRM_ERROR("invalid con_obj_id %d for device tag 0x%04x\n", + con_obj_id, le16_to_cpu(path->usDeviceTag)); + continue; + } + connector_type = object_connector_convert[con_obj_id]; connector_object_id = con_obj_id; @@ -566,28 +579,19 @@ int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev) le16_to_cpu(firmware_info->info.usReferenceClock); ppll->reference_div = 0; - if (crev < 2) - ppll->pll_out_min = - le16_to_cpu(firmware_info->info.usMinPixelClockPLL_Output); - else - ppll->pll_out_min = - le32_to_cpu(firmware_info->info_12.ulMinPixelClockPLL_Output); + ppll->pll_out_min = + le32_to_cpu(firmware_info->info_12.ulMinPixelClockPLL_Output); ppll->pll_out_max = le32_to_cpu(firmware_info->info.ulMaxPixelClockPLL_Output); - if (crev >= 4) { - ppll->lcd_pll_out_min = - le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100; - if (ppll->lcd_pll_out_min == 0) - ppll->lcd_pll_out_min = ppll->pll_out_min; - ppll->lcd_pll_out_max = - le16_to_cpu(firmware_info->info_14.usLcdMaxPixelClockPLL_Output) * 100; - if (ppll->lcd_pll_out_max == 0) - ppll->lcd_pll_out_max = ppll->pll_out_max; - } else { + ppll->lcd_pll_out_min = + le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100; + if (ppll->lcd_pll_out_min == 0) ppll->lcd_pll_out_min = ppll->pll_out_min; + ppll->lcd_pll_out_max = + le16_to_cpu(firmware_info->info_14.usLcdMaxPixelClockPLL_Output) * 100; + if (ppll->lcd_pll_out_max == 0) ppll->lcd_pll_out_max = ppll->pll_out_max; - } if (ppll->pll_out_min == 0) ppll->pll_out_min = 64800; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c index 5a8fbadbd..29adbbe22 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "amdgpu_acpi.h" @@ -256,6 +257,10 @@ static int amdgpu_atpx_set_discrete_state(struct amdgpu_atpx *atpx, u8 state) if (!info) return -EIO; kfree(info); + + /* 200ms delay is required after off */ + if (state == 0) + msleep(200); } return 0; } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index f82a2dd83..3c7a72359 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -117,7 +117,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, entry->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; } entry->tv.bo = &entry->robj->tbo; - entry->tv.shared = true; + entry->tv.shared = !entry->robj->prime_shared_count; if (entry->prefered_domains == AMDGPU_GEM_DOMAIN_GDS) gds_obj = entry->robj; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index 119cdc2c4..930083336 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -194,12 +194,12 @@ int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector) bpc = 8; DRM_DEBUG("%s: HDMI deep color 10 bpc exceeds max tmds clock. Using %d bpc.\n", connector->name, bpc); - } else if (bpc > 8) { - /* max_tmds_clock missing, but hdmi spec mandates it for deep color. */ - DRM_DEBUG("%s: Required max tmds clock for HDMI deep color missing. Using 8 bpc.\n", - connector->name); - bpc = 8; } + } else if (bpc > 8) { + /* max_tmds_clock missing, but hdmi spec mandates it for deep color. */ + DRM_DEBUG("%s: Required max tmds clock for HDMI deep color missing. Using 8 bpc.\n", + connector->name); + bpc = 8; } } @@ -1690,7 +1690,6 @@ amdgpu_connector_add(struct amdgpu_device *adev, DRM_MODE_SCALE_NONE); /* no HPD on analog connectors */ amdgpu_connector->hpd.hpd = AMDGPU_HPD_NONE; - connector->polled = DRM_CONNECTOR_POLL_CONNECT; connector->interlace_allowed = true; connector->doublescan_allowed = true; break; @@ -1893,8 +1892,10 @@ amdgpu_connector_add(struct amdgpu_device *adev, } if (amdgpu_connector->hpd.hpd == AMDGPU_HPD_NONE) { - if (i2c_bus->valid) - connector->polled = DRM_CONNECTOR_POLL_CONNECT; + if (i2c_bus->valid) { + connector->polled = DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT; + } } else connector->polled = DRM_CONNECTOR_POLL_HPD; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index c961fe093..16302f7d5 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1793,7 +1793,23 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon) } drm_kms_helper_poll_enable(dev); + + /* + * Most of the connector probing functions try to acquire runtime pm + * refs to ensure that the GPU is powered on when connector polling is + * performed. Since we're calling this from a runtime PM callback, + * trying to acquire rpm refs will cause us to deadlock. + * + * Since we're guaranteed to be holding the rpm lock, it's safe to + * temporarily disable the rpm helpers so this doesn't deadlock us. + */ +#ifdef CONFIG_PM + dev->dev->power.disable_depth++; +#endif drm_helper_hpd_irq_event(dev); +#ifdef CONFIG_PM + dev->dev->power.disable_depth--; +#endif if (fbcon) { amdgpu_fbdev_set_suspend(adev, 0); diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c index 7b7f4aba6..14f57d991 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c @@ -113,24 +113,26 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev, printk("\n"); } + u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev) { struct drm_device *dev = adev->ddev; struct drm_crtc *crtc; struct amdgpu_crtc *amdgpu_crtc; - u32 line_time_us, vblank_lines; + u32 vblank_in_pixels; u32 vblank_time_us = 0xffffffff; /* if the displays are off, vblank time is max */ if (adev->mode_info.num_crtc && adev->mode_info.mode_config_initialized) { list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { amdgpu_crtc = to_amdgpu_crtc(crtc); if (crtc->enabled && amdgpu_crtc->enabled && amdgpu_crtc->hw_mode.clock) { - line_time_us = (amdgpu_crtc->hw_mode.crtc_htotal * 1000) / - amdgpu_crtc->hw_mode.clock; - vblank_lines = amdgpu_crtc->hw_mode.crtc_vblank_end - + vblank_in_pixels = + amdgpu_crtc->hw_mode.crtc_htotal * + (amdgpu_crtc->hw_mode.crtc_vblank_end - amdgpu_crtc->hw_mode.crtc_vdisplay + - (amdgpu_crtc->v_border * 2); - vblank_time_us = vblank_lines * line_time_us; + (amdgpu_crtc->v_border * 2)); + + vblank_time_us = vblank_in_pixels * 1000 / amdgpu_crtc->hw_mode.clock; break; } } @@ -150,7 +152,7 @@ u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev) list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { amdgpu_crtc = to_amdgpu_crtc(crtc); if (crtc->enabled && amdgpu_crtc->enabled && amdgpu_crtc->hw_mode.clock) { - vrefresh = amdgpu_crtc->hw_mode.vrefresh; + vrefresh = drm_mode_vrefresh(&amdgpu_crtc->hw_mode); break; } } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index 7312d729d..22a613a95 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -221,7 +221,7 @@ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev) * Unbinds the requested pages from the gart page table and * replaces them with the dummy page (all asics). */ -void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset, +void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, int pages) { unsigned t; @@ -269,7 +269,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset, * (all asics). * Returns 0 for success, -EINVAL for failure. */ -int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset, +int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, int pages, struct page **pagelist, dma_addr_t *dma_addr, uint32_t flags) { diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 9e25edafa..c77a1ebfc 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -288,7 +288,7 @@ void amdgpu_ib_pool_fini(struct amdgpu_device *adev) int amdgpu_ib_ring_tests(struct amdgpu_device *adev) { unsigned i; - int r; + int r, ret = 0; for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; @@ -309,10 +309,11 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev) } else { /* still not good, but we can live with it */ DRM_ERROR("amdgpu: failed testing IB on ring %d (%d).\n", i, r); + ret = r; } } } - return 0; + return ret; } /* diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index e23843f4d..a5c824078 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -227,7 +227,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file type = AMD_IP_BLOCK_TYPE_UVD; ring_mask = adev->uvd.ring.ready ? 1 : 0; ib_start_alignment = AMDGPU_GPU_PAGE_SIZE; - ib_size_alignment = 8; + ib_size_alignment = 16; break; case AMDGPU_HW_IP_VCE: type = AMD_IP_BLOCK_TYPE_VCE; @@ -303,7 +303,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file fw_info.feature = adev->vce.fb_version; break; case AMDGPU_INFO_FW_UVD: - fw_info.ver = 0; + fw_info.ver = adev->uvd.fw_version; fw_info.feature = 0; break; case AMDGPU_INFO_FW_GMC: diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h index 064ebb347..89df78716 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h @@ -52,7 +52,7 @@ struct amdgpu_hpd; #define AMDGPU_MAX_HPD_PINS 6 #define AMDGPU_MAX_CRTCS 6 -#define AMDGPU_MAX_AFMT_BLOCKS 7 +#define AMDGPU_MAX_AFMT_BLOCKS 9 enum amdgpu_rmx_type { RMX_OFF, @@ -308,8 +308,8 @@ struct amdgpu_mode_info { struct atom_context *atom_context; struct card_info *atom_card_info; bool mode_config_initialized; - struct amdgpu_crtc *crtcs[6]; - struct amdgpu_afmt *afmt[7]; + struct amdgpu_crtc *crtcs[AMDGPU_MAX_CRTCS]; + struct amdgpu_afmt *afmt[AMDGPU_MAX_AFMT_BLOCKS]; /* DVI-I properties */ struct drm_property *coherent_mode_property; /* DAC enable load detect */ diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index b8fbbd769..73628c759 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -540,6 +540,7 @@ int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, if (!metadata_size) { if (bo->metadata_size) { kfree(bo->metadata); + bo->metadata = NULL; bo->metadata_size = 0; } return 0; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c index 59f735a93..e6a7d30c3 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c @@ -77,20 +77,36 @@ struct drm_gem_object *amdgpu_gem_prime_import_sg_table(struct drm_device *dev, list_add_tail(&bo->list, &adev->gem.objects); mutex_unlock(&adev->gem.mutex); + bo->prime_shared_count = 1; return &bo->gem_base; } int amdgpu_gem_prime_pin(struct drm_gem_object *obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); - int ret = 0; + long ret = 0; ret = amdgpu_bo_reserve(bo, false); if (unlikely(ret != 0)) return ret; + /* + * Wait for all shared fences to complete before we switch to future + * use of exclusive fence on this prime shared bo. + */ + ret = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false, + MAX_SCHEDULE_TIMEOUT); + if (unlikely(ret < 0)) { + DRM_DEBUG_PRIME("Fence wait failed: %li\n", ret); + amdgpu_bo_unreserve(bo); + return ret; + } + /* pin buffer into GTT */ ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT, NULL); + if (likely(ret == 0)) + bo->prime_shared_count++; + amdgpu_bo_unreserve(bo); return ret; } @@ -105,6 +121,8 @@ void amdgpu_gem_prime_unpin(struct drm_gem_object *obj) return; amdgpu_bo_unpin(bo); + if (bo->prime_shared_count) + bo->prime_shared_count--; amdgpu_bo_unreserve(bo); } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 1cbb16e15..475c38fe9 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -233,8 +233,8 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, adev = amdgpu_get_adev(bo->bdev); ring = adev->mman.buffer_funcs_ring; - old_start = old_mem->start << PAGE_SHIFT; - new_start = new_mem->start << PAGE_SHIFT; + old_start = (u64)old_mem->start << PAGE_SHIFT; + new_start = (u64)new_mem->start << PAGE_SHIFT; switch (old_mem->mem_type) { case TTM_PL_VRAM: diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 53f987aee..3b35ad838 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -156,6 +156,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) DRM_INFO("Found UVD firmware Version: %hu.%hu Family ID: %hu\n", version_major, version_minor, family_id); + adev->uvd.fw_version = ((version_major << 24) | (version_minor << 16) | + (family_id << 8)); + bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8) + AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE; r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true, @@ -273,6 +276,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) memcpy(adev->uvd.cpu_addr, (adev->uvd.fw->data) + offset, (adev->uvd.fw->size) - offset); + cancel_delayed_work_sync(&adev->uvd.idle_work); + size = amdgpu_bo_size(adev->uvd.vcpu_bo); size -= le32_to_cpu(hdr->ucode_size_bytes); ptr = adev->uvd.cpu_addr; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index a745eeeb5..bb0da7605 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -220,6 +220,7 @@ int amdgpu_vce_suspend(struct amdgpu_device *adev) if (i == AMDGPU_MAX_VCE_HANDLES) return 0; + cancel_delayed_work_sync(&adev->vce.idle_work); /* TODO: suspending running encoding sessions isn't supported */ return -EINVAL; } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/kernel/drivers/gpu/drm/amd/amdgpu/atombios_dp.c index 92b6acadf..7f85c2c1d 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/atombios_dp.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/atombios_dp.c @@ -243,7 +243,7 @@ static void amdgpu_atombios_dp_get_adjust_train(const u8 link_status[DP_LINK_STA /* convert bits per color to bits per pixel */ /* get bpc from the EDID */ -static int amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc) +static unsigned amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc) { if (bpc == 0) return 24; @@ -251,64 +251,44 @@ static int amdgpu_atombios_dp_convert_bpc_to_bpp(int bpc) return bpc * 3; } -/* get the max pix clock supported by the link rate and lane num */ -static int amdgpu_atombios_dp_get_max_dp_pix_clock(int link_rate, - int lane_num, - int bpp) -{ - return (link_rate * lane_num * 8) / bpp; -} - /***** amdgpu specific DP functions *****/ -/* First get the min lane# when low rate is used according to pixel clock - * (prefer low rate), second check max lane# supported by DP panel, - * if the max lane# < low rate lane# then use max lane# instead. - */ -static int amdgpu_atombios_dp_get_dp_lane_number(struct drm_connector *connector, +static int amdgpu_atombios_dp_get_dp_link_config(struct drm_connector *connector, const u8 dpcd[DP_DPCD_SIZE], - int pix_clock) + unsigned pix_clock, + unsigned *dp_lanes, unsigned *dp_rate) { - int bpp = amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector)); - int max_link_rate = drm_dp_max_link_rate(dpcd); - int max_lane_num = drm_dp_max_lane_count(dpcd); - int lane_num; - int max_dp_pix_clock; - - for (lane_num = 1; lane_num < max_lane_num; lane_num <<= 1) { - max_dp_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(max_link_rate, lane_num, bpp); - if (pix_clock <= max_dp_pix_clock) - break; - } - - return lane_num; -} - -static int amdgpu_atombios_dp_get_dp_link_clock(struct drm_connector *connector, - const u8 dpcd[DP_DPCD_SIZE], - int pix_clock) -{ - int bpp = amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector)); - int lane_num, max_pix_clock; + unsigned bpp = + amdgpu_atombios_dp_convert_bpc_to_bpp(amdgpu_connector_get_monitor_bpc(connector)); + static const unsigned link_rates[3] = { 162000, 270000, 540000 }; + unsigned max_link_rate = drm_dp_max_link_rate(dpcd); + unsigned max_lane_num = drm_dp_max_lane_count(dpcd); + unsigned lane_num, i, max_pix_clock; if (amdgpu_connector_encoder_get_dp_bridge_encoder_id(connector) == - ENCODER_OBJECT_ID_NUTMEG) - return 270000; - - lane_num = amdgpu_atombios_dp_get_dp_lane_number(connector, dpcd, pix_clock); - max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(162000, lane_num, bpp); - if (pix_clock <= max_pix_clock) - return 162000; - max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(270000, lane_num, bpp); - if (pix_clock <= max_pix_clock) - return 270000; - if (amdgpu_connector_is_dp12_capable(connector)) { - max_pix_clock = amdgpu_atombios_dp_get_max_dp_pix_clock(540000, lane_num, bpp); - if (pix_clock <= max_pix_clock) - return 540000; + ENCODER_OBJECT_ID_NUTMEG) { + for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) { + max_pix_clock = (lane_num * 270000 * 8) / bpp; + if (max_pix_clock >= pix_clock) { + *dp_lanes = lane_num; + *dp_rate = 270000; + return 0; + } + } + } else { + for (i = 0; i < ARRAY_SIZE(link_rates) && link_rates[i] <= max_link_rate; i++) { + for (lane_num = 1; lane_num <= max_lane_num; lane_num <<= 1) { + max_pix_clock = (lane_num * link_rates[i] * 8) / bpp; + if (max_pix_clock >= pix_clock) { + *dp_lanes = lane_num; + *dp_rate = link_rates[i]; + return 0; + } + } + } } - return drm_dp_max_link_rate(dpcd); + return -EINVAL; } static u8 amdgpu_atombios_dp_encoder_service(struct amdgpu_device *adev, @@ -422,6 +402,7 @@ void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector, { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); struct amdgpu_connector_atom_dig *dig_connector; + int ret; if (!amdgpu_connector->con_priv) return; @@ -429,10 +410,14 @@ void amdgpu_atombios_dp_set_link_config(struct drm_connector *connector, if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) { - dig_connector->dp_clock = - amdgpu_atombios_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock); - dig_connector->dp_lane_count = - amdgpu_atombios_dp_get_dp_lane_number(connector, dig_connector->dpcd, mode->clock); + ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd, + mode->clock, + &dig_connector->dp_lane_count, + &dig_connector->dp_clock); + if (ret) { + dig_connector->dp_clock = 0; + dig_connector->dp_lane_count = 0; + } } } @@ -441,14 +426,17 @@ int amdgpu_atombios_dp_mode_valid_helper(struct drm_connector *connector, { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); struct amdgpu_connector_atom_dig *dig_connector; - int dp_clock; + unsigned dp_lanes, dp_clock; + int ret; if (!amdgpu_connector->con_priv) return MODE_CLOCK_HIGH; dig_connector = amdgpu_connector->con_priv; - dp_clock = - amdgpu_atombios_dp_get_dp_link_clock(connector, dig_connector->dpcd, mode->clock); + ret = amdgpu_atombios_dp_get_dp_link_config(connector, dig_connector->dpcd, + mode->clock, &dp_lanes, &dp_clock); + if (ret) + return MODE_CLOCK_HIGH; if ((dp_clock == 540000) && (!amdgpu_connector_is_dp12_capable(connector))) diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/kernel/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c index 1e0bba29e..542517d4e 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c @@ -98,6 +98,7 @@ amdgpu_atombios_encoder_set_backlight_level(struct amdgpu_encoder *amdgpu_encode case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: + case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3: if (dig->backlight_level == 0) amdgpu_atombios_encoder_setup_dig_transmitter(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); @@ -298,6 +299,10 @@ bool amdgpu_atombios_encoder_mode_fixup(struct drm_encoder *encoder, && (mode->crtc_vsync_start < (mode->crtc_vdisplay + 2))) adjusted_mode->crtc_vsync_start = adjusted_mode->crtc_vdisplay + 2; + /* vertical FP must be at least 1 */ + if (mode->crtc_vsync_start == mode->crtc_vdisplay) + adjusted_mode->crtc_vsync_start++; + /* get the native mode for scaling */ if (amdgpu_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT)) amdgpu_panel_mode_fixup(encoder, adjusted_mode); diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 5f712cedd..c568293cb 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -52,6 +52,7 @@ static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev); static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev); static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev); static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev); +static int cik_sdma_soft_reset(void *handle); MODULE_FIRMWARE("radeon/bonaire_sdma.bin"); MODULE_FIRMWARE("radeon/bonaire_sdma1.bin"); @@ -1030,6 +1031,8 @@ static int cik_sdma_resume(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + cik_sdma_soft_reset(handle); + return cik_sdma_hw_init(adev); } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/kernel/drivers/gpu/drm/amd/amdgpu/cz_dpm.c index 8035d4d6a..653917a3b 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/cz_dpm.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/cz_dpm.c @@ -1955,10 +1955,8 @@ static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate) } } else { /*pi->caps_vce_pg*/ cz_update_vce_dpm(adev); - cz_enable_vce_dpm(adev, true); + cz_enable_vce_dpm(adev, !gate); } - - return; } const struct amd_ip_funcs cz_dpm_ip_funcs = { diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 4dcc8fba5..5b261adb4 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -419,16 +419,6 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev) list_for_each_entry(connector, &dev->mode_config.connector_list, head) { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || - connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { - /* don't try to enable hpd on eDP or LVDS avoid breaking the - * aux dp channel on imac and help (but not completely fix) - * https://bugzilla.redhat.com/show_bug.cgi?id=726143 - * also avoid interrupt storms during dpms. - */ - continue; - } - switch (amdgpu_connector->hpd.hpd) { case AMDGPU_HPD_1: idx = 0; @@ -452,6 +442,19 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev) continue; } + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { + /* don't try to enable hpd on eDP or LVDS avoid breaking the + * aux dp channel on imac and help (but not completely fix) + * https://bugzilla.redhat.com/show_bug.cgi?id=726143 + * also avoid interrupt storms during dpms. + */ + tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]); + tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0); + WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp); + continue; + } + tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]); tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1); WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp); diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index 8f1e51128..c161eeda4 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -409,16 +409,6 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev) list_for_each_entry(connector, &dev->mode_config.connector_list, head) { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || - connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { - /* don't try to enable hpd on eDP or LVDS avoid breaking the - * aux dp channel on imac and help (but not completely fix) - * https://bugzilla.redhat.com/show_bug.cgi?id=726143 - * also avoid interrupt storms during dpms. - */ - continue; - } - switch (amdgpu_connector->hpd.hpd) { case AMDGPU_HPD_1: idx = 0; @@ -442,6 +432,19 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev) continue; } + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { + /* don't try to enable hpd on eDP or LVDS avoid breaking the + * aux dp channel on imac and help (but not completely fix) + * https://bugzilla.redhat.com/show_bug.cgi?id=726143 + * also avoid interrupt storms during dpms. + */ + tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]); + tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0); + WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp); + continue; + } + tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]); tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1); WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp); @@ -3030,6 +3033,7 @@ static int dce_v11_0_sw_fini(void *handle) dce_v11_0_afmt_fini(adev); + drm_mode_config_cleanup(adev->ddev); adev->mode_info.mode_config_initialized = false; return 0; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 42d954dc4..9b4dcf76c 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -392,15 +392,6 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev) list_for_each_entry(connector, &dev->mode_config.connector_list, head) { struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector); - if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || - connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { - /* don't try to enable hpd on eDP or LVDS avoid breaking the - * aux dp channel on imac and help (but not completely fix) - * https://bugzilla.redhat.com/show_bug.cgi?id=726143 - * also avoid interrupt storms during dpms. - */ - continue; - } switch (amdgpu_connector->hpd.hpd) { case AMDGPU_HPD_1: WREG32(mmDC_HPD1_CONTROL, tmp); @@ -423,6 +414,45 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev) default: break; } + + if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { + /* don't try to enable hpd on eDP or LVDS avoid breaking the + * aux dp channel on imac and help (but not completely fix) + * https://bugzilla.redhat.com/show_bug.cgi?id=726143 + * also avoid interrupt storms during dpms. + */ + u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl; + + switch (amdgpu_connector->hpd.hpd) { + case AMDGPU_HPD_1: + dc_hpd_int_cntl_reg = mmDC_HPD1_INT_CONTROL; + break; + case AMDGPU_HPD_2: + dc_hpd_int_cntl_reg = mmDC_HPD2_INT_CONTROL; + break; + case AMDGPU_HPD_3: + dc_hpd_int_cntl_reg = mmDC_HPD3_INT_CONTROL; + break; + case AMDGPU_HPD_4: + dc_hpd_int_cntl_reg = mmDC_HPD4_INT_CONTROL; + break; + case AMDGPU_HPD_5: + dc_hpd_int_cntl_reg = mmDC_HPD5_INT_CONTROL; + break; + case AMDGPU_HPD_6: + dc_hpd_int_cntl_reg = mmDC_HPD6_INT_CONTROL; + break; + default: + continue; + } + + dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg); + dc_hpd_int_cntl &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK; + WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl); + continue; + } + dce_v8_0_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd); amdgpu_irq_get(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd); } diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index aa491540b..b57fffc2d 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -3628,7 +3628,7 @@ static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vm_id, uint64_t pd_addr) { int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); - uint32_t seq = ring->fence_drv.sync_seq; + uint32_t seq = ring->fence_drv.sync_seq[ring->idx]; uint64_t addr = ring->fence_drv.gpu_addr; amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); @@ -5463,7 +5463,7 @@ static int gfx_v7_0_eop_irq(struct amdgpu_device *adev, case 2: for (i = 0; i < adev->gfx.num_compute_rings; i++) { ring = &adev->gfx.compute_ring[i]; - if ((ring->me == me_id) & (ring->pipe == pipe_id)) + if ((ring->me == me_id) && (ring->pipe == pipe_id)) amdgpu_fence_process(ring); } break; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 272110cc1..df17fabab 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -167,6 +167,7 @@ static int gmc_v7_0_init_microcode(struct amdgpu_device *adev) break; case CHIP_KAVERI: case CHIP_KABINI: + case CHIP_MULLINS: return 0; default: BUG(); } @@ -898,14 +899,6 @@ static int gmc_v7_0_early_init(void *handle) gmc_v7_0_set_gart_funcs(adev); gmc_v7_0_set_irq_funcs(adev); - if (adev->flags & AMD_IS_APU) { - adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; - } else { - u32 tmp = RREG32(mmMC_SEQ_MISC0); - tmp &= MC_SEQ_MISC0__MT__MASK; - adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp); - } - return 0; } @@ -926,6 +919,14 @@ static int gmc_v7_0_sw_init(void *handle) if (r) return r; + if (adev->flags & AMD_IS_APU) { + adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; + } else { + u32 tmp = RREG32(mmMC_SEQ_MISC0); + tmp &= MC_SEQ_MISC0__MT__MASK; + adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp); + } + r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault); if (r) return r; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index ba4ad00ba..08423089f 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -852,14 +852,6 @@ static int gmc_v8_0_early_init(void *handle) gmc_v8_0_set_gart_funcs(adev); gmc_v8_0_set_irq_funcs(adev); - if (adev->flags & AMD_IS_APU) { - adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; - } else { - u32 tmp = RREG32(mmMC_SEQ_MISC0); - tmp &= MC_SEQ_MISC0__MT__MASK; - adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp); - } - return 0; } @@ -870,6 +862,8 @@ static int gmc_v8_0_late_init(void *handle) return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); } +#define mmMC_SEQ_MISC0_FIJI 0xA71 + static int gmc_v8_0_sw_init(void *handle) { int r; @@ -880,6 +874,19 @@ static int gmc_v8_0_sw_init(void *handle) if (r) return r; + if (adev->flags & AMD_IS_APU) { + adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; + } else { + u32 tmp; + + if (adev->asic_type == CHIP_FIJI) + tmp = RREG32(mmMC_SEQ_MISC0_FIJI); + else + tmp = RREG32(mmMC_SEQ_MISC0); + tmp &= MC_SEQ_MISC0__MT__MASK; + adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp); + } + r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault); if (r) return r; diff --git a/kernel/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/kernel/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 2cf50180c..b1c7a9b36 100644 --- a/kernel/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/kernel/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -32,8 +32,8 @@ #include "oss/oss_2_4_d.h" #include "oss/oss_2_4_sh_mask.h" -#include "gmc/gmc_8_1_d.h" -#include "gmc/gmc_8_1_sh_mask.h" +#include "gmc/gmc_7_1_d.h" +#include "gmc/gmc_7_1_sh_mask.h" #include "gca/gfx_8_0_d.h" #include "gca/gfx_8_0_enum.h" diff --git a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c index c34c393e9..d5e19b5fb 100644 --- a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c +++ b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c @@ -513,7 +513,7 @@ static int dbgdev_wave_control_set_registers( union SQ_CMD_BITS *in_reg_sq_cmd, union GRBM_GFX_INDEX_BITS *in_reg_gfx_index) { - int status; + int status = 0; union SQ_CMD_BITS reg_sq_cmd; union GRBM_GFX_INDEX_BITS reg_gfx_index; struct HsaDbgWaveMsgAMDGen2 *pMsg; diff --git a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 9be007081..eb1da83c9 100644 --- a/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/kernel/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -242,13 +242,19 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn, pqm_uninit(&p->pqm); /* Iterate over all process device data structure and check - * if we should reset all wavefronts */ - list_for_each_entry(pdd, &p->per_device_data, per_device_list) + * if we should delete debug managers and reset all wavefronts + */ + list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + if ((pdd->dev->dbgmgr) && + (pdd->dev->dbgmgr->pasid == p->pasid)) + kfd_dbgmgr_destroy(pdd->dev->dbgmgr); + if (pdd->reset_wavefronts) { pr_warn("amdkfd: Resetting all wave fronts\n"); dbgdev_wave_reset_wavefronts(pdd->dev, p); pdd->reset_wavefronts = false; } + } mutex_unlock(&p->mutex); @@ -404,42 +410,52 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid) idx = srcu_read_lock(&kfd_processes_srcu); + /* + * Look for the process that matches the pasid. If there is no such + * process, we either released it in amdkfd's own notifier, or there + * is a bug. Unfortunately, there is no way to tell... + */ hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes) - if (p->pasid == pasid) - break; + if (p->pasid == pasid) { - srcu_read_unlock(&kfd_processes_srcu, idx); + srcu_read_unlock(&kfd_processes_srcu, idx); - BUG_ON(p->pasid != pasid); + pr_debug("Unbinding process %d from IOMMU\n", pasid); - mutex_lock(&p->mutex); + mutex_lock(&p->mutex); - if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid)) - kfd_dbgmgr_destroy(dev->dbgmgr); + if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid)) + kfd_dbgmgr_destroy(dev->dbgmgr); - pqm_uninit(&p->pqm); + pqm_uninit(&p->pqm); - pdd = kfd_get_process_device_data(dev, p); + pdd = kfd_get_process_device_data(dev, p); - if (!pdd) { - mutex_unlock(&p->mutex); - return; - } + if (!pdd) { + mutex_unlock(&p->mutex); + return; + } - if (pdd->reset_wavefronts) { - dbgdev_wave_reset_wavefronts(pdd->dev, p); - pdd->reset_wavefronts = false; - } + if (pdd->reset_wavefronts) { + dbgdev_wave_reset_wavefronts(pdd->dev, p); + pdd->reset_wavefronts = false; + } - /* - * Just mark pdd as unbound, because we still need it to call - * amd_iommu_unbind_pasid() in when the process exits. - * We don't call amd_iommu_unbind_pasid() here - * because the IOMMU called us. - */ - pdd->bound = false; + /* + * Just mark pdd as unbound, because we still need it + * to call amd_iommu_unbind_pasid() in when the + * process exits. + * We don't call amd_iommu_unbind_pasid() here + * because the IOMMU called us. + */ + pdd->bound = false; - mutex_unlock(&p->mutex); + mutex_unlock(&p->mutex); + + return; + } + + srcu_read_unlock(&kfd_processes_srcu, idx); } struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p) diff --git a/kernel/drivers/gpu/drm/ast/ast_main.c b/kernel/drivers/gpu/drm/ast/ast_main.c index e0b4586a2..9b8f0b975 100644 --- a/kernel/drivers/gpu/drm/ast/ast_main.c +++ b/kernel/drivers/gpu/drm/ast/ast_main.c @@ -223,7 +223,8 @@ static int ast_get_dram_info(struct drm_device *dev) ast_write32(ast, 0x10000, 0xfc600309); do { - ; + if (pci_channel_offline(dev->pdev)) + return -EIO; } while (ast_read32(ast, 0x10000) != 0x01); data = ast_read32(ast, 0x10004); @@ -429,7 +430,9 @@ int ast_driver_load(struct drm_device *dev, unsigned long flags) ast_detect_chip(dev, &need_post); if (ast->chip != AST1180) { - ast_get_dram_info(dev); + ret = ast_get_dram_info(dev); + if (ret) + goto out_free; ast->vram_size = ast_get_vram_info(dev); DRM_INFO("dram %d %d %d %08x\n", ast->mclk, ast->dram_type, ast->dram_bus_width, ast->vram_size); } diff --git a/kernel/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/kernel/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c index d0299aed5..e231176cb 100644 --- a/kernel/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c +++ b/kernel/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c @@ -316,25 +316,27 @@ atmel_hlcdc_plane_update_pos_and_size(struct atmel_hlcdc_plane *plane, u32 *coeff_tab = heo_upscaling_ycoef; u32 max_memsize; - if (state->crtc_w < state->src_w) + if (state->crtc_h < state->src_h) coeff_tab = heo_downscaling_ycoef; for (i = 0; i < ARRAY_SIZE(heo_upscaling_ycoef); i++) atmel_hlcdc_layer_update_cfg(&plane->layer, 33 + i, 0xffffffff, coeff_tab[i]); - factor = ((8 * 256 * state->src_w) - (256 * 4)) / - state->crtc_w; + factor = ((8 * 256 * state->src_h) - (256 * 4)) / + state->crtc_h; factor++; - max_memsize = ((factor * state->crtc_w) + (256 * 4)) / + max_memsize = ((factor * state->crtc_h) + (256 * 4)) / 2048; - if (max_memsize > state->src_w) + if (max_memsize > state->src_h) factor--; factor_reg |= (factor << 16) | 0x80000000; } atmel_hlcdc_layer_update_cfg(&plane->layer, 13, 0xffffffff, factor_reg); + } else { + atmel_hlcdc_layer_update_cfg(&plane->layer, 13, 0xffffffff, 0); } } diff --git a/kernel/drivers/gpu/drm/drm_atomic.c b/kernel/drivers/gpu/drm/drm_atomic.c index aeee083c7..6253775b8 100644 --- a/kernel/drivers/gpu/drm/drm_atomic.c +++ b/kernel/drivers/gpu/drm/drm_atomic.c @@ -150,7 +150,7 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state) for (i = 0; i < state->num_connector; i++) { struct drm_connector *connector = state->connectors[i]; - if (!connector) + if (!connector || !connector->funcs) continue; /* @@ -367,6 +367,8 @@ int drm_atomic_set_mode_prop_for_crtc(struct drm_crtc_state *state, drm_property_unreference_blob(state->mode_blob); state->mode_blob = NULL; + memset(&state->mode, 0, sizeof(state->mode)); + if (blob) { if (blob->length != sizeof(struct drm_mode_modeinfo) || drm_mode_convert_umode(&state->mode, @@ -379,7 +381,6 @@ int drm_atomic_set_mode_prop_for_crtc(struct drm_crtc_state *state, DRM_DEBUG_ATOMIC("Set [MODE:%s] for CRTC state %p\n", state->mode.name, state); } else { - memset(&state->mode, 0, sizeof(state->mode)); state->enable = false; DRM_DEBUG_ATOMIC("Set [NOMODE] for CRTC state %p\n", state); diff --git a/kernel/drivers/gpu/drm/drm_atomic_helper.c b/kernel/drivers/gpu/drm/drm_atomic_helper.c index e5aec45bf..1ac29d703 100644 --- a/kernel/drivers/gpu/drm/drm_atomic_helper.c +++ b/kernel/drivers/gpu/drm/drm_atomic_helper.c @@ -108,7 +108,6 @@ steal_encoder(struct drm_atomic_state *state, struct drm_crtc_state *crtc_state; struct drm_connector *connector; struct drm_connector_state *connector_state; - int ret; /* * We can only steal an encoder coming from a connector, which means we @@ -139,9 +138,6 @@ steal_encoder(struct drm_atomic_state *state, if (IS_ERR(connector_state)) return PTR_ERR(connector_state); - ret = drm_atomic_set_crtc_for_connector(connector_state, NULL); - if (ret) - return ret; connector_state->best_encoder = NULL; } diff --git a/kernel/drivers/gpu/drm/drm_cache.c b/kernel/drivers/gpu/drm/drm_cache.c index 6743ff7dc..7f4a6c550 100644 --- a/kernel/drivers/gpu/drm/drm_cache.c +++ b/kernel/drivers/gpu/drm/drm_cache.c @@ -136,6 +136,7 @@ drm_clflush_virt_range(void *addr, unsigned long length) mb(); for (; addr < end; addr += size) clflushopt(addr); + clflushopt(end - 1); /* force serialisation */ mb(); return; } diff --git a/kernel/drivers/gpu/drm/drm_crtc.c b/kernel/drivers/gpu/drm/drm_crtc.c index 24c5434ab..5e4bb4837 100644 --- a/kernel/drivers/gpu/drm/drm_crtc.c +++ b/kernel/drivers/gpu/drm/drm_crtc.c @@ -2682,8 +2682,6 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data, goto out; } - drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V); - /* * Check whether the primary plane supports the fb pixel format. * Drivers not implementing the universal planes API use a @@ -3316,6 +3314,24 @@ int drm_mode_addfb2(struct drm_device *dev, return 0; } +struct drm_mode_rmfb_work { + struct work_struct work; + struct list_head fbs; +}; + +static void drm_mode_rmfb_work_fn(struct work_struct *w) +{ + struct drm_mode_rmfb_work *arg = container_of(w, typeof(*arg), work); + + while (!list_empty(&arg->fbs)) { + struct drm_framebuffer *fb = + list_first_entry(&arg->fbs, typeof(*fb), filp_head); + + list_del_init(&fb->filp_head); + drm_framebuffer_remove(fb); + } +} + /** * drm_mode_rmfb - remove an FB from the configuration * @dev: drm device for the ioctl @@ -3356,7 +3372,25 @@ int drm_mode_rmfb(struct drm_device *dev, mutex_unlock(&dev->mode_config.fb_lock); mutex_unlock(&file_priv->fbs_lock); - drm_framebuffer_unreference(fb); + /* + * we now own the reference that was stored in the fbs list + * + * drm_framebuffer_remove may fail with -EINTR on pending signals, + * so run this in a separate stack as there's no way to correctly + * handle this after the fb is already removed from the lookup table. + */ + if (atomic_read(&fb->refcount.refcount) > 1) { + struct drm_mode_rmfb_work arg; + + INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn); + INIT_LIST_HEAD(&arg.fbs); + list_add_tail(&fb->filp_head, &arg.fbs); + + schedule_work(&arg.work); + flush_work(&arg.work); + destroy_work_on_stack(&arg.work); + } else + drm_framebuffer_unreference(fb); return 0; @@ -3509,7 +3543,6 @@ out_err1: return ret; } - /** * drm_fb_release - remove and free the FBs on this file * @priv: drm file for the ioctl @@ -3524,6 +3557,9 @@ out_err1: void drm_fb_release(struct drm_file *priv) { struct drm_framebuffer *fb, *tfb; + struct drm_mode_rmfb_work arg; + + INIT_LIST_HEAD(&arg.fbs); /* * When the file gets released that means no one else can access the fb @@ -3536,10 +3572,22 @@ void drm_fb_release(struct drm_file *priv) * at it any more. */ list_for_each_entry_safe(fb, tfb, &priv->fbs, filp_head) { - list_del_init(&fb->filp_head); + if (atomic_read(&fb->refcount.refcount) > 1) { + list_move_tail(&fb->filp_head, &arg.fbs); + } else { + list_del_init(&fb->filp_head); - /* This drops the fpriv->fbs reference. */ - drm_framebuffer_unreference(fb); + /* This drops the fpriv->fbs reference. */ + drm_framebuffer_unreference(fb); + } + } + + if (!list_empty(&arg.fbs)) { + INIT_WORK_ONSTACK(&arg.work, drm_mode_rmfb_work_fn); + + schedule_work(&arg.work); + flush_work(&arg.work); + destroy_work_on_stack(&arg.work); } } @@ -5183,6 +5231,9 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev, unsigned long flags; int ret = -EINVAL; + if (!drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + if (page_flip->flags & ~DRM_MODE_PAGE_FLIP_FLAGS || page_flip->reserved != 0) return -EINVAL; diff --git a/kernel/drivers/gpu/drm/drm_dp_helper.c b/kernel/drivers/gpu/drm/drm_dp_helper.c index 9535c5b60..7e5a97204 100644 --- a/kernel/drivers/gpu/drm/drm_dp_helper.c +++ b/kernel/drivers/gpu/drm/drm_dp_helper.c @@ -178,7 +178,7 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, { struct drm_dp_aux_msg msg; unsigned int retry; - int err; + int err = 0; memset(&msg, 0, sizeof(msg)); msg.address = offset; @@ -186,6 +186,8 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, msg.buffer = buffer; msg.size = size; + mutex_lock(&aux->hw_mutex); + /* * The specification doesn't give any recommendation on how often to * retry native transactions. We used to retry 7 times like for @@ -194,25 +196,24 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, */ for (retry = 0; retry < 32; retry++) { - mutex_lock(&aux->hw_mutex); err = aux->transfer(aux, &msg); - mutex_unlock(&aux->hw_mutex); if (err < 0) { if (err == -EBUSY) continue; - return err; + goto unlock; } switch (msg.reply & DP_AUX_NATIVE_REPLY_MASK) { case DP_AUX_NATIVE_REPLY_ACK: if (err < size) - return -EPROTO; - return err; + err = -EPROTO; + goto unlock; case DP_AUX_NATIVE_REPLY_NACK: - return -EIO; + err = -EIO; + goto unlock; case DP_AUX_NATIVE_REPLY_DEFER: usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100); @@ -221,7 +222,11 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, } DRM_DEBUG_KMS("too many retries, giving up\n"); - return -EIO; + err = -EIO; + +unlock: + mutex_unlock(&aux->hw_mutex); + return err; } /** @@ -543,9 +548,7 @@ static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) int max_retries = max(7, drm_dp_i2c_retry_count(msg, dp_aux_i2c_speed_khz)); for (retry = 0, defer_i2c = 0; retry < (max_retries + defer_i2c); retry++) { - mutex_lock(&aux->hw_mutex); ret = aux->transfer(aux, msg); - mutex_unlock(&aux->hw_mutex); if (ret < 0) { if (ret == -EBUSY) continue; @@ -684,6 +687,8 @@ static int drm_dp_i2c_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, memset(&msg, 0, sizeof(msg)); + mutex_lock(&aux->hw_mutex); + for (i = 0; i < num; i++) { msg.address = msgs[i].addr; drm_dp_i2c_msg_set_request(&msg, &msgs[i]); @@ -738,6 +743,8 @@ static int drm_dp_i2c_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, msg.size = 0; (void)drm_dp_i2c_do_msg(aux, &msg); + mutex_unlock(&aux->hw_mutex); + return err; } diff --git a/kernel/drivers/gpu/drm/drm_dp_mst_topology.c b/kernel/drivers/gpu/drm/drm_dp_mst_topology.c index 39d7e2e15..7cb2815e8 100644 --- a/kernel/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/kernel/drivers/gpu/drm/drm_dp_mst_topology.c @@ -909,6 +909,7 @@ static void drm_dp_destroy_port(struct kref *kref) /* no need to clean up vcpi * as if we have no connector we never setup a vcpi */ drm_dp_port_teardown_pdt(port, port->pdt); + port->pdt = DP_PEER_DEVICE_NONE; } kfree(port); } @@ -1154,7 +1155,9 @@ static void drm_dp_add_port(struct drm_dp_mst_branch *mstb, drm_dp_put_port(port); goto out; } - if (port->port_num >= DP_MST_LOGICAL_PORT_0) { + if ((port->pdt == DP_PEER_DEVICE_DP_LEGACY_CONV || + port->pdt == DP_PEER_DEVICE_SST_SINK) && + port->port_num >= DP_MST_LOGICAL_PORT_0) { port->cached_edid = drm_get_edid(port->connector, &port->aux.ddc); drm_mode_connector_set_tile_property(port->connector); } @@ -1665,13 +1668,19 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_branch *mstb; int len, ret, port_num; + port = drm_dp_get_validated_port_ref(mgr, port); + if (!port) + return -EINVAL; + port_num = port->port_num; mstb = drm_dp_get_validated_mstb_ref(mgr, port->parent); if (!mstb) { mstb = drm_dp_get_last_connected_port_and_mstb(mgr, port->parent, &port_num); - if (!mstb) + if (!mstb) { + drm_dp_put_port(port); return -EINVAL; + } } txmsg = kzalloc(sizeof(*txmsg), GFP_KERNEL); @@ -1697,6 +1706,7 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr, kfree(txmsg); fail_put: drm_dp_put_mst_branch_device(mstb); + drm_dp_put_port(port); return ret; } @@ -1779,6 +1789,11 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr) req_payload.start_slot = cur_slots; if (mgr->proposed_vcpis[i]) { port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi); + port = drm_dp_get_validated_port_ref(mgr, port); + if (!port) { + mutex_unlock(&mgr->payload_lock); + return -EINVAL; + } req_payload.num_slots = mgr->proposed_vcpis[i]->num_slots; } else { port = NULL; @@ -1804,6 +1819,9 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr) mgr->payloads[i].payload_state = req_payload.payload_state; } cur_slots += req_payload.num_slots; + + if (port) + drm_dp_put_port(port); } for (i = 0; i < mgr->max_payloads; i++) { @@ -2109,6 +2127,8 @@ int drm_dp_mst_topology_mgr_resume(struct drm_dp_mst_topology_mgr *mgr) if (mgr->mst_primary) { int sret; + u8 guid[16]; + sret = drm_dp_dpcd_read(mgr->aux, DP_DPCD_REV, mgr->dpcd, DP_RECEIVER_CAP_SIZE); if (sret != DP_RECEIVER_CAP_SIZE) { DRM_DEBUG_KMS("dpcd read failed - undocked during suspend?\n"); @@ -2123,6 +2143,16 @@ int drm_dp_mst_topology_mgr_resume(struct drm_dp_mst_topology_mgr *mgr) ret = -1; goto out_unlock; } + + /* Some hubs forget their guids after they resume */ + sret = drm_dp_dpcd_read(mgr->aux, DP_GUID, guid, 16); + if (sret != 16) { + DRM_DEBUG_KMS("dpcd read failed - undocked during suspend?\n"); + ret = -1; + goto out_unlock; + } + drm_dp_check_mstb_guid(mgr->mst_primary, guid); + ret = 0; } else ret = -1; @@ -2845,13 +2875,12 @@ static void drm_dp_destroy_connector_work(struct work_struct *work) mgr->cbs->destroy_connector(mgr, port->connector); drm_dp_port_teardown_pdt(port, port->pdt); + port->pdt = DP_PEER_DEVICE_NONE; if (!port->input && port->vcpi.vcpi > 0) { - if (mgr->mst_state) { - drm_dp_mst_reset_vcpi_slots(mgr, port); - drm_dp_update_payload_part1(mgr); - drm_dp_mst_put_payload_id(mgr, port->vcpi.vcpi); - } + drm_dp_mst_reset_vcpi_slots(mgr, port); + drm_dp_update_payload_part1(mgr); + drm_dp_mst_put_payload_id(mgr, port->vcpi.vcpi); } kref_put(&port->kref, drm_dp_free_mst_port); diff --git a/kernel/drivers/gpu/drm/drm_edid.c b/kernel/drivers/gpu/drm/drm_edid.c index d5d2c03fd..8c9ac0216 100644 --- a/kernel/drivers/gpu/drm/drm_edid.c +++ b/kernel/drivers/gpu/drm/drm_edid.c @@ -73,6 +73,8 @@ #define EDID_QUIRK_FORCE_8BPC (1 << 8) /* Force 12bpc */ #define EDID_QUIRK_FORCE_12BPC (1 << 9) +/* Force 6bpc */ +#define EDID_QUIRK_FORCE_6BPC (1 << 10) struct detailed_mode_closure { struct drm_connector *connector; @@ -99,6 +101,9 @@ static struct edid_quirk { /* Unknown Acer */ { "ACR", 2423, EDID_QUIRK_FIRST_DETAILED_PREFERRED }, + /* AEO model 0 reports 8 bpc, but is a 6 bpc panel */ + { "AEO", 0, EDID_QUIRK_FORCE_6BPC }, + /* Belinea 10 15 55 */ { "MAX", 1516, EDID_QUIRK_PREFER_LARGE_60 }, { "MAX", 0x77e, EDID_QUIRK_PREFER_LARGE_60 }, @@ -3820,6 +3825,9 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid) drm_add_display_info(edid, &connector->display_info, connector); + if (quirks & EDID_QUIRK_FORCE_6BPC) + connector->display_info.bpc = 6; + if (quirks & EDID_QUIRK_FORCE_8BPC) connector->display_info.bpc = 8; diff --git a/kernel/drivers/gpu/drm/drm_fb_helper.c b/kernel/drivers/gpu/drm/drm_fb_helper.c index 69cbab5e5..5ad036741 100644 --- a/kernel/drivers/gpu/drm/drm_fb_helper.c +++ b/kernel/drivers/gpu/drm/drm_fb_helper.c @@ -1899,7 +1899,6 @@ static int drm_pick_crtcs(struct drm_fb_helper *fb_helper, int n, int width, int height) { int c, o; - struct drm_device *dev = fb_helper->dev; struct drm_connector *connector; const struct drm_connector_helper_funcs *connector_funcs; struct drm_encoder *encoder; @@ -1918,7 +1917,7 @@ static int drm_pick_crtcs(struct drm_fb_helper *fb_helper, if (modes[n] == NULL) return best_score; - crtcs = kzalloc(dev->mode_config.num_connector * + crtcs = kzalloc(fb_helper->connector_count * sizeof(struct drm_fb_helper_crtc *), GFP_KERNEL); if (!crtcs) return best_score; @@ -1964,7 +1963,7 @@ static int drm_pick_crtcs(struct drm_fb_helper *fb_helper, if (score > best_score) { best_score = score; memcpy(best_crtcs, crtcs, - dev->mode_config.num_connector * + fb_helper->connector_count * sizeof(struct drm_fb_helper_crtc *)); } } diff --git a/kernel/drivers/gpu/drm/drm_gem.c b/kernel/drivers/gpu/drm/drm_gem.c index c7de454e8..b205224f1 100644 --- a/kernel/drivers/gpu/drm/drm_gem.c +++ b/kernel/drivers/gpu/drm/drm_gem.c @@ -338,27 +338,32 @@ drm_gem_handle_create_tail(struct drm_file *file_priv, spin_unlock(&file_priv->table_lock); idr_preload_end(); mutex_unlock(&dev->object_name_lock); - if (ret < 0) { - drm_gem_object_handle_unreference_unlocked(obj); - return ret; - } + if (ret < 0) + goto err_unref; + *handlep = ret; ret = drm_vma_node_allow(&obj->vma_node, file_priv->filp); - if (ret) { - drm_gem_handle_delete(file_priv, *handlep); - return ret; - } + if (ret) + goto err_remove; if (dev->driver->gem_open_object) { ret = dev->driver->gem_open_object(obj, file_priv); - if (ret) { - drm_gem_handle_delete(file_priv, *handlep); - return ret; - } + if (ret) + goto err_revoke; } return 0; + +err_revoke: + drm_vma_node_revoke(&obj->vma_node, file_priv->filp); +err_remove: + spin_lock(&file_priv->table_lock); + idr_remove(&file_priv->object_idr, *handlep); + spin_unlock(&file_priv->table_lock); +err_unref: + drm_gem_object_handle_unreference_unlocked(obj); + return ret; } /** diff --git a/kernel/drivers/gpu/drm/drm_ioc32.c b/kernel/drivers/gpu/drm/drm_ioc32.c index 57676f8d7..a6289752b 100644 --- a/kernel/drivers/gpu/drm/drm_ioc32.c +++ b/kernel/drivers/gpu/drm/drm_ioc32.c @@ -1015,6 +1015,7 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd, return 0; } +#if defined(CONFIG_X86) || defined(CONFIG_IA64) typedef struct drm_mode_fb_cmd232 { u32 fb_id; u32 width; @@ -1071,6 +1072,7 @@ static int compat_drm_mode_addfb2(struct file *file, unsigned int cmd, return 0; } +#endif static drm_ioctl_compat_t *drm_compat_ioctls[] = { [DRM_IOCTL_NR(DRM_IOCTL_VERSION32)] = compat_drm_version, @@ -1104,7 +1106,9 @@ static drm_ioctl_compat_t *drm_compat_ioctls[] = { [DRM_IOCTL_NR(DRM_IOCTL_UPDATE_DRAW32)] = compat_drm_update_draw, #endif [DRM_IOCTL_NR(DRM_IOCTL_WAIT_VBLANK32)] = compat_drm_wait_vblank, +#if defined(CONFIG_X86) || defined(CONFIG_IA64) [DRM_IOCTL_NR(DRM_IOCTL_MODE_ADDFB232)] = compat_drm_mode_addfb2, +#endif }; /** diff --git a/kernel/drivers/gpu/drm/drm_modes.c b/kernel/drivers/gpu/drm/drm_modes.c index cd74a0953..71a10f085 100644 --- a/kernel/drivers/gpu/drm/drm_modes.c +++ b/kernel/drivers/gpu/drm/drm_modes.c @@ -1401,6 +1401,13 @@ drm_mode_create_from_cmdline_mode(struct drm_device *dev, return NULL; mode->type |= DRM_MODE_TYPE_USERDEF; + /* fix up 1368x768: GFT/CVT can't express 1366 width due to alignment */ + if (cmd->xres == 1366 && mode->hdisplay == 1368) { + mode->hdisplay = 1366; + mode->hsync_start--; + mode->hsync_end--; + drm_mode_set_name(mode); + } drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V); return mode; } @@ -1487,6 +1494,8 @@ int drm_mode_convert_umode(struct drm_display_mode *out, if (out->status != MODE_OK) goto out; + drm_mode_set_crtcinfo(out, CRTC_INTERLACE_HALVE_V); + ret = 0; out: diff --git a/kernel/drivers/gpu/drm/drm_prime.c b/kernel/drivers/gpu/drm/drm_prime.c index 9f935f55d..968b31f39 100644 --- a/kernel/drivers/gpu/drm/drm_prime.c +++ b/kernel/drivers/gpu/drm/drm_prime.c @@ -339,14 +339,17 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = { * using the PRIME helpers. */ struct dma_buf *drm_gem_prime_export(struct drm_device *dev, - struct drm_gem_object *obj, int flags) + struct drm_gem_object *obj, + int flags) { - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - - exp_info.ops = &drm_gem_prime_dmabuf_ops; - exp_info.size = obj->size; - exp_info.flags = flags; - exp_info.priv = obj; + struct dma_buf_export_info exp_info = { + .exp_name = KBUILD_MODNAME, /* white lie for debug */ + .owner = dev->driver->fops->owner, + .ops = &drm_gem_prime_dmabuf_ops, + .size = obj->size, + .flags = flags, + .priv = obj, + }; if (dev->driver->gem_prime_res_obj) exp_info.resv = dev->driver->gem_prime_res_obj(obj); diff --git a/kernel/drivers/gpu/drm/exynos/exynos_drm_core.c b/kernel/drivers/gpu/drm/exynos/exynos_drm_core.c index 7f55ba677..011211e41 100644 --- a/kernel/drivers/gpu/drm/exynos/exynos_drm_core.c +++ b/kernel/drivers/gpu/drm/exynos/exynos_drm_core.c @@ -101,7 +101,7 @@ int exynos_drm_subdrv_open(struct drm_device *dev, struct drm_file *file) return 0; err: - list_for_each_entry_reverse(subdrv, &subdrv->list, list) { + list_for_each_entry_continue_reverse(subdrv, &exynos_drm_subdrv_list, list) { if (subdrv->close) subdrv->close(dev, subdrv->dev, file); } diff --git a/kernel/drivers/gpu/drm/gma500/cdv_intel_dp.c b/kernel/drivers/gpu/drm/gma500/cdv_intel_dp.c index 17cea400a..d3de377dc 100644 --- a/kernel/drivers/gpu/drm/gma500/cdv_intel_dp.c +++ b/kernel/drivers/gpu/drm/gma500/cdv_intel_dp.c @@ -220,7 +220,7 @@ i2c_dp_aux_prepare_bus(struct i2c_adapter *adapter) * FIXME: This is the old dp aux helper, gma500 is the last driver that needs to * be ported over to the new helper code in drm_dp_helper.c like i915 or radeon. */ -static int __deprecated +static int i2c_dp_aux_add_bus(struct i2c_adapter *adapter) { int error; diff --git a/kernel/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c b/kernel/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c index 6b43ae3ff..1616af209 100644 --- a/kernel/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c +++ b/kernel/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c @@ -72,7 +72,7 @@ static const char *const dsi_errors[] = { "RX Prot Violation", "HS Generic Write FIFO Full", "LP Generic Write FIFO Full", - "Generic Read Data Avail" + "Generic Read Data Avail", "Special Packet Sent", "Tearing Effect", }; diff --git a/kernel/drivers/gpu/drm/gma500/psb_drv.c b/kernel/drivers/gpu/drm/gma500/psb_drv.c index 92e7e5795..db98ab5cd 100644 --- a/kernel/drivers/gpu/drm/gma500/psb_drv.c +++ b/kernel/drivers/gpu/drm/gma500/psb_drv.c @@ -484,6 +484,9 @@ static const struct file_operations psb_gem_fops = { .open = drm_open, .release = drm_release, .unlocked_ioctl = psb_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = drm_compat_ioctl, +#endif .mmap = drm_gem_mmap, .poll = drm_poll, .read = drm_read, diff --git a/kernel/drivers/gpu/drm/i915/i915_drv.h b/kernel/drivers/gpu/drm/i915/i915_drv.h index d3ce4da6a..fb9f647bb 100644 --- a/kernel/drivers/gpu/drm/i915/i915_drv.h +++ b/kernel/drivers/gpu/drm/i915/i915_drv.h @@ -2150,21 +2150,19 @@ struct drm_i915_gem_object { /** Record of address bit 17 of each page at last unbind. */ unsigned long *bit_17; - union { - /** for phy allocated objects */ - struct drm_dma_handle *phys_handle; - - struct i915_gem_userptr { - uintptr_t ptr; - unsigned read_only :1; - unsigned workers :4; + struct i915_gem_userptr { + uintptr_t ptr; + unsigned read_only :1; + unsigned workers :4; #define I915_GEM_USERPTR_MAX_WORKERS 15 - struct i915_mm_struct *mm; - struct i915_mmu_object *mmu_object; - struct work_struct *work; - } userptr; - }; + struct i915_mm_struct *mm; + struct i915_mmu_object *mmu_object; + struct work_struct *work; + } userptr; + + /** for phys allocated objects */ + struct drm_dma_handle *phys_handle; }; #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base) @@ -3313,6 +3311,9 @@ static inline bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) } extern void intel_i2c_reset(struct drm_device *dev); +/* intel_bios.c */ +bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port); + /* intel_opregion.c */ #ifdef CONFIG_ACPI extern int intel_opregion_setup(struct drm_device *dev); diff --git a/kernel/drivers/gpu/drm/i915/i915_gem_gtt.c b/kernel/drivers/gpu/drm/i915/i915_gem_gtt.c index 86c750045..b37fe0df7 100644 --- a/kernel/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/kernel/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -2747,6 +2747,7 @@ void i915_global_gtt_cleanup(struct drm_device *dev) struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; ppgtt->base.cleanup(&ppgtt->base); + kfree(ppgtt); } if (drm_mm_initialized(&vm->mm)) { diff --git a/kernel/drivers/gpu/drm/i915/i915_gem_shrinker.c b/kernel/drivers/gpu/drm/i915/i915_gem_shrinker.c index 9686fa273..deb1e207f 100644 --- a/kernel/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/kernel/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -39,7 +39,7 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) if (!mutex_is_locked(mutex)) return false; -#if (defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)) && !defined(CONFIG_PREEMPT_RT_BASE) +#if (defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)) && !defined(CONFIG_PREEMPT_RT_BASE) return mutex->owner == task; #else /* Since UP may be pre-empted, we cannot assume that we own the lock */ diff --git a/kernel/drivers/gpu/drm/i915/i915_gem_stolen.c b/kernel/drivers/gpu/drm/i915/i915_gem_stolen.c index 87e919a06..5d2323a40 100644 --- a/kernel/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/kernel/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -108,17 +108,28 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) pci_read_config_dword(dev->pdev, 0x5c, &base); base &= ~((1<<20) - 1); } else if (IS_I865G(dev)) { + u32 tseg_size = 0; u16 toud = 0; + u8 tmp; + + pci_bus_read_config_byte(dev->pdev->bus, PCI_DEVFN(0, 0), + I845_ESMRAMC, &tmp); + + if (tmp & TSEG_ENABLE) { + switch (tmp & I845_TSEG_SIZE_MASK) { + case I845_TSEG_SIZE_512K: + tseg_size = KB(512); + break; + case I845_TSEG_SIZE_1M: + tseg_size = MB(1); + break; + } + } - /* - * FIXME is the graphics stolen memory region - * always at TOUD? Ie. is it always the last - * one to be allocated by the BIOS? - */ pci_bus_read_config_word(dev->pdev->bus, PCI_DEVFN(0, 0), I865_TOUD, &toud); - base = toud << 16; + base = (toud << 16) + tseg_size; } else if (IS_I85X(dev)) { u32 tseg_size = 0; u32 tom; diff --git a/kernel/drivers/gpu/drm/i915/i915_reg.h b/kernel/drivers/gpu/drm/i915/i915_reg.h index bc7b8faba..cace154bb 100644 --- a/kernel/drivers/gpu/drm/i915/i915_reg.h +++ b/kernel/drivers/gpu/drm/i915/i915_reg.h @@ -2838,7 +2838,14 @@ enum skl_disp_power_wells { #define GEN6_RP_STATE_CAP (MCHBAR_MIRROR_BASE_SNB + 0x5998) #define BXT_RP_STATE_CAP 0x138170 -#define INTERVAL_1_28_US(us) (((us) * 100) >> 7) +/* + * Make these a multiple of magic 25 to avoid SNB (eg. Dell XPS + * 8300) freezing up around GPU hangs. Looks as if even + * scheduling/timer interrupts start misbehaving if the RPS + * EI/thresholds are "bad", leading to a very sluggish or even + * frozen machine. + */ +#define INTERVAL_1_28_US(us) roundup(((us) * 100) >> 7, 25) #define INTERVAL_1_33_US(us) (((us) * 3) >> 2) #define INTERVAL_0_833_US(us) (((us) * 6) / 5) #define GT_INTERVAL_FROM_US(dev_priv, us) (IS_GEN9(dev_priv) ? \ @@ -3233,19 +3240,20 @@ enum skl_disp_power_wells { #define PORT_HOTPLUG_STAT (dev_priv->info.display_mmio_offset + 0x61114) /* - * HDMI/DP bits are gen4+ + * HDMI/DP bits are g4x+ * * WARNING: Bspec for hpd status bits on gen4 seems to be completely confused. * Please check the detailed lore in the commit message for for experimental * evidence. */ -#define PORTD_HOTPLUG_LIVE_STATUS_G4X (1 << 29) +/* Bspec says GM45 should match G4X/VLV/CHV, but reality disagrees */ +#define PORTD_HOTPLUG_LIVE_STATUS_GM45 (1 << 29) +#define PORTC_HOTPLUG_LIVE_STATUS_GM45 (1 << 28) +#define PORTB_HOTPLUG_LIVE_STATUS_GM45 (1 << 27) +/* G4X/VLV/CHV DP/HDMI bits again match Bspec */ +#define PORTD_HOTPLUG_LIVE_STATUS_G4X (1 << 27) #define PORTC_HOTPLUG_LIVE_STATUS_G4X (1 << 28) -#define PORTB_HOTPLUG_LIVE_STATUS_G4X (1 << 27) -/* VLV DP/HDMI bits again match Bspec */ -#define PORTD_HOTPLUG_LIVE_STATUS_VLV (1 << 27) -#define PORTC_HOTPLUG_LIVE_STATUS_VLV (1 << 28) -#define PORTB_HOTPLUG_LIVE_STATUS_VLV (1 << 29) +#define PORTB_HOTPLUG_LIVE_STATUS_G4X (1 << 29) #define PORTD_HOTPLUG_INT_STATUS (3 << 21) #define PORTD_HOTPLUG_INT_LONG_PULSE (2 << 21) #define PORTD_HOTPLUG_INT_SHORT_PULSE (1 << 21) @@ -7350,6 +7358,8 @@ enum skl_disp_power_wells { #define TRANS_CLK_SEL_DISABLED (0x0<<29) #define TRANS_CLK_SEL_PORT(x) (((x)+1)<<29) +#define CDCLK_FREQ 0x46200 + #define TRANSA_MSA_MISC 0x60410 #define TRANSB_MSA_MISC 0x61410 #define TRANSC_MSA_MISC 0x62410 diff --git a/kernel/drivers/gpu/drm/i915/intel_bios.c b/kernel/drivers/gpu/drm/i915/intel_bios.c index ce82f9c7d..d14bdc537 100644 --- a/kernel/drivers/gpu/drm/i915/intel_bios.c +++ b/kernel/drivers/gpu/drm/i915/intel_bios.c @@ -1351,3 +1351,42 @@ intel_parse_bios(struct drm_device *dev) return 0; } + +/** + * intel_bios_is_port_present - is the specified digital port present + * @dev_priv: i915 device instance + * @port: port to check + * + * Return true if the device in %port is present. + */ +bool intel_bios_is_port_present(struct drm_i915_private *dev_priv, enum port port) +{ + static const struct { + u16 dp, hdmi; + } port_mapping[] = { + [PORT_B] = { DVO_PORT_DPB, DVO_PORT_HDMIB, }, + [PORT_C] = { DVO_PORT_DPC, DVO_PORT_HDMIC, }, + [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, + [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, + }; + int i; + + /* FIXME maybe deal with port A as well? */ + if (WARN_ON(port == PORT_A) || port >= ARRAY_SIZE(port_mapping)) + return false; + + if (!dev_priv->vbt.child_dev_num) + return false; + + for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { + const union child_device_config *p_child = + &dev_priv->vbt.child_dev[i]; + if ((p_child->common.dvo_port == port_mapping[port].dp || + p_child->common.dvo_port == port_mapping[port].hdmi) && + (p_child->common.device_type & (DEVICE_TYPE_TMDS_DVI_SIGNALING | + DEVICE_TYPE_DISPLAYPORT_OUTPUT))) + return true; + } + + return false; +} diff --git a/kernel/drivers/gpu/drm/i915/intel_crt.c b/kernel/drivers/gpu/drm/i915/intel_crt.c index 6a2c76e36..63fea6a28 100644 --- a/kernel/drivers/gpu/drm/i915/intel_crt.c +++ b/kernel/drivers/gpu/drm/i915/intel_crt.c @@ -248,8 +248,14 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder, pipe_config->has_pch_encoder = true; /* LPT FDI RX only supports 8bpc. */ - if (HAS_PCH_LPT(dev)) + if (HAS_PCH_LPT(dev)) { + if (pipe_config->bw_constrained && pipe_config->pipe_bpp < 24) { + DRM_DEBUG_KMS("LPT only supports 24bpp\n"); + return false; + } + pipe_config->pipe_bpp = 24; + } /* FDI must always be 2.7 GHz */ if (HAS_DDI(dev)) { @@ -439,6 +445,7 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private; struct edid *edid; struct i2c_adapter *i2c; + bool ret = false; BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG); @@ -455,17 +462,17 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) */ if (!is_digital) { DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n"); - return true; + ret = true; + } else { + DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n"); } - - DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [EDID reports a digital panel]\n"); } else { DRM_DEBUG_KMS("CRT not detected via DDC:0x50 [no valid EDID found]\n"); } kfree(edid); - return false; + return ret; } static enum drm_connector_status diff --git a/kernel/drivers/gpu/drm/i915/intel_csr.c b/kernel/drivers/gpu/drm/i915/intel_csr.c index 9e530a739..fc28c512e 100644 --- a/kernel/drivers/gpu/drm/i915/intel_csr.c +++ b/kernel/drivers/gpu/drm/i915/intel_csr.c @@ -180,7 +180,8 @@ struct stepping_info { static const struct stepping_info skl_stepping_info[] = { {'A', '0'}, {'B', '0'}, {'C', '0'}, {'D', '0'}, {'E', '0'}, {'F', '0'}, - {'G', '0'}, {'H', '0'}, {'I', '0'} + {'G', '0'}, {'H', '0'}, {'I', '0'}, + {'J', '0'}, {'K', '0'} }; static struct stepping_info bxt_stepping_info[] = { diff --git a/kernel/drivers/gpu/drm/i915/intel_ddi.c b/kernel/drivers/gpu/drm/i915/intel_ddi.c index 7e6158b88..3c6b07683 100644 --- a/kernel/drivers/gpu/drm/i915/intel_ddi.c +++ b/kernel/drivers/gpu/drm/i915/intel_ddi.c @@ -464,9 +464,17 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, } else if (IS_BROADWELL(dev)) { ddi_translations_fdi = bdw_ddi_translations_fdi; ddi_translations_dp = bdw_ddi_translations_dp; - ddi_translations_edp = bdw_ddi_translations_edp; + + if (dev_priv->edp_low_vswing) { + ddi_translations_edp = bdw_ddi_translations_edp; + n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_edp); + } else { + ddi_translations_edp = bdw_ddi_translations_dp; + n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_dp); + } + ddi_translations_hdmi = bdw_ddi_translations_hdmi; - n_edp_entries = ARRAY_SIZE(bdw_ddi_translations_edp); + n_dp_entries = ARRAY_SIZE(bdw_ddi_translations_dp); n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi); hdmi_default_entry = 7; @@ -3188,12 +3196,6 @@ void intel_ddi_get_config(struct intel_encoder *encoder, intel_ddi_clock_get(encoder, pipe_config); } -static void intel_ddi_destroy(struct drm_encoder *encoder) -{ - /* HDMI has nothing special to destroy, so we can go with this. */ - intel_dp_encoder_destroy(encoder); -} - static bool intel_ddi_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { @@ -3212,7 +3214,8 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder, } static const struct drm_encoder_funcs intel_ddi_funcs = { - .destroy = intel_ddi_destroy, + .reset = intel_dp_encoder_reset, + .destroy = intel_dp_encoder_destroy, }; static struct intel_connector * @@ -3284,6 +3287,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port) intel_encoder->post_disable = intel_ddi_post_disable; intel_encoder->get_hw_state = intel_ddi_get_hw_state; intel_encoder->get_config = intel_ddi_get_config; + intel_encoder->suspend = intel_dp_encoder_suspend; intel_dig_port->port = port; intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) & diff --git a/kernel/drivers/gpu/drm/i915/intel_display.c b/kernel/drivers/gpu/drm/i915/intel_display.c index f1c6c974c..8ecd5c016 100644 --- a/kernel/drivers/gpu/drm/i915/intel_display.c +++ b/kernel/drivers/gpu/drm/i915/intel_display.c @@ -2950,13 +2950,13 @@ u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier, } } -unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane, - struct drm_i915_gem_object *obj, - unsigned int plane) +u32 intel_plane_obj_offset(struct intel_plane *intel_plane, + struct drm_i915_gem_object *obj, + unsigned int plane) { const struct i915_ggtt_view *view = &i915_ggtt_view_normal; struct i915_vma *vma; - unsigned char *offset; + u64 offset; if (intel_rotation_90_or_270(intel_plane->base.state->rotation)) view = &i915_ggtt_view_rotated; @@ -2966,14 +2966,16 @@ unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane, view->type)) return -1; - offset = (unsigned char *)vma->node.start; + offset = vma->node.start; if (plane == 1) { offset += vma->ggtt_view.rotation_info.uv_start_page * PAGE_SIZE; } - return (unsigned long)offset; + WARN_ON(upper_32_bits(offset)); + + return lower_32_bits(offset); } static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) @@ -3099,7 +3101,7 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc, u32 tile_height, plane_offset, plane_size; unsigned int rotation; int x_offset, y_offset; - unsigned long surf_addr; + u32 surf_addr; struct intel_crtc_state *crtc_state = intel_crtc->config; struct intel_plane_state *plane_state; int src_x = 0, src_y = 0, src_w = 0, src_h = 0; @@ -3946,10 +3948,10 @@ static void page_flip_completed(struct intel_crtc *intel_crtc) drm_crtc_vblank_put(&intel_crtc->base); wake_up_all(&dev_priv->pending_flip_queue); - queue_work(dev_priv->wq, &work->work); - trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); + + queue_work(dev_priv->wq, &work->work); } void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) @@ -4447,7 +4449,7 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state) intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX); return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, - &state->scaler_state.scaler_id, DRM_ROTATE_0, + &state->scaler_state.scaler_id, BIT(DRM_ROTATE_0), state->pipe_src_w, state->pipe_src_h, adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay); } @@ -8228,12 +8230,14 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_encoder *encoder; + int i; u32 val, final; bool has_lvds = false; bool has_cpu_edp = false; bool has_panel = false; bool has_ck505 = false; bool can_ssc = false; + bool using_ssc_source = false; /* We need to take the global config into account */ for_each_intel_encoder(dev, encoder) { @@ -8260,8 +8264,22 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) can_ssc = true; } - DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d\n", - has_panel, has_lvds, has_ck505); + /* Check if any DPLLs are using the SSC source */ + for (i = 0; i < dev_priv->num_shared_dpll; i++) { + u32 temp = I915_READ(PCH_DPLL(i)); + + if (!(temp & DPLL_VCO_ENABLE)) + continue; + + if ((temp & PLL_REF_INPUT_MASK) == + PLLB_REF_INPUT_SPREADSPECTRUMIN) { + using_ssc_source = true; + break; + } + } + + DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n", + has_panel, has_lvds, has_ck505, using_ssc_source); /* Ironlake: try to setup display ref clock before DPLL * enabling. This is only under driver's control after @@ -8298,9 +8316,9 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; } else final |= DREF_CPU_SOURCE_OUTPUT_DISABLE; - } else { - final |= DREF_SSC_SOURCE_DISABLE; - final |= DREF_CPU_SOURCE_OUTPUT_DISABLE; + } else if (using_ssc_source) { + final |= DREF_SSC_SOURCE_ENABLE; + final |= DREF_SSC1_ENABLE; } if (final == val) @@ -8346,7 +8364,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) POSTING_READ(PCH_DREF_CONTROL); udelay(200); } else { - DRM_DEBUG_KMS("Disabling SSC entirely\n"); + DRM_DEBUG_KMS("Disabling CPU source output\n"); val &= ~DREF_CPU_SOURCE_OUTPUT_MASK; @@ -8357,16 +8375,20 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) POSTING_READ(PCH_DREF_CONTROL); udelay(200); - /* Turn off the SSC source */ - val &= ~DREF_SSC_SOURCE_MASK; - val |= DREF_SSC_SOURCE_DISABLE; + if (!using_ssc_source) { + DRM_DEBUG_KMS("Disabling SSC source\n"); - /* Turn off SSC1 */ - val &= ~DREF_SSC1_ENABLE; + /* Turn off the SSC source */ + val &= ~DREF_SSC_SOURCE_MASK; + val |= DREF_SSC_SOURCE_DISABLE; - I915_WRITE(PCH_DREF_CONTROL, val); - POSTING_READ(PCH_DREF_CONTROL); - udelay(200); + /* Turn off SSC1 */ + val &= ~DREF_SSC1_ENABLE; + + I915_WRITE(PCH_DREF_CONTROL, val); + POSTING_READ(PCH_DREF_CONTROL); + udelay(200); + } } BUG_ON(val != final); @@ -9669,6 +9691,8 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data); mutex_unlock(&dev_priv->rps.hw_lock); + I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); + intel_update_cdclk(dev); WARN(cdclk != dev_priv->cdclk_freq, @@ -11930,21 +11954,11 @@ connected_sink_compute_bpp(struct intel_connector *connector, pipe_config->pipe_bpp = connector->base.display_info.bpc*3; } - /* Clamp bpp to default limit on screens without EDID 1.4 */ - if (connector->base.display_info.bpc == 0) { - int type = connector->base.connector_type; - int clamp_bpp = 24; - - /* Fall back to 18 bpp when DP sink capability is unknown. */ - if (type == DRM_MODE_CONNECTOR_DisplayPort || - type == DRM_MODE_CONNECTOR_eDP) - clamp_bpp = 18; - - if (bpp > clamp_bpp) { - DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of %d\n", - bpp, clamp_bpp); - pipe_config->pipe_bpp = clamp_bpp; - } + /* Clamp bpp to 8 on screens without EDID 1.4 */ + if (connector->base.display_info.bpc == 0 && bpp > 24) { + DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n", + bpp); + pipe_config->pipe_bpp = 24; } } @@ -14148,6 +14162,8 @@ static void intel_setup_outputs(struct drm_device *dev) if (I915_READ(PCH_DP_D) & DP_DETECTED) intel_dp_init(dev, PCH_DP_D, PORT_D); } else if (IS_VALLEYVIEW(dev)) { + bool has_edp, has_port; + /* * The DP_DETECTED bit is the latched state of the DDC * SDA pin at boot. However since eDP doesn't require DDC @@ -14156,27 +14172,37 @@ static void intel_setup_outputs(struct drm_device *dev) * Thus we can't rely on the DP_DETECTED bit alone to detect * eDP ports. Consult the VBT as well as DP_DETECTED to * detect eDP ports. + * + * Sadly the straps seem to be missing sometimes even for HDMI + * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap + * and VBT for the presence of the port. Additionally we can't + * trust the port type the VBT declares as we've seen at least + * HDMI ports that the VBT claim are DP or eDP. */ - if (I915_READ(VLV_HDMIB) & SDVO_DETECTED && - !intel_dp_is_edp(dev, PORT_B)) + has_edp = intel_dp_is_edp(dev, PORT_B); + has_port = intel_bios_is_port_present(dev_priv, PORT_B); + if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port) + has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B); + if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp) intel_hdmi_init(dev, VLV_HDMIB, PORT_B); - if (I915_READ(VLV_DP_B) & DP_DETECTED || - intel_dp_is_edp(dev, PORT_B)) - intel_dp_init(dev, VLV_DP_B, PORT_B); - if (I915_READ(VLV_HDMIC) & SDVO_DETECTED && - !intel_dp_is_edp(dev, PORT_C)) + has_edp = intel_dp_is_edp(dev, PORT_C); + has_port = intel_bios_is_port_present(dev_priv, PORT_C); + if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port) + has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C); + if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp) intel_hdmi_init(dev, VLV_HDMIC, PORT_C); - if (I915_READ(VLV_DP_C) & DP_DETECTED || - intel_dp_is_edp(dev, PORT_C)) - intel_dp_init(dev, VLV_DP_C, PORT_C); if (IS_CHERRYVIEW(dev)) { - /* eDP not supported on port D, so don't check VBT */ - if (I915_READ(CHV_HDMID) & SDVO_DETECTED) - intel_hdmi_init(dev, CHV_HDMID, PORT_D); - if (I915_READ(CHV_DP_D) & DP_DETECTED) + /* + * eDP not supported on port D, + * so no need to worry about it + */ + has_port = intel_bios_is_port_present(dev_priv, PORT_D); + if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port) intel_dp_init(dev, CHV_DP_D, PORT_D); + if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port) + intel_hdmi_init(dev, CHV_HDMID, PORT_D); } intel_dsi_init(dev); diff --git a/kernel/drivers/gpu/drm/i915/intel_dp.c b/kernel/drivers/gpu/drm/i915/intel_dp.c index 78b8ec84d..0f8367da0 100644 --- a/kernel/drivers/gpu/drm/i915/intel_dp.c +++ b/kernel/drivers/gpu/drm/i915/intel_dp.c @@ -3628,8 +3628,7 @@ static bool intel_dp_reset_link_train(struct intel_dp *intel_dp, uint32_t *DP, uint8_t dp_train_pat) { - if (!intel_dp->train_set_valid) - memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); + memset(intel_dp->train_set, 0, sizeof(intel_dp->train_set)); intel_dp_set_signal_levels(intel_dp, DP); return intel_dp_set_link_train(intel_dp, DP, dp_train_pat); } @@ -3746,22 +3745,6 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) break; } - /* - * if we used previously trained voltage and pre-emphasis values - * and we don't get clock recovery, reset link training values - */ - if (intel_dp->train_set_valid) { - DRM_DEBUG_KMS("clock recovery not ok, reset"); - /* clear the flag as we are not reusing train set */ - intel_dp->train_set_valid = false; - if (!intel_dp_reset_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_1 | - DP_LINK_SCRAMBLING_DISABLE)) { - DRM_ERROR("failed to enable link training\n"); - return; - } - continue; - } /* Check to see if we've tried the max voltage */ for (i = 0; i < intel_dp->lane_count; i++) @@ -3854,7 +3837,6 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) /* Make sure clock is still ok */ if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { - intel_dp->train_set_valid = false; intel_dp_link_training_clock_recovery(intel_dp); intel_dp_set_link_train(intel_dp, &DP, training_pattern | @@ -3871,7 +3853,6 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) /* Try 5 times, then try clock recovery if that fails */ if (tries > 5) { - intel_dp->train_set_valid = false; intel_dp_link_training_clock_recovery(intel_dp); intel_dp_set_link_train(intel_dp, &DP, training_pattern | @@ -3893,10 +3874,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) intel_dp->DP = DP; - if (channel_eq) { - intel_dp->train_set_valid = true; + if (channel_eq) DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n"); - } } void intel_dp_stop_link_train(struct intel_dp *intel_dp) @@ -4613,20 +4592,20 @@ static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv, return I915_READ(PORT_HOTPLUG_STAT) & bit; } -static bool vlv_digital_port_connected(struct drm_i915_private *dev_priv, - struct intel_digital_port *port) +static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv, + struct intel_digital_port *port) { u32 bit; switch (port->port) { case PORT_B: - bit = PORTB_HOTPLUG_LIVE_STATUS_VLV; + bit = PORTB_HOTPLUG_LIVE_STATUS_GM45; break; case PORT_C: - bit = PORTC_HOTPLUG_LIVE_STATUS_VLV; + bit = PORTC_HOTPLUG_LIVE_STATUS_GM45; break; case PORT_D: - bit = PORTD_HOTPLUG_LIVE_STATUS_VLV; + bit = PORTD_HOTPLUG_LIVE_STATUS_GM45; break; default: MISSING_CASE(port->port); @@ -4669,7 +4648,7 @@ static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv, * * Return %true if @port is connected, %false otherwise. */ -bool intel_digital_port_connected(struct drm_i915_private *dev_priv, +static bool intel_digital_port_connected(struct drm_i915_private *dev_priv, struct intel_digital_port *port) { if (HAS_PCH_IBX(dev_priv)) @@ -4678,8 +4657,8 @@ bool intel_digital_port_connected(struct drm_i915_private *dev_priv, return cpt_digital_port_connected(dev_priv, port); else if (IS_BROXTON(dev_priv)) return bxt_digital_port_connected(dev_priv, port); - else if (IS_VALLEYVIEW(dev_priv)) - return vlv_digital_port_connected(dev_priv, port); + else if (IS_GM45(dev_priv)) + return gm45_digital_port_connected(dev_priv, port); else return g4x_digital_port_connected(dev_priv, port); } @@ -5035,7 +5014,7 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) kfree(intel_dig_port); } -static void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) +void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); @@ -5077,15 +5056,17 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) edp_panel_vdd_schedule_off(intel_dp); } -static void intel_dp_encoder_reset(struct drm_encoder *encoder) +void intel_dp_encoder_reset(struct drm_encoder *encoder) { - struct intel_dp *intel_dp; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + if (!HAS_DDI(dev_priv)) + intel_dp->DP = I915_READ(intel_dp->output_reg); if (to_intel_encoder(encoder)->type != INTEL_OUTPUT_EDP) return; - intel_dp = enc_to_intel_dp(encoder); - pps_lock(intel_dp); /* @@ -5157,9 +5138,6 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) intel_display_power_get(dev_priv, power_domain); if (long_hpd) { - /* indicate that we need to restart link training */ - intel_dp->train_set_valid = false; - if (!intel_digital_port_connected(dev_priv, intel_dig_port)) goto mst_fail; @@ -6135,8 +6113,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, return true; } -void -intel_dp_init(struct drm_device *dev, int output_reg, enum port port) +bool intel_dp_init(struct drm_device *dev, + int output_reg, + enum port port) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_digital_port *intel_dig_port; @@ -6146,7 +6125,7 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) intel_dig_port = kzalloc(sizeof(*intel_dig_port), GFP_KERNEL); if (!intel_dig_port) - return; + return false; intel_connector = intel_connector_alloc(); if (!intel_connector) @@ -6201,15 +6180,14 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) if (!intel_dp_init_connector(intel_dig_port, intel_connector)) goto err_init_connector; - return; + return true; err_init_connector: drm_encoder_cleanup(encoder); kfree(intel_connector); err_connector_alloc: kfree(intel_dig_port); - - return; + return false; } void intel_dp_mst_suspend(struct drm_device *dev) diff --git a/kernel/drivers/gpu/drm/i915/intel_dp_mst.c b/kernel/drivers/gpu/drm/i915/intel_dp_mst.c index 0639275fc..06bd9257a 100644 --- a/kernel/drivers/gpu/drm/i915/intel_dp_mst.c +++ b/kernel/drivers/gpu/drm/i915/intel_dp_mst.c @@ -477,6 +477,8 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; + intel_connector->unregister(intel_connector); + /* need to nuke the connector */ drm_modeset_lock_all(dev); if (connector->state->crtc) { @@ -490,11 +492,7 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, WARN(ret, "Disabling mst crtc failed with %i\n", ret); } - drm_modeset_unlock_all(dev); - intel_connector->unregister(intel_connector); - - drm_modeset_lock_all(dev); intel_connector_remove_from_fbdev(intel_connector); drm_connector_cleanup(connector); drm_modeset_unlock_all(dev); diff --git a/kernel/drivers/gpu/drm/i915/intel_drv.h b/kernel/drivers/gpu/drm/i915/intel_drv.h index 0d00f07b7..722aa159c 100644 --- a/kernel/drivers/gpu/drm/i915/intel_drv.h +++ b/kernel/drivers/gpu/drm/i915/intel_drv.h @@ -783,7 +783,6 @@ struct intel_dp { bool has_aux_irq, int send_bytes, uint32_t aux_clock_divider); - bool train_set_valid; /* Displayport compliance testing */ unsigned long compliance_test_type; @@ -1178,9 +1177,9 @@ void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file); int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state); int skl_max_scale(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); -unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane, - struct drm_i915_gem_object *obj, - unsigned int plane); +u32 intel_plane_obj_offset(struct intel_plane *intel_plane, + struct drm_i915_gem_object *obj, + unsigned int plane); u32 skl_plane_ctl_format(uint32_t pixel_format); u32 skl_plane_ctl_tiling(uint64_t fb_modifier); @@ -1196,7 +1195,7 @@ void intel_csr_ucode_fini(struct drm_device *dev); void assert_csr_loaded(struct drm_i915_private *dev_priv); /* intel_dp.c */ -void intel_dp_init(struct drm_device *dev, int output_reg, enum port port); +bool intel_dp_init(struct drm_device *dev, int output_reg, enum port port); bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector); void intel_dp_set_link_params(struct intel_dp *intel_dp, @@ -1204,6 +1203,8 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp, void intel_dp_start_link_train(struct intel_dp *intel_dp); void intel_dp_stop_link_train(struct intel_dp *intel_dp); void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); +void intel_dp_encoder_reset(struct drm_encoder *encoder); +void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder); void intel_dp_encoder_destroy(struct drm_encoder *encoder); int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc); bool intel_dp_compute_config(struct intel_encoder *encoder, @@ -1230,8 +1231,6 @@ void intel_edp_drrs_disable(struct intel_dp *intel_dp); void intel_edp_drrs_invalidate(struct drm_device *dev, unsigned frontbuffer_bits); void intel_edp_drrs_flush(struct drm_device *dev, unsigned frontbuffer_bits); -bool intel_digital_port_connected(struct drm_i915_private *dev_priv, - struct intel_digital_port *port); void hsw_dp_set_ddi_pll_sel(struct intel_crtc_state *pipe_config); /* intel_dp_mst.c */ diff --git a/kernel/drivers/gpu/drm/i915/intel_fbdev.c b/kernel/drivers/gpu/drm/i915/intel_fbdev.c index 4fd5fdfef..c0c094d5b 100644 --- a/kernel/drivers/gpu/drm/i915/intel_fbdev.c +++ b/kernel/drivers/gpu/drm/i915/intel_fbdev.c @@ -362,12 +362,12 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, uint64_t conn_configured = 0, mask; int pass = 0; - save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool), + save_enabled = kcalloc(fb_helper->connector_count, sizeof(bool), GFP_KERNEL); if (!save_enabled) return false; - memcpy(save_enabled, enabled, dev->mode_config.num_connector); + memcpy(save_enabled, enabled, fb_helper->connector_count); mask = (1 << fb_helper->connector_count) - 1; retry: for (i = 0; i < fb_helper->connector_count; i++) { @@ -501,7 +501,7 @@ retry: if (fallback) { bail: DRM_DEBUG_KMS("Not using firmware configuration\n"); - memcpy(enabled, save_enabled, dev->mode_config.num_connector); + memcpy(enabled, save_enabled, fb_helper->connector_count); kfree(save_enabled); return false; } diff --git a/kernel/drivers/gpu/drm/i915/intel_hdmi.c b/kernel/drivers/gpu/drm/i915/intel_hdmi.c index e6c035b0f..1ea8532f5 100644 --- a/kernel/drivers/gpu/drm/i915/intel_hdmi.c +++ b/kernel/drivers/gpu/drm/i915/intel_hdmi.c @@ -1331,19 +1331,18 @@ intel_hdmi_unset_edid(struct drm_connector *connector) } static bool -intel_hdmi_set_edid(struct drm_connector *connector, bool force) +intel_hdmi_set_edid(struct drm_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); - struct edid *edid = NULL; + struct edid *edid; bool connected = false; intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); - if (force) - edid = drm_get_edid(connector, - intel_gmbus_get_adapter(dev_priv, - intel_hdmi->ddc_bus)); + edid = drm_get_edid(connector, + intel_gmbus_get_adapter(dev_priv, + intel_hdmi->ddc_bus)); intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS); @@ -1371,29 +1370,16 @@ static enum drm_connector_status intel_hdmi_detect(struct drm_connector *connector, bool force) { enum drm_connector_status status; - struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); struct drm_i915_private *dev_priv = to_i915(connector->dev); - bool live_status = false; - unsigned int try; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); - for (try = 0; !live_status && try < 9; try++) { - if (try) - msleep(10); - live_status = intel_digital_port_connected(dev_priv, - hdmi_to_dig_port(intel_hdmi)); - } - - if (!live_status) - DRM_DEBUG_KMS("Live status not up!"); - intel_hdmi_unset_edid(connector); - if (intel_hdmi_set_edid(connector, live_status)) { + if (intel_hdmi_set_edid(connector)) { struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; @@ -1419,7 +1405,7 @@ intel_hdmi_force(struct drm_connector *connector) if (connector->status != connector_status_connected) return; - intel_hdmi_set_edid(connector, true); + intel_hdmi_set_edid(connector); hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI; } @@ -2011,6 +1997,50 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *c intel_hdmi->aspect_ratio = HDMI_PICTURE_ASPECT_NONE; } +static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv, + enum port port) +{ + const struct ddi_vbt_port_info *info = + &dev_priv->vbt.ddi_port_info[port]; + u8 ddc_pin; + + if (info->alternate_ddc_pin) { + DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (VBT)\n", + info->alternate_ddc_pin, port_name(port)); + return info->alternate_ddc_pin; + } + + switch (port) { + case PORT_B: + if (IS_BROXTON(dev_priv)) + ddc_pin = GMBUS_PIN_1_BXT; + else + ddc_pin = GMBUS_PIN_DPB; + break; + case PORT_C: + if (IS_BROXTON(dev_priv)) + ddc_pin = GMBUS_PIN_2_BXT; + else + ddc_pin = GMBUS_PIN_DPC; + break; + case PORT_D: + if (IS_CHERRYVIEW(dev_priv)) + ddc_pin = GMBUS_PIN_DPD_CHV; + else + ddc_pin = GMBUS_PIN_DPD; + break; + default: + MISSING_CASE(port); + ddc_pin = GMBUS_PIN_DPB; + break; + } + + DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (platform default)\n", + ddc_pin, port_name(port)); + + return ddc_pin; +} + void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector) { @@ -2020,7 +2050,9 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_dig_port->port; - uint8_t alternate_ddc_pin; + + DRM_DEBUG_KMS("Adding HDMI connector on port %c\n", + port_name(port)); drm_connector_init(dev, connector, &intel_hdmi_connector_funcs, DRM_MODE_CONNECTOR_HDMIA); @@ -2030,12 +2062,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, connector->doublescan_allowed = 0; connector->stereo_allowed = 1; + intel_hdmi->ddc_bus = intel_hdmi_ddc_pin(dev_priv, port); + switch (port) { case PORT_B: - if (IS_BROXTON(dev_priv)) - intel_hdmi->ddc_bus = GMBUS_PIN_1_BXT; - else - intel_hdmi->ddc_bus = GMBUS_PIN_DPB; /* * On BXT A0/A1, sw needs to activate DDIA HPD logic and * interrupts to check the external panel connection. @@ -2046,46 +2076,17 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_encoder->hpd_pin = HPD_PORT_B; break; case PORT_C: - if (IS_BROXTON(dev_priv)) - intel_hdmi->ddc_bus = GMBUS_PIN_2_BXT; - else - intel_hdmi->ddc_bus = GMBUS_PIN_DPC; intel_encoder->hpd_pin = HPD_PORT_C; break; case PORT_D: - if (WARN_ON(IS_BROXTON(dev_priv))) - intel_hdmi->ddc_bus = GMBUS_PIN_DISABLED; - else if (IS_CHERRYVIEW(dev_priv)) - intel_hdmi->ddc_bus = GMBUS_PIN_DPD_CHV; - else - intel_hdmi->ddc_bus = GMBUS_PIN_DPD; intel_encoder->hpd_pin = HPD_PORT_D; break; case PORT_E: - /* On SKL PORT E doesn't have seperate GMBUS pin - * We rely on VBT to set a proper alternate GMBUS pin. */ - alternate_ddc_pin = - dev_priv->vbt.ddi_port_info[PORT_E].alternate_ddc_pin; - switch (alternate_ddc_pin) { - case DDC_PIN_B: - intel_hdmi->ddc_bus = GMBUS_PIN_DPB; - break; - case DDC_PIN_C: - intel_hdmi->ddc_bus = GMBUS_PIN_DPC; - break; - case DDC_PIN_D: - intel_hdmi->ddc_bus = GMBUS_PIN_DPD; - break; - default: - MISSING_CASE(alternate_ddc_pin); - } intel_encoder->hpd_pin = HPD_PORT_E; break; - case PORT_A: - intel_encoder->hpd_pin = HPD_PORT_A; - /* Internal port only for eDP. */ default: - BUG(); + MISSING_CASE(port); + return; } if (IS_VALLEYVIEW(dev)) { diff --git a/kernel/drivers/gpu/drm/i915/intel_lrc.c b/kernel/drivers/gpu/drm/i915/intel_lrc.c index d69547a65..7058f75c7 100644 --- a/kernel/drivers/gpu/drm/i915/intel_lrc.c +++ b/kernel/drivers/gpu/drm/i915/intel_lrc.c @@ -776,11 +776,11 @@ static int logical_ring_prepare(struct drm_i915_gem_request *req, int bytes) if (unlikely(total_bytes > remain_usable)) { /* * The base request will fit but the reserved space - * falls off the end. So only need to to wait for the - * reserved size after flushing out the remainder. + * falls off the end. So don't need an immediate wrap + * and only need to effectively wait for the reserved + * size space from the start of ringbuffer. */ wait_bytes = remain_actual + ringbuf->reserved_size; - need_wrap = true; } else if (total_bytes > ringbuf->space) { /* No wrapping required, just waiting. */ wait_bytes = total_bytes; diff --git a/kernel/drivers/gpu/drm/i915/intel_opregion.c b/kernel/drivers/gpu/drm/i915/intel_opregion.c index 6dc13c02c..e362a3077 100644 --- a/kernel/drivers/gpu/drm/i915/intel_opregion.c +++ b/kernel/drivers/gpu/drm/i915/intel_opregion.c @@ -682,7 +682,7 @@ static void intel_didl_outputs(struct drm_device *dev) } if (!acpi_video_bus) { - DRM_ERROR("No ACPI video bus found\n"); + DRM_DEBUG_KMS("No ACPI video bus found\n"); return; } diff --git a/kernel/drivers/gpu/drm/i915/intel_pm.c b/kernel/drivers/gpu/drm/i915/intel_pm.c index f091ad12d..3f802163f 100644 --- a/kernel/drivers/gpu/drm/i915/intel_pm.c +++ b/kernel/drivers/gpu/drm/i915/intel_pm.c @@ -1789,16 +1789,20 @@ static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate, const struct intel_plane_state *pstate, uint32_t mem_value) { - int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0; + /* + * We treat the cursor plane as always-on for the purposes of watermark + * calculation. Until we have two-stage watermark programming merged, + * this is necessary to avoid flickering. + */ + int cpp = 4; + int width = pstate->visible ? pstate->base.crtc_w : 64; - if (!cstate->base.active || !pstate->visible) + if (!cstate->base.active) return 0; return ilk_wm_method2(ilk_pipe_pixel_rate(cstate), cstate->base.adjusted_mode.crtc_htotal, - drm_rect_width(&pstate->dst), - bpp, - mem_value); + width, cpp, mem_value); } /* Only for WM_LP. */ @@ -2092,33 +2096,35 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & GEN9_MEM_LATENCY_LEVEL_MASK; + /* + * If a level n (n > 1) has a 0us latency, all levels m (m >= n) + * need to be disabled. We make sure to sanitize the values out + * of the punit to satisfy this requirement. + */ + for (level = 1; level <= max_level; level++) { + if (wm[level] == 0) { + for (i = level + 1; i <= max_level; i++) + wm[i] = 0; + break; + } + } + /* * WaWmMemoryReadLatency:skl * * punit doesn't take into account the read latency so we need - * to add 2us to the various latency levels we retrieve from - * the punit. - * - W0 is a bit special in that it's the only level that - * can't be disabled if we want to have display working, so - * we always add 2us there. - * - For levels >=1, punit returns 0us latency when they are - * disabled, so we respect that and don't add 2us then - * - * Additionally, if a level n (n > 1) has a 0us latency, all - * levels m (m >= n) need to be disabled. We make sure to - * sanitize the values out of the punit to satisfy this - * requirement. + * to add 2us to the various latency levels we retrieve from the + * punit when level 0 response data us 0us. */ - wm[0] += 2; - for (level = 1; level <= max_level; level++) - if (wm[level] != 0) + if (wm[0] == 0) { + wm[0] += 2; + for (level = 1; level <= max_level; level++) { + if (wm[level] == 0) + break; wm[level] += 2; - else { - for (i = level + 1; i <= max_level; i++) - wm[i] = 0; - - break; } + } + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { uint64_t sskpd = I915_READ64(MCH_SSKPD); @@ -3880,6 +3886,8 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) if (IS_HASWELL(dev) || IS_BROADWELL(dev)) hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); + memset(active, 0, sizeof(*active)); + active->pipe_enabled = intel_crtc->active; if (active->pipe_enabled) { @@ -4520,7 +4528,8 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) else gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); dev_priv->rps.last_adj = 0; - I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); + I915_WRITE(GEN6_PMINTRMSK, + gen6_sanitize_rps_pm_mask(dev_priv, ~0)); } mutex_unlock(&dev_priv->rps.hw_lock); @@ -6620,6 +6629,12 @@ static void broadwell_init_clock_gating(struct drm_device *dev) misccpctl = I915_READ(GEN7_MISCCPCTL); I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT); + /* + * Wait at least 100 clocks before re-enabling clock gating. See + * the definition of L3SQCREG1 in BSpec. + */ + POSTING_READ(GEN8_L3SQCREG1); + udelay(1); I915_WRITE(GEN7_MISCCPCTL, misccpctl); /* diff --git a/kernel/drivers/gpu/drm/i915/intel_ringbuffer.c b/kernel/drivers/gpu/drm/i915/intel_ringbuffer.c index f6b2a814e..9d48443bc 100644 --- a/kernel/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/kernel/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -1922,6 +1922,17 @@ i915_dispatch_execbuffer(struct drm_i915_gem_request *req, return 0; } +static void cleanup_phys_status_page(struct intel_engine_cs *ring) +{ + struct drm_i915_private *dev_priv = to_i915(ring->dev); + + if (!dev_priv->status_page_dmah) + return; + + drm_pci_free(ring->dev, dev_priv->status_page_dmah); + ring->status_page.page_addr = NULL; +} + static void cleanup_status_page(struct intel_engine_cs *ring) { struct drm_i915_gem_object *obj; @@ -1938,9 +1949,9 @@ static void cleanup_status_page(struct intel_engine_cs *ring) static int init_status_page(struct intel_engine_cs *ring) { - struct drm_i915_gem_object *obj; + struct drm_i915_gem_object *obj = ring->status_page.obj; - if ((obj = ring->status_page.obj) == NULL) { + if (obj == NULL) { unsigned flags; int ret; @@ -2134,7 +2145,7 @@ static int intel_init_ring_buffer(struct drm_device *dev, if (ret) goto error; } else { - BUG_ON(ring->id != RCS); + WARN_ON(ring->id != RCS); ret = init_phys_status_page(ring); if (ret) goto error; @@ -2179,7 +2190,12 @@ void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) if (ring->cleanup) ring->cleanup(ring); - cleanup_status_page(ring); + if (I915_NEED_GFX_HWS(ring->dev)) { + cleanup_status_page(ring); + } else { + WARN_ON(ring->id != RCS); + cleanup_phys_status_page(ring); + } i915_cmd_parser_fini_ring(ring); i915_gem_batch_pool_fini(&ring->batch_pool); @@ -2341,11 +2357,11 @@ static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes) if (unlikely(total_bytes > remain_usable)) { /* * The base request will fit but the reserved space - * falls off the end. So only need to to wait for the - * reserved size after flushing out the remainder. + * falls off the end. So don't need an immediate wrap + * and only need to effectively wait for the reserved + * size space from the start of ringbuffer. */ wait_bytes = remain_actual + ringbuf->reserved_size; - need_wrap = true; } else if (total_bytes > ringbuf->space) { /* No wrapping required, just waiting. */ wait_bytes = total_bytes; diff --git a/kernel/drivers/gpu/drm/i915/intel_sprite.c b/kernel/drivers/gpu/drm/i915/intel_sprite.c index 8771d6646..b79d33f14 100644 --- a/kernel/drivers/gpu/drm/i915/intel_sprite.c +++ b/kernel/drivers/gpu/drm/i915/intel_sprite.c @@ -198,7 +198,7 @@ skl_update_plane(struct drm_plane *drm_plane, struct drm_crtc *crtc, int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); const struct drm_intel_sprite_colorkey *key = &to_intel_plane_state(drm_plane->state)->ckey; - unsigned long surf_addr; + u32 surf_addr; u32 tile_height, plane_offset, plane_size; unsigned int rotation; int x_offset, y_offset; diff --git a/kernel/drivers/gpu/drm/i915/intel_uncore.c b/kernel/drivers/gpu/drm/i915/intel_uncore.c index 43cba129a..cc91ae832 100644 --- a/kernel/drivers/gpu/drm/i915/intel_uncore.c +++ b/kernel/drivers/gpu/drm/i915/intel_uncore.c @@ -1132,7 +1132,11 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get_with_thread_status; - dev_priv->uncore.funcs.force_wake_put = fw_domains_put; + if (IS_HASWELL(dev)) + dev_priv->uncore.funcs.force_wake_put = + fw_domains_put_with_fifo; + else + dev_priv->uncore.funcs.force_wake_put = fw_domains_put; fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, FORCEWAKE_MT, FORCEWAKE_ACK_HSW); } else if (IS_IVYBRIDGE(dev)) { diff --git a/kernel/drivers/gpu/drm/imx/imx-drm-core.c b/kernel/drivers/gpu/drm/imx/imx-drm-core.c index 7b990b4e9..5378bdc3b 100644 --- a/kernel/drivers/gpu/drm/imx/imx-drm-core.c +++ b/kernel/drivers/gpu/drm/imx/imx-drm-core.c @@ -26,6 +26,7 @@ #include #include #include +#include