On Sat, Aug 04, 2018 at 01:12:17PM +0100, Johannes Lundberg wrote: > No panic over night with that tunable so it seems you're on the right > track. Please try this, on top of r337316. diff --git a/sys/amd64/linux/linux_machdep.c b/sys/amd64/linux/linux_machdep.c index 6c5b014853f..434ea0eac07 100644 --- a/sys/amd64/linux/linux_machdep.c +++ b/sys/amd64/linux/linux_machdep.c _at__at_ -78,6 +78,9 _at__at_ __FBSDID("$FreeBSD$"); #include <vm/vm_kern.h> #include <vm/vm_map.h> +#include <x86/ifunc.h> +#include <x86/sysarch.h> + #include <amd64/linux/linux.h> #include <amd64/linux/linux_proto.h> #include <compat/linux/linux_emul.h> _at__at_ -88,8 +91,6 _at__at_ __FBSDID("$FreeBSD$"); #include <compat/linux/linux_signal.h> #include <compat/linux/linux_util.h> -#include <x86/include/sysarch.h> - int linux_execve(struct thread *td, struct linux_execve_args *args) { _at__at_ -276,3 +277,48 _at__at_ linux_set_cloned_tls(struct thread *td, void *desc) return (0); } + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); +} diff --git a/sys/amd64/linux/linux_support.s b/sys/amd64/linux/linux_support.s index a9f02160be2..391f76414f2 100644 --- a/sys/amd64/linux/linux_support.s +++ b/sys/amd64/linux/linux_support.s _at__at_ -38,7 +38,7 _at__at_ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -49,25 +49,58 _at__at_ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_nosmap) + +ENTRY(futex_addl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac #ifdef SMP lock #endif xaddl %edi,(%rsi) + clac movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_smap) -ENTRY(futex_orl) +ENTRY(futex_orl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -85,9 +118,31 _at__at_ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -105,9 +160,51 _at__at_ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) + +ENTRY(futex_andl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_xorl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -116,13 +213,15 _at__at_ ENTRY(futex_xorl) movl (%rsi),%eax 1: movl %eax,%ecx xorl %edi,%ecx + stac #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) + clac jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_smap) diff --git a/sys/amd64/linux32/linux32_machdep.c b/sys/amd64/linux32/linux32_machdep.c index ce06be57e9f..61ecc87dc77 100644 --- a/sys/amd64/linux32/linux32_machdep.c +++ b/sys/amd64/linux32/linux32_machdep.c _at__at_ -58,10 +58,12 _at__at_ __FBSDID("$FreeBSD$"); #include <sys/wait.h> #include <machine/frame.h> +#include <machine/md_var.h> #include <machine/pcb.h> #include <machine/psl.h> #include <machine/segments.h> #include <machine/specialreg.h> +#include <x86/ifunc.h> #include <vm/pmap.h> #include <vm/vm.h> _at__at_ -822,3 +824,48 _at__at_ linux_set_thread_area(struct thread *td, return (0); } + +int futex_xchgl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xchgl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xchgl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xchgl_smap : futex_xchgl_nosmap); +} + +int futex_addl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_addl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_addl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_addl_smap : futex_addl_nosmap); +} + +int futex_orl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_orl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_orl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_orl_smap : futex_orl_nosmap); +} + +int futex_andl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_andl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_andl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_andl_smap : futex_andl_nosmap); +} + +int futex_xorl_nosmap(int oparg, uint32_t *uaddr, int *oldval); +int futex_xorl_smap(int oparg, uint32_t *uaddr, int *oldval); +DEFINE_IFUNC(, int, futex_xorl, (int, uint32_t *, int *), static) +{ + + return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? + futex_xorl_smap : futex_xorl_nosmap); +} diff --git a/sys/amd64/linux32/linux32_support.s b/sys/amd64/linux32/linux32_support.s index bba0d8d5e71..981bba9f582 100644 --- a/sys/amd64/linux32/linux32_support.s +++ b/sys/amd64/linux32/linux32_support.s _at__at_ -38,7 +38,7 _at__at_ futex_fault: movl $-EFAULT,%eax ret -ENTRY(futex_xchgl) +ENTRY(futex_xchgl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -49,25 +49,58 _at__at_ ENTRY(futex_xchgl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xchgl) +END(futex_xchgl_nosmap) -ENTRY(futex_addl) +ENTRY(futex_xchgl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault + stac + xchgl %edi,(%rsi) + clac + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xchgl_smap) + +ENTRY(futex_addl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault +#ifdef SMP + lock +#endif + xaddl %edi,(%rsi) + movl %edi,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_addl_nosmap) + +ENTRY(futex_addl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + stac #ifdef SMP lock #endif xaddl %edi,(%rsi) + clac movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_addl) +END(futex_addl_smap) -ENTRY(futex_orl) +ENTRY(futex_orl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -85,9 +118,31 _at__at_ ENTRY(futex_orl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_orl) +END(futex_orl_nosmap) -ENTRY(futex_andl) +ENTRY(futex_orl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + orl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_orl_smap) + +ENTRY(futex_andl_nosmap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -105,9 +160,51 _at__at_ ENTRY(futex_andl) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_andl) +END(futex_andl_nosmap) + +ENTRY(futex_andl_smap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + andl %edi,%ecx + stac +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + clac + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_andl_smap) + +ENTRY(futex_xorl_nosmap) + movq PCPU(CURPCB),%r8 + movq $futex_fault,PCB_ONFAULT(%r8) + movq $VM_MAXUSER_ADDRESS-4,%rax + cmpq %rax,%rsi + ja futex_fault + movl (%rsi),%eax +1: movl %eax,%ecx + xorl %edi,%ecx +#ifdef SMP + lock +#endif + cmpxchgl %ecx,(%rsi) + jnz 1b + movl %eax,(%rdx) + xorl %eax,%eax + movq %rax,PCB_ONFAULT(%r8) + ret +END(futex_xorl_nosmap) -ENTRY(futex_xorl) +ENTRY(futex_xorl_smap) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax _at__at_ -116,13 +213,15 _at__at_ ENTRY(futex_xorl) movl (%rsi),%eax 1: movl %eax,%ecx xorl %edi,%ecx + stac #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) + clac jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret -END(futex_xorl) +END(futex_xorl_smap)Received on Sat Aug 04 2018 - 12:22:47 UTC
This archive was generated by hypermail 2.4.0 : Wed May 19 2021 - 11:41:17 UTC