# git rev-parse -q --verify de8a34242c88473ee615f66a7ded095f094e708e^{commit} de8a34242c88473ee615f66a7ded095f094e708e already have revision, skipping fetch # git checkout -q -f -B kisskb de8a34242c88473ee615f66a7ded095f094e708e # git clean -qxdf # < git log -1 # commit de8a34242c88473ee615f66a7ded095f094e708e # Author: Nicholas Piggin # Date: Thu Jul 9 17:07:59 2020 +1000 # # lazy tlb: shoot lazies, a non-refcounting lazy tlb option # # On big systems, the mm refcount can become highly contented when doing # a lot of context switching with threaded applications (particularly # switching between the idle thread and an application thread). # # Abandoning lazy tlb slows switching down quite a bit in the important # user->idle->user cases, so so instead implement a non-refcounted scheme # that causes __mmdrop() to IPI all CPUs in the mm_cpumask and shoot down # any remaining lazy ones. # # On a 16-socket 192-core POWER8 system, a context switching benchmark # with as many software threads as CPUs (so each switch will go in and # out of idle), upstream can achieve a rate of about 1 million context # switches per second. After this patch it goes up to 118 million. # # Signed-off-by: Nicholas Piggin # < /opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc --version # < /opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux-ld --version # < git log --format=%s --max-count=1 de8a34242c88473ee615f66a7ded095f094e708e # < make -s -j 48 ARCH=x86 O=/kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 CROSS_COMPILE=/opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux- x86_64_defconfig # < make -s -j 48 ARCH=x86 O=/kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 CROSS_COMPILE=/opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux- help # make -s -j 48 ARCH=x86 O=/kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 CROSS_COMPILE=/opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux- olddefconfig # make -s -j 48 ARCH=x86 O=/kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 CROSS_COMPILE=/opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux- In file included from /kisskb/src/include/linux/init.h:5, from /kisskb/src/arch/x86/platform/efi/quirks.c:4: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[4]: *** [/kisskb/src/scripts/Makefile.build:280: arch/x86/platform/efi/quirks.o] Error 1 make[4]: *** Waiting for unfinished jobs.... In file included from /kisskb/src/include/linux/export.h:43, from /kisskb/src/include/linux/linkage.h:7, from /kisskb/src/include/linux/kernel.h:8, from /kisskb/src/arch/x86/platform/efi/efi.c:34: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[4]: *** [/kisskb/src/scripts/Makefile.build:280: arch/x86/platform/efi/efi.o] Error 1 make[3]: *** [/kisskb/src/scripts/Makefile.build:497: arch/x86/platform/efi] Error 2 make[2]: *** [/kisskb/src/scripts/Makefile.build:497: arch/x86/platform] Error 2 make[2]: *** Waiting for unfinished jobs.... In file included from /kisskb/src/include/linux/dev_printk.h:14, from /kisskb/src/include/linux/device.h:15, from /kisskb/src/include/linux/node.h:18, from /kisskb/src/include/linux/cpu.h:17, from /kisskb/src/arch/x86/kernel/process_64.c:18: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[3]: *** [/kisskb/src/scripts/Makefile.build:280: arch/x86/kernel/process_64.o] Error 1 make[3]: *** Waiting for unfinished jobs.... In file included from /kisskb/src/include/asm-generic/bug.h:5, from /kisskb/src/arch/x86/include/asm/bug.h:92, from /kisskb/src/include/linux/bug.h:5, from /kisskb/src/include/linux/signal.h:5, from /kisskb/src/arch/x86/mm/init_64.c:10: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ In file included from /kisskb/src/arch/x86/include/asm/current.h:5, from /kisskb/src/include/linux/sched.h:12, from /kisskb/src/arch/x86/mm/fault.c:7: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[3]: *** [/kisskb/src/scripts/Makefile.build:280: arch/x86/mm/init_64.o] Error 1 make[3]: *** Waiting for unfinished jobs.... make[3]: *** [/kisskb/src/scripts/Makefile.build:280: arch/x86/mm/fault.o] Error 1 make[2]: *** [/kisskb/src/scripts/Makefile.build:497: arch/x86/mm] Error 2 In file included from /kisskb/src/include/linux/export.h:43, from /kisskb/src/include/linux/linkage.h:7, from /kisskb/src/include/linux/kernel.h:8, from /kisskb/src/include/linux/list.h:9, from /kisskb/src/include/linux/module.h:12, from /kisskb/src/kernel/power/snapshot.c:14: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[3]: *** [/kisskb/src/scripts/Makefile.build:281: kernel/power/snapshot.o] Error 1 make[3]: *** Waiting for unfinished jobs.... make[2]: *** [/kisskb/src/scripts/Makefile.build:497: kernel/power] Error 2 make[2]: *** Waiting for unfinished jobs.... make[1]: *** [/kisskb/src/Makefile:1756: kernel] Error 2 make[1]: *** Waiting for unfinished jobs.... In file included from /kisskb/src/include/asm-generic/bug.h:5, from /kisskb/src/arch/x86/include/asm/bug.h:92, from /kisskb/src/include/linux/bug.h:5, from /kisskb/src/include/linux/mmdebug.h:5, from /kisskb/src/include/linux/mm.h:9, from /kisskb/src/include/linux/pagewalk.h:5, from /kisskb/src/mm/mprotect.c:12: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[2]: *** [/kisskb/src/scripts/Makefile.build:281: mm/mprotect.o] Error 1 make[2]: *** Waiting for unfinished jobs.... make[1]: *** [/kisskb/src/Makefile:1756: mm] Error 2 In file included from /kisskb/src/include/linux/export.h:43, from /kisskb/src/include/linux/linkage.h:7, from /kisskb/src/include/linux/kernel.h:8, from /kisskb/src/drivers/video/fbdev/efifb.c:10: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[4]: *** [/kisskb/src/scripts/Makefile.build:281: drivers/video/fbdev/efifb.o] Error 1 make[3]: *** [/kisskb/src/scripts/Makefile.build:497: drivers/video/fbdev] Error 2 make[2]: *** [/kisskb/src/scripts/Makefile.build:497: drivers/video] Error 2 make[2]: *** Waiting for unfinished jobs.... In file included from /kisskb/src/include/asm-generic/bug.h:5, from /kisskb/src/arch/x86/include/asm/bug.h:92, from /kisskb/src/include/linux/bug.h:5, from /kisskb/src/include/linux/mmdebug.h:5, from /kisskb/src/include/linux/gfp.h:5, from /kisskb/src/include/linux/slab.h:15, from /kisskb/src/fs/coredump.c:2: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[2]: *** [/kisskb/src/scripts/Makefile.build:281: fs/coredump.o] Error 1 make[2]: *** Waiting for unfinished jobs.... make[1]: *** [/kisskb/src/Makefile:1756: fs] Error 2 make[2]: *** [/kisskb/src/scripts/Makefile.build:497: arch/x86/kernel] Error 2 make[1]: *** [/kisskb/src/Makefile:1756: arch/x86] Error 2 In file included from /kisskb/src/include/linux/export.h:43, from /kisskb/src/include/linux/linkage.h:7, from /kisskb/src/include/linux/preempt.h:10, from /kisskb/src/include/linux/spinlock.h:51, from /kisskb/src/arch/x86/include/asm/mmu.h:5, from /kisskb/src/arch/x86/include/asm/desc.h:7, from /kisskb/src/arch/x86/include/asm/mmu_context.h:5, from /kisskb/src/include/linux/mmu_context.h:5, from /kisskb/src/drivers/gpu/drm/i915/gem/i915_gem_userptr.c:7: /kisskb/src/arch/x86/include/asm/mmu_context.h: In function 'exit_lazy_tlb': /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: error: 'MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE' undeclared (first use in this function) MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ /kisskb/src/arch/x86/include/asm/mmu_context.h:116:22: note: each undeclared identifier is reported only once for each function it appears in MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /kisskb/src/include/linux/compiler.h:77:40: note: in definition of macro 'likely' # define likely(x) __builtin_expect(!!(x), 1) ^ make[5]: *** [/kisskb/src/scripts/Makefile.build:281: drivers/gpu/drm/i915/gem/i915_gem_userptr.o] Error 1 make[5]: *** Waiting for unfinished jobs.... make[4]: *** [/kisskb/src/scripts/Makefile.build:497: drivers/gpu/drm/i915] Error 2 make[3]: *** [/kisskb/src/scripts/Makefile.build:497: drivers/gpu/drm] Error 2 make[2]: *** [/kisskb/src/scripts/Makefile.build:497: drivers/gpu] Error 2 make[1]: *** [/kisskb/src/Makefile:1756: drivers] Error 2 make: *** [Makefile:185: __sub-make] Error 2 Command 'make -s -j 48 ARCH=x86 O=/kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 CROSS_COMPILE=/opt/cross/kisskb/korg/gcc-8.1.0-nolibc/x86_64-linux/bin/x86_64-linux- ' returned non-zero exit status 2 # rm -rf /kisskb/build/npiggin_x86_64_defconfig_x86_64-gcc8 # Build took: 0:01:16.347195