External interrupts (EVENT_TYPE_EXTINT) and system calls (EVENT_TYPE_OTHER) occur more frequently than other events in a typical system. Prioritizing these events saves CPU cycles and optimizes the efficiency of performance- critical paths.
When examining the compiler-generated assembly code for event dispatching in the functions fred_entry_from_user() and fred_entry_from_kernel(), it was observed that the compiler intelligently uses a binary search to match all event type values (0-7) and perform dispatching. As a result, even if the following cases:
case EVENT_TYPE_EXTINT: return fred_extint(regs); case EVENT_TYPE_OTHER: return fred_other(regs);
are placed at the beginning of the switch() statement, the generated assembly code would remain the same, and the expected prioritization would not be achieved.
Command line to check the assembly code generated by the compiler for fred_entry_from_user():
$objdump -d vmlinux.o | awk '/<fred_entry_from_user>:/{c=65} c&&c--'
00000000000015a0 <fred_entry_from_user>: 15a0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax 15a7: 48 8b 77 78 mov 0x78(%rdi),%rsi 15ab: 55 push %rbp 15ac: 48 c7 47 78 ff ff ff movq $0xffffffffffffffff,0x78(%rdi) 15b3: ff 15b4: 83 e0 0f and $0xf,%eax 15b7: 48 89 e5 mov %rsp,%rbp 15ba: 3c 04 cmp $0x4,%al -->> /* match 4(EVENT_TYPE_SWINT) first */ 15bc: 74 78 je 1636 <fred_entry_from_user+0x96> 15be: 77 15 ja 15d5 <fred_entry_from_user+0x35> 15c0: 3c 02 cmp $0x2,%al 15c2: 74 53 je 1617 <fred_entry_from_user+0x77> 15c4: 77 65 ja 162b <fred_entry_from_user+0x8b> 15c6: 84 c0 test %al,%al 15c8: 75 42 jne 160c <fred_entry_from_user+0x6c> 15ca: e8 71 fc ff ff callq 1240 <fred_extint> 15cf: 5d pop %rbp 15d0: e9 00 00 00 00 jmpq 15d5 <fred_entry_from_user+0x35> 15d5: 3c 06 cmp $0x6,%al 15d7: 74 7c je 1655 <fred_entry_from_user+0xb5> 15d9: 72 66 jb 1641 <fred_entry_from_user+0xa1> 15db: 3c 07 cmp $0x7,%al 15dd: 75 2d jne 160c <fred_entry_from_user+0x6c> 15df: 8b 87 a4 00 00 00 mov 0xa4(%rdi),%eax 15e5: 25 ff 00 00 02 and $0x20000ff,%eax 15ea: 3d 01 00 00 02 cmp $0x2000001,%eax 15ef: 75 6f jne 1660 <fred_entry_from_user+0xc0> 15f1: 48 8b 77 50 mov 0x50(%rdi),%rsi 15f5: 48 c7 47 50 da ff ff movq $0xffffffffffffffda,0x50(%rdi) ... ...
Command line to check the assembly code generated by the compiler for fred_entry_from_kernel():
$objdump -d vmlinux.o | awk '/<fred_entry_from_kernel>:/{c=65} c&&c--'
00000000000016b0 <fred_entry_from_kernel>: 16b0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax 16b7: 48 8b 77 78 mov 0x78(%rdi),%rsi 16bb: 55 push %rbp 16bc: 48 c7 47 78 ff ff ff movq $0xffffffffffffffff,0x78(%rdi) 16c3: ff 16c4: 83 e0 0f and $0xf,%eax 16c7: 48 89 e5 mov %rsp,%rbp 16ca: 3c 03 cmp $0x3,%al -->> /* match 3(EVENT_TYPE_HWEXC) first */ 16cc: 74 3c je 170a <fred_entry_from_kernel+0x5a> 16ce: 76 13 jbe 16e3 <fred_entry_from_kernel+0x33> 16d0: 3c 05 cmp $0x5,%al 16d2: 74 41 je 1715 <fred_entry_from_kernel+0x65> 16d4: 3c 06 cmp $0x6,%al 16d6: 75 27 jne 16ff <fred_entry_from_kernel+0x4f> 16d8: e8 73 fe ff ff callq 1550 <fred_swexc.isra.3> 16dd: 5d pop %rbp ... ...
Therefore, it is necessary to handle EVENT_TYPE_EXTINT and EVENT_TYPE_OTHER before the switch statement using if-else syntax to ensure the compiler generates the desired code. After applying the patch, the verification results are as follows:
$objdump -d vmlinux.o | awk '/<fred_entry_from_user>:/{c=65} c&&c--'
00000000000015a0 <fred_entry_from_user>: 15a0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax 15a7: 48 8b 77 78 mov 0x78(%rdi),%rsi 15ab: 55 push %rbp 15ac: 48 c7 47 78 ff ff ff movq $0xffffffffffffffff,0x78(%rdi) 15b3: ff 15b4: 48 89 e5 mov %rsp,%rbp 15b7: 83 e0 0f and $0xf,%eax 15ba: 74 34 je 15f0 <fred_entry_from_user+0x50> -->> /* match 0(EVENT_TYPE_EXTINT) first */ 15bc: 3c 07 cmp $0x7,%al -->> /* match 7(EVENT_TYPE_OTHER) second * 15be: 74 6e je 162e <fred_entry_from_user+0x8e> 15c0: 3c 04 cmp $0x4,%al 15c2: 0f 84 93 00 00 00 je 165b <fred_entry_from_user+0xbb> 15c8: 76 13 jbe 15dd <fred_entry_from_user+0x3d> 15ca: 3c 05 cmp $0x5,%al 15cc: 74 41 je 160f <fred_entry_from_user+0x6f> 15ce: 3c 06 cmp $0x6,%al 15d0: 75 51 jne 1623 <fred_entry_from_user+0x83> 15d2: e8 79 ff ff ff callq 1550 <fred_swexc.isra.3> 15d7: 5d pop %rbp 15d8: e9 00 00 00 00 jmpq 15dd <fred_entry_from_user+0x3d> 15dd: 3c 02 cmp $0x2,%al 15df: 74 1a je 15fb <fred_entry_from_user+0x5b> 15e1: 3c 03 cmp $0x3,%al 15e3: 75 3e jne 1623 <fred_entry_from_user+0x83> ... ...
The same desired code in fred_entry_from_kernel is no longer repeated.
While the C code with if-else placed before switch() may appear ugly, it works. Additionally, using a jump table is not advisable; even if the jump table resides in the L1 cache, the cost of loading it is over 10 times the latency of a cmp instruction.
Signed-off-by: Ethan Zhao haifeng.zhao@linux.intel.com --- base commit: 619f0b6fad524f08d493a98d55bac9ab8895e3a6 --- arch/x86/entry/entry_fred.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index f004a4dc74c2..591f47771ecf 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c @@ -228,9 +228,18 @@ __visible noinstr void fred_entry_from_user(struct pt_regs *regs) /* Invalidate orig_ax so that syscall_get_nr() works correctly */ regs->orig_ax = -1;
- switch (regs->fred_ss.type) { - case EVENT_TYPE_EXTINT: + if (regs->fred_ss.type == EVENT_TYPE_EXTINT) return fred_extint(regs); + else if (regs->fred_ss.type == EVENT_TYPE_OTHER) + return fred_other(regs); + + /* + * Dispatch EVENT_TYPE_EXTINT and EVENT_TYPE_OTHER(syscall) type events + * first due to their high probability and let the compiler create binary search + * dispatching for the remaining events + */ + + switch (regs->fred_ss.type) { case EVENT_TYPE_NMI: if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) return fred_exc_nmi(regs); @@ -245,8 +254,6 @@ __visible noinstr void fred_entry_from_user(struct pt_regs *regs) break; case EVENT_TYPE_SWEXC: return fred_swexc(regs, error_code); - case EVENT_TYPE_OTHER: - return fred_other(regs); default: break; }
@@ -260,9 +267,15 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) /* Invalidate orig_ax so that syscall_get_nr() works correctly */ regs->orig_ax = -1;
- switch (regs->fred_ss.type) { - case EVENT_TYPE_EXTINT: + if (regs->fred_ss.type == EVENT_TYPE_EXTINT) return fred_extint(regs); + + /* + * Dispatch EVENT_TYPE_EXTINT type event first due to its high probability + * and let the compiler do binary search dispatching for the other events + */ + + switch (regs->fred_ss.type) { case EVENT_TYPE_NMI: if (likely(regs->fred_ss.vector == X86_TRAP_NMI)) return fred_exc_nmi(regs);