Linux v6.18.37 · 원본 파일 · 온라인 원본
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Exception handling code
4 *
5 * Copyright (C) 2019 ARM Ltd.
6 */
7
8#include <linux/context_tracking.h>
9#include <linux/irq-entry-common.h>
10#include <linux/kasan.h>
11#include <linux/linkage.h>
12#include <linux/livepatch.h>
13#include <linux/lockdep.h>
14#include <linux/ptrace.h>
15#include <linux/resume_user_mode.h>
16#include <linux/sched.h>
17#include <linux/sched/debug.h>
18#include <linux/thread_info.h>
19
20#include <asm/cpufeature.h>
21#include <asm/daifflags.h>
22#include <asm/esr.h>
23#include <asm/exception.h>
24#include <asm/fpsimd.h>
25#include <asm/irq_regs.h>
26#include <asm/kprobes.h>
27#include <asm/mmu.h>
28#include <asm/processor.h>
29#include <asm/sdei.h>
30#include <asm/stacktrace.h>
31#include <asm/sysreg.h>
32#include <asm/system_misc.h>
33
34/*
35 * Handle IRQ/context state management when entering from kernel mode.
36 * Before this function is called it is not safe to call regular kernel code,
37 * instrumentable code, or any code which may trigger an exception.
38 *
39 * This is intended to match the logic in irqentry_enter(), handling the kernel
40 * mode transitions only.
41 */
42static __always_inline irqentry_state_t __enter_from_kernel_mode(struct pt_regs *regs)
43{
44 return irqentry_enter(regs);
45}
46
47static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs)
48{
49 irqentry_state_t state;
50
51 state = __enter_from_kernel_mode(regs);
52 mte_check_tfsr_entry();
53 mte_disable_tco_entry(current);
54
55 return state;
56}
57
58/*
59 * Handle IRQ/context state management when exiting to kernel mode.
60 * After this function returns it is not safe to call regular kernel code,
61 * instrumentable code, or any code which may trigger an exception.
62 *
63 * This is intended to match the logic in irqentry_exit(), handling the kernel
64 * mode transitions only, and with preemption handled elsewhere.
65 */
66static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs,
67 irqentry_state_t state)
68{
69 irqentry_exit(regs, state);
70}
71
72static void noinstr exit_to_kernel_mode(struct pt_regs *regs,
73 irqentry_state_t state)
74{
75 mte_check_tfsr_exit();
76 __exit_to_kernel_mode(regs, state);
77}
78
79/*
80 * Handle IRQ/context state management when entering from user mode.
81 * Before this function is called it is not safe to call regular kernel code,
82 * instrumentable code, or any code which may trigger an exception.
83 */
84static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
85{
86 enter_from_user_mode(regs);
87 mte_disable_tco_entry(current);
88 sme_enter_from_user_mode();
89}
90
91static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
92{
93 __enter_from_user_mode(regs);
94}
95
96/*
97 * Handle IRQ/context state management when exiting to user mode.
98 * After this function returns it is not safe to call regular kernel code,
99 * instrumentable code, or any code which may trigger an exception.
100 */
101
102static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
103{
104 local_irq_disable();
105 exit_to_user_mode_prepare(regs);
106 local_daif_mask();
107 sme_exit_to_user_mode();
108 mte_check_tfsr_exit();
109 exit_to_user_mode();
110}
111
112asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
113{
114 arm64_exit_to_user_mode(regs);
115}
116
117/*
118 * Handle IRQ/context state management when entering a debug exception from
119 * kernel mode. Before this function is called it is not safe to call regular
120 * kernel code, instrumentable code, or any code which may trigger an exception.
121 */
122static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs)
123{
124 irqentry_state_t state;
125
126 state.lockdep = lockdep_hardirqs_enabled();
127
128 lockdep_hardirqs_off(CALLER_ADDR0);
129 ct_nmi_enter();
130
131 trace_hardirqs_off_finish();
132
133 return state;
134}
135
136/*
137 * Handle IRQ/context state management when exiting a debug exception from
138 * kernel mode. After this function returns it is not safe to call regular
139 * kernel code, instrumentable code, or any code which may trigger an exception.
140 */
141static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs,
142 irqentry_state_t state)
143{
144 if (state.lockdep) {
145 trace_hardirqs_on_prepare();
146 lockdep_hardirqs_on_prepare();
147 }
148
149 ct_nmi_exit();
150 if (state.lockdep)
151 lockdep_hardirqs_on(CALLER_ADDR0);
152}
153
154static void do_interrupt_handler(struct pt_regs *regs,
155 void (*handler)(struct pt_regs *))
156{
157 struct pt_regs *old_regs = set_irq_regs(regs);
158
159 if (on_thread_stack())
160 call_on_irq_stack(regs, handler);
161 else
162 handler(regs);
163
164 set_irq_regs(old_regs);
165}
166
167extern void (*handle_arch_irq)(struct pt_regs *);
168extern void (*handle_arch_fiq)(struct pt_regs *);
169
170static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
171 unsigned long esr)
172{
173 irqentry_nmi_enter(regs);
174
175 console_verbose();
176
177 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n",
178 vector, smp_processor_id(), esr,
179 esr_get_class_string(esr));
180
181 __show_regs(regs);
182 panic("Unhandled exception");
183}
184
185#define UNHANDLED(el, regsize, vector) \
186asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
187{ \
188 const char *desc = #regsize "-bit " #el " " #vector; \
189 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
190}
191
192#ifdef CONFIG_ARM64_ERRATUM_1463225
193static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
194
195static void cortex_a76_erratum_1463225_svc_handler(void)
196{
197 u64 reg, val;
198
199 if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
200 return;
201
202 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
203 return;
204
205 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
206 reg = read_sysreg(mdscr_el1);
207 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE;
208 write_sysreg(val, mdscr_el1);
209 asm volatile("msr daifclr, #8");
210 isb();
211
212 /* We will have taken a single-step exception by this point */
213
214 write_sysreg(reg, mdscr_el1);
215 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
216}
217
218static __always_inline bool
219cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
220{
221 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
222 return false;
223
224 /*
225 * We've taken a dummy step exception from the kernel to ensure
226 * that interrupts are re-enabled on the syscall path. Return back
227 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
228 * masked so that we can safely restore the mdscr and get on with
229 * handling the syscall.
230 */
231 regs->pstate |= PSR_D_BIT;
232 return true;
233}
234#else /* CONFIG_ARM64_ERRATUM_1463225 */
235static void cortex_a76_erratum_1463225_svc_handler(void) { }
236static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
237{
238 return false;
239}
240#endif /* CONFIG_ARM64_ERRATUM_1463225 */
241
242/*
243 * As per the ABI exit SME streaming mode and clear the SVE state not
244 * shared with FPSIMD on syscall entry.
245 */
246static inline void fpsimd_syscall_enter(void)
247{
248 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */
249 if (system_supports_sme())
250 sme_smstop_sm();
251
252 /*
253 * The CPU is not in streaming mode. If non-streaming SVE is not
254 * supported, there is no SVE state that needs to be discarded.
255 */
256 if (!system_supports_sve())
257 return;
258
259 if (test_thread_flag(TIF_SVE)) {
260 unsigned int sve_vq_minus_one;
261
262 sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
263 sve_flush_live(true, sve_vq_minus_one);
264 }
265
266 /*
267 * Any live non-FPSIMD SVE state has been zeroed. Allow
268 * fpsimd_save_user_state() to lazily discard SVE state until either
269 * the live state is unbound or fpsimd_syscall_exit() is called.
270 */
271 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD);
272}
273
274static __always_inline void fpsimd_syscall_exit(void)
275{
276 if (!system_supports_sve())
277 return;
278
279 /*
280 * The current task's user FPSIMD/SVE/SME state is now bound to this
281 * CPU. The fpsimd_last_state.to_save value is either:
282 *
283 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU
284 * since fpsimd_syscall_enter().
285 *
286 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at
287 * any point.
288 *
289 * Reset this to FP_STATE_CURRENT to stop lazy discarding.
290 */
291 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT);
292}
293
294/*
295 * In debug exception context, we explicitly disable preemption despite
296 * having interrupts disabled.
297 * This serves two purposes: it makes it much less likely that we would
298 * accidentally schedule in exception context and it will force a warning
299 * if we somehow manage to schedule by accident.
300 */
301static void debug_exception_enter(struct pt_regs *regs)
302{
303 preempt_disable();
304
305 /* This code is a bit fragile. Test it. */
306 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
307}
308NOKPROBE_SYMBOL(debug_exception_enter);
309
310static void debug_exception_exit(struct pt_regs *regs)
311{
312 preempt_enable_no_resched();
313}
314NOKPROBE_SYMBOL(debug_exception_exit);
315
316UNHANDLED(el1t, 64, sync)
317UNHANDLED(el1t, 64, irq)
318UNHANDLED(el1t, 64, fiq)
319UNHANDLED(el1t, 64, error)
320
321static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
322{
323 unsigned long far = read_sysreg(far_el1);
324 irqentry_state_t state;
325
326 state = enter_from_kernel_mode(regs);
327 local_daif_inherit(regs);
328 do_mem_abort(far, esr, regs);
329 local_daif_mask();
330 exit_to_kernel_mode(regs, state);
331}
332
333static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
334{
335 unsigned long far = read_sysreg(far_el1);
336 irqentry_state_t state;
337
338 state = enter_from_kernel_mode(regs);
339 local_daif_inherit(regs);
340 do_sp_pc_abort(far, esr, regs);
341 local_daif_mask();
342 exit_to_kernel_mode(regs, state);
343}
344
345static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
346{
347 irqentry_state_t state;
348
349 state = enter_from_kernel_mode(regs);
350 local_daif_inherit(regs);
351 do_el1_undef(regs, esr);
352 local_daif_mask();
353 exit_to_kernel_mode(regs, state);
354}
355
356static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
357{
358 irqentry_state_t state;
359
360 state = enter_from_kernel_mode(regs);
361 local_daif_inherit(regs);
362 do_el1_bti(regs, esr);
363 local_daif_mask();
364 exit_to_kernel_mode(regs, state);
365}
366
367static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)
368{
369 irqentry_state_t state;
370
371 state = enter_from_kernel_mode(regs);
372 local_daif_inherit(regs);
373 do_el1_gcs(regs, esr);
374 local_daif_mask();
375 exit_to_kernel_mode(regs, state);
376}
377
378static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)
379{
380 irqentry_state_t state;
381
382 state = enter_from_kernel_mode(regs);
383 local_daif_inherit(regs);
384 do_el1_mops(regs, esr);
385 local_daif_mask();
386 exit_to_kernel_mode(regs, state);
387}
388
389static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr)
390{
391 irqentry_state_t state;
392
393 state = arm64_enter_el1_dbg(regs);
394 debug_exception_enter(regs);
395 do_breakpoint(esr, regs);
396 debug_exception_exit(regs);
397 arm64_exit_el1_dbg(regs, state);
398}
399
400static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)
401{
402 irqentry_state_t state;
403
404 state = arm64_enter_el1_dbg(regs);
405 if (!cortex_a76_erratum_1463225_debug_handler(regs)) {
406 debug_exception_enter(regs);
407 /*
408 * After handling a breakpoint, we suspend the breakpoint
409 * and use single-step to move to the next instruction.
410 * If we are stepping a suspended breakpoint there's nothing more to do:
411 * the single-step is complete.
412 */
413 if (!try_step_suspended_breakpoints(regs))
414 do_el1_softstep(esr, regs);
415 debug_exception_exit(regs);
416 }
417 arm64_exit_el1_dbg(regs, state);
418}
419
420static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr)
421{
422 /* Watchpoints are the only debug exception to write FAR_EL1 */
423 unsigned long far = read_sysreg(far_el1);
424 irqentry_state_t state;
425
426 state = arm64_enter_el1_dbg(regs);
427 debug_exception_enter(regs);
428 do_watchpoint(far, esr, regs);
429 debug_exception_exit(regs);
430 arm64_exit_el1_dbg(regs, state);
431}
432
433static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr)
434{
435 irqentry_state_t state;
436
437 state = arm64_enter_el1_dbg(regs);
438 debug_exception_enter(regs);
439 do_el1_brk64(esr, regs);
440 debug_exception_exit(regs);
441 arm64_exit_el1_dbg(regs, state);
442}
443
444static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
445{
446 irqentry_state_t state;
447
448 state = enter_from_kernel_mode(regs);
449 local_daif_inherit(regs);
450 do_el1_fpac(regs, esr);
451 local_daif_mask();
452 exit_to_kernel_mode(regs, state);
453}
454
455asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
456{
457 unsigned long esr = read_sysreg(esr_el1);
458
459 switch (ESR_ELx_EC(esr)) {
460 case ESR_ELx_EC_DABT_CUR:
461 case ESR_ELx_EC_IABT_CUR:
462 el1_abort(regs, esr);
463 break;
464 /*
465 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
466 * recursive exception when trying to push the initial pt_regs.
467 */
468 case ESR_ELx_EC_PC_ALIGN:
469 el1_pc(regs, esr);
470 break;
471 case ESR_ELx_EC_SYS64:
472 case ESR_ELx_EC_UNKNOWN:
473 el1_undef(regs, esr);
474 break;
475 case ESR_ELx_EC_BTI:
476 el1_bti(regs, esr);
477 break;
478 case ESR_ELx_EC_GCS:
479 el1_gcs(regs, esr);
480 break;
481 case ESR_ELx_EC_MOPS:
482 el1_mops(regs, esr);
483 break;
484 case ESR_ELx_EC_BREAKPT_CUR:
485 el1_breakpt(regs, esr);
486 break;
487 case ESR_ELx_EC_SOFTSTP_CUR:
488 el1_softstp(regs, esr);
489 break;
490 case ESR_ELx_EC_WATCHPT_CUR:
491 el1_watchpt(regs, esr);
492 break;
493 case ESR_ELx_EC_BRK64:
494 el1_brk64(regs, esr);
495 break;
496 case ESR_ELx_EC_FPAC:
497 el1_fpac(regs, esr);
498 break;
499 default:
500 __panic_unhandled(regs, "64-bit el1h sync", esr);
501 }
502}
503
504static __always_inline void __el1_pnmi(struct pt_regs *regs,
505 void (*handler)(struct pt_regs *))
506{
507 irqentry_state_t state;
508
509 state = irqentry_nmi_enter(regs);
510 do_interrupt_handler(regs, handler);
511 irqentry_nmi_exit(regs, state);
512}
513
514static __always_inline void __el1_irq(struct pt_regs *regs,
515 void (*handler)(struct pt_regs *))
516{
517 irqentry_state_t state;
518
519 state = enter_from_kernel_mode(regs);
520
521 irq_enter_rcu();
522 do_interrupt_handler(regs, handler);
523 irq_exit_rcu();
524
525 exit_to_kernel_mode(regs, state);
526}
527static void noinstr el1_interrupt(struct pt_regs *regs,
528 void (*handler)(struct pt_regs *))
529{
530 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
531
532 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
533 __el1_pnmi(regs, handler);
534 else
535 __el1_irq(regs, handler);
536}
537
538asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
539{
540 el1_interrupt(regs, handle_arch_irq);
541}
542
543asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
544{
545 el1_interrupt(regs, handle_arch_fiq);
546}
547
548asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
549{
550 unsigned long esr = read_sysreg(esr_el1);
551 irqentry_state_t state;
552
553 local_daif_restore(DAIF_ERRCTX);
554 state = irqentry_nmi_enter(regs);
555 do_serror(regs, esr);
556 irqentry_nmi_exit(regs, state);
557}
558
559static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
560{
561 unsigned long far = read_sysreg(far_el1);
562
563 arm64_enter_from_user_mode(regs);
564 local_daif_restore(DAIF_PROCCTX);
565 do_mem_abort(far, esr, regs);
566 arm64_exit_to_user_mode(regs);
567}
568
569static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
570{
571 unsigned long far = read_sysreg(far_el1);
572
573 /*
574 * We've taken an instruction abort from userspace and not yet
575 * re-enabled IRQs. If the address is a kernel address, apply
576 * BP hardening prior to enabling IRQs and pre-emption.
577 */
578 if (!is_ttbr0_addr(far))
579 arm64_apply_bp_hardening();
580
581 arm64_enter_from_user_mode(regs);
582 local_daif_restore(DAIF_PROCCTX);
583 do_mem_abort(far, esr, regs);
584 arm64_exit_to_user_mode(regs);
585}
586
587static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
588{
589 arm64_enter_from_user_mode(regs);
590 local_daif_restore(DAIF_PROCCTX);
591 do_fpsimd_acc(esr, regs);
592 arm64_exit_to_user_mode(regs);
593}
594
595static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
596{
597 arm64_enter_from_user_mode(regs);
598 local_daif_restore(DAIF_PROCCTX);
599 do_sve_acc(esr, regs);
600 arm64_exit_to_user_mode(regs);
601}
602
603static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
604{
605 arm64_enter_from_user_mode(regs);
606 local_daif_restore(DAIF_PROCCTX);
607 do_sme_acc(esr, regs);
608 arm64_exit_to_user_mode(regs);
609}
610
611static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
612{
613 arm64_enter_from_user_mode(regs);
614 local_daif_restore(DAIF_PROCCTX);
615 do_fpsimd_exc(esr, regs);
616 arm64_exit_to_user_mode(regs);
617}
618
619static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
620{
621 arm64_enter_from_user_mode(regs);
622 local_daif_restore(DAIF_PROCCTX);
623 do_el0_sys(esr, regs);
624 arm64_exit_to_user_mode(regs);
625}
626
627static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
628{
629 unsigned long far = read_sysreg(far_el1);
630
631 if (!is_ttbr0_addr(instruction_pointer(regs)))
632 arm64_apply_bp_hardening();
633
634 arm64_enter_from_user_mode(regs);
635 local_daif_restore(DAIF_PROCCTX);
636 do_sp_pc_abort(far, esr, regs);
637 arm64_exit_to_user_mode(regs);
638}
639
640static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
641{
642 arm64_enter_from_user_mode(regs);
643 local_daif_restore(DAIF_PROCCTX);
644 do_sp_pc_abort(regs->sp, esr, regs);
645 arm64_exit_to_user_mode(regs);
646}
647
648static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
649{
650 arm64_enter_from_user_mode(regs);
651 local_daif_restore(DAIF_PROCCTX);
652 do_el0_undef(regs, esr);
653 arm64_exit_to_user_mode(regs);
654}
655
656static void noinstr el0_bti(struct pt_regs *regs)
657{
658 arm64_enter_from_user_mode(regs);
659 local_daif_restore(DAIF_PROCCTX);
660 do_el0_bti(regs);
661 arm64_exit_to_user_mode(regs);
662}
663
664static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
665{
666 arm64_enter_from_user_mode(regs);
667 local_daif_restore(DAIF_PROCCTX);
668 do_el0_mops(regs, esr);
669 arm64_exit_to_user_mode(regs);
670}
671
672static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)
673{
674 arm64_enter_from_user_mode(regs);
675 local_daif_restore(DAIF_PROCCTX);
676 do_el0_gcs(regs, esr);
677 arm64_exit_to_user_mode(regs);
678}
679
680static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
681{
682 arm64_enter_from_user_mode(regs);
683 local_daif_restore(DAIF_PROCCTX);
684 bad_el0_sync(regs, 0, esr);
685 arm64_exit_to_user_mode(regs);
686}
687
688static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
689{
690 if (!is_ttbr0_addr(regs->pc))
691 arm64_apply_bp_hardening();
692
693 arm64_enter_from_user_mode(regs);
694 debug_exception_enter(regs);
695 do_breakpoint(esr, regs);
696 debug_exception_exit(regs);
697 local_daif_restore(DAIF_PROCCTX);
698 arm64_exit_to_user_mode(regs);
699}
700
701static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
702{
703 bool step_done;
704
705 if (!is_ttbr0_addr(regs->pc))
706 arm64_apply_bp_hardening();
707
708 arm64_enter_from_user_mode(regs);
709 /*
710 * After handling a breakpoint, we suspend the breakpoint
711 * and use single-step to move to the next instruction.
712 * If we are stepping a suspended breakpoint there's nothing more to do:
713 * the single-step is complete.
714 */
715 step_done = try_step_suspended_breakpoints(regs);
716 local_daif_restore(DAIF_PROCCTX);
717 if (!step_done)
718 do_el0_softstep(esr, regs);
719 arm64_exit_to_user_mode(regs);
720}
721
722static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr)
723{
724 /* Watchpoints are the only debug exception to write FAR_EL1 */
725 unsigned long far = read_sysreg(far_el1);
726
727 arm64_enter_from_user_mode(regs);
728 debug_exception_enter(regs);
729 do_watchpoint(far, esr, regs);
730 debug_exception_exit(regs);
731 local_daif_restore(DAIF_PROCCTX);
732 arm64_exit_to_user_mode(regs);
733}
734
735static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
736{
737 arm64_enter_from_user_mode(regs);
738 local_daif_restore(DAIF_PROCCTX);
739 do_el0_brk64(esr, regs);
740 arm64_exit_to_user_mode(regs);
741}
742
743static void noinstr el0_svc(struct pt_regs *regs)
744{
745 arm64_enter_from_user_mode(regs);
746 cortex_a76_erratum_1463225_svc_handler();
747 fpsimd_syscall_enter();
748 local_daif_restore(DAIF_PROCCTX);
749 do_el0_svc(regs);
750 arm64_exit_to_user_mode(regs);
751 fpsimd_syscall_exit();
752}
753
754static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
755{
756 arm64_enter_from_user_mode(regs);
757 local_daif_restore(DAIF_PROCCTX);
758 do_el0_fpac(regs, esr);
759 arm64_exit_to_user_mode(regs);
760}
761
762asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
763{
764 unsigned long esr = read_sysreg(esr_el1);
765
766 switch (ESR_ELx_EC(esr)) {
767 case ESR_ELx_EC_SVC64:
768 el0_svc(regs);
769 break;
770 case ESR_ELx_EC_DABT_LOW:
771 el0_da(regs, esr);
772 break;
773 case ESR_ELx_EC_IABT_LOW:
774 el0_ia(regs, esr);
775 break;
776 case ESR_ELx_EC_FP_ASIMD:
777 el0_fpsimd_acc(regs, esr);
778 break;
779 case ESR_ELx_EC_SVE:
780 el0_sve_acc(regs, esr);
781 break;
782 case ESR_ELx_EC_SME:
783 el0_sme_acc(regs, esr);
784 break;
785 case ESR_ELx_EC_FP_EXC64:
786 el0_fpsimd_exc(regs, esr);
787 break;
788 case ESR_ELx_EC_SYS64:
789 case ESR_ELx_EC_WFx:
790 el0_sys(regs, esr);
791 break;
792 case ESR_ELx_EC_SP_ALIGN:
793 el0_sp(regs, esr);
794 break;
795 case ESR_ELx_EC_PC_ALIGN:
796 el0_pc(regs, esr);
797 break;
798 case ESR_ELx_EC_UNKNOWN:
799 el0_undef(regs, esr);
800 break;
801 case ESR_ELx_EC_BTI:
802 el0_bti(regs);
803 break;
804 case ESR_ELx_EC_MOPS:
805 el0_mops(regs, esr);
806 break;
807 case ESR_ELx_EC_GCS:
808 el0_gcs(regs, esr);
809 break;
810 case ESR_ELx_EC_BREAKPT_LOW:
811 el0_breakpt(regs, esr);
812 break;
813 case ESR_ELx_EC_SOFTSTP_LOW:
814 el0_softstp(regs, esr);
815 break;
816 case ESR_ELx_EC_WATCHPT_LOW:
817 el0_watchpt(regs, esr);
818 break;
819 case ESR_ELx_EC_BRK64:
820 el0_brk64(regs, esr);
821 break;
822 case ESR_ELx_EC_FPAC:
823 el0_fpac(regs, esr);
824 break;
825 default:
826 el0_inv(regs, esr);
827 }
828}
829
830static void noinstr el0_interrupt(struct pt_regs *regs,
831 void (*handler)(struct pt_regs *))
832{
833 arm64_enter_from_user_mode(regs);
834
835 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
836
837 if (regs->pc & BIT(55))
838 arm64_apply_bp_hardening();
839
840 irq_enter_rcu();
841 do_interrupt_handler(regs, handler);
842 irq_exit_rcu();
843
844 arm64_exit_to_user_mode(regs);
845}
846
847static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
848{
849 el0_interrupt(regs, handle_arch_irq);
850}
851
852asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
853{
854 __el0_irq_handler_common(regs);
855}
856
857static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
858{
859 el0_interrupt(regs, handle_arch_fiq);
860}
861
862asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
863{
864 __el0_fiq_handler_common(regs);
865}
866
867static void noinstr __el0_error_handler_common(struct pt_regs *regs)
868{
869 unsigned long esr = read_sysreg(esr_el1);
870 irqentry_state_t state;
871
872 arm64_enter_from_user_mode(regs);
873 local_daif_restore(DAIF_ERRCTX);
874 state = irqentry_nmi_enter(regs);
875 do_serror(regs, esr);
876 irqentry_nmi_exit(regs, state);
877 local_daif_restore(DAIF_PROCCTX);
878 arm64_exit_to_user_mode(regs);
879}
880
881asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
882{
883 __el0_error_handler_common(regs);
884}
885
886#ifdef CONFIG_COMPAT
887static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
888{
889 arm64_enter_from_user_mode(regs);
890 local_daif_restore(DAIF_PROCCTX);
891 do_el0_cp15(esr, regs);
892 arm64_exit_to_user_mode(regs);
893}
894
895static void noinstr el0_svc_compat(struct pt_regs *regs)
896{
897 arm64_enter_from_user_mode(regs);
898 cortex_a76_erratum_1463225_svc_handler();
899 local_daif_restore(DAIF_PROCCTX);
900 do_el0_svc_compat(regs);
901 arm64_exit_to_user_mode(regs);
902}
903
904static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
905{
906 arm64_enter_from_user_mode(regs);
907 local_daif_restore(DAIF_PROCCTX);
908 do_bkpt32(esr, regs);
909 arm64_exit_to_user_mode(regs);
910}
911
912asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
913{
914 unsigned long esr = read_sysreg(esr_el1);
915
916 switch (ESR_ELx_EC(esr)) {
917 case ESR_ELx_EC_SVC32:
918 el0_svc_compat(regs);
919 break;
920 case ESR_ELx_EC_DABT_LOW:
921 el0_da(regs, esr);
922 break;
923 case ESR_ELx_EC_IABT_LOW:
924 el0_ia(regs, esr);
925 break;
926 case ESR_ELx_EC_FP_ASIMD:
927 el0_fpsimd_acc(regs, esr);
928 break;
929 case ESR_ELx_EC_FP_EXC32:
930 el0_fpsimd_exc(regs, esr);
931 break;
932 case ESR_ELx_EC_PC_ALIGN:
933 el0_pc(regs, esr);
934 break;
935 case ESR_ELx_EC_UNKNOWN:
936 case ESR_ELx_EC_CP14_MR:
937 case ESR_ELx_EC_CP14_LS:
938 case ESR_ELx_EC_CP14_64:
939 el0_undef(regs, esr);
940 break;
941 case ESR_ELx_EC_CP15_32:
942 case ESR_ELx_EC_CP15_64:
943 el0_cp15(regs, esr);
944 break;
945 case ESR_ELx_EC_BREAKPT_LOW:
946 el0_breakpt(regs, esr);
947 break;
948 case ESR_ELx_EC_SOFTSTP_LOW:
949 el0_softstp(regs, esr);
950 break;
951 case ESR_ELx_EC_WATCHPT_LOW:
952 el0_watchpt(regs, esr);
953 break;
954 case ESR_ELx_EC_BKPT32:
955 el0_bkpt32(regs, esr);
956 break;
957 default:
958 el0_inv(regs, esr);
959 }
960}
961
962asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
963{
964 __el0_irq_handler_common(regs);
965}
966
967asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
968{
969 __el0_fiq_handler_common(regs);
970}
971
972asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
973{
974 __el0_error_handler_common(regs);
975}
976#else /* CONFIG_COMPAT */
977UNHANDLED(el0t, 32, sync)
978UNHANDLED(el0t, 32, irq)
979UNHANDLED(el0t, 32, fiq)
980UNHANDLED(el0t, 32, error)
981#endif /* CONFIG_COMPAT */
982
983asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
984{
985 unsigned long esr = read_sysreg(esr_el1);
986 unsigned long far = read_sysreg(far_el1);
987
988 irqentry_nmi_enter(regs);
989 panic_bad_stack(regs, esr, far);
990}
991
992#ifdef CONFIG_ARM_SDE_INTERFACE
993asmlinkage noinstr unsigned long
994__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
995{
996 irqentry_state_t state;
997 unsigned long ret;
998
999 /*
1000 * We didn't take an exception to get here, so the HW hasn't
1001 * set/cleared bits in PSTATE that we may rely on.
1002 *
1003 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
1004 * whether PSTATE bits are inherited unchanged or generated from
1005 * scratch, and the TF-A implementation always clears PAN and always
1006 * clears UAO. There are no other known implementations.
1007 *
1008 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
1009 * PSTATE is modified upon architectural exceptions, and so PAN is
1010 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
1011 * cleared.
1012 *
1013 * We must explicitly reset PAN to the expected state, including
1014 * clearing it when the host isn't using it, in case a VM had it set.
1015 */
1016 if (system_uses_hw_pan())
1017 set_pstate_pan(1);
1018 else if (cpu_has_pan())
1019 set_pstate_pan(0);
1020
1021 state = irqentry_nmi_enter(regs);
1022 ret = do_sdei_event(regs, arg);
1023 irqentry_nmi_exit(regs, state);
1024
1025 return ret;
1026}
1027#endif /* CONFIG_ARM_SDE_INTERFACE */