diff --git a/arch/alpha/kernel/irq.c b/arch/alpha/kernel/irq.c index c67047c5d830..4a6a8b1d5a8b 100644 --- a/arch/alpha/kernel/irq.c +++ b/arch/alpha/kernel/irq.c @@ -72,16 +72,16 @@ int arch_show_interrupts(struct seq_file *p, int prec) int j; #ifdef CONFIG_SMP - seq_puts(p, "IPI: "); + seq_puts(p, " IPI: "); for_each_online_cpu(j) seq_printf(p, "%10lu ", cpu_data[j].ipi_count); seq_putc(p, '\n'); #endif - seq_puts(p, "PMI: "); + seq_puts(p, " PMI: "); for_each_online_cpu(j) seq_printf(p, "%10lu ", per_cpu(irq_pmi_count, j)); - seq_puts(p, " Performance Monitoring\n"); - seq_printf(p, "ERR: %10lu\n", irq_err_count); + seq_puts(p, " Performance Monitoring\n"); + seq_printf(p, " ERR: %10lu\n", irq_err_count); return 0; } diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 4e8e89a26ca3..b5fb4697bc3f 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -551,8 +551,7 @@ void show_ipi_list(struct seq_file *p, int prec) if (!ipi_desc[i]) continue; - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, - prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u:", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 1aa324104afb..1d0e0e6a5b92 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -833,11 +833,10 @@ int arch_show_interrupts(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < MAX_IPI; i++) { - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, - prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu)); - seq_printf(p, " %s\n", ipi_types[i]); + seq_printf(p, " %s\n", ipi_types[i]); } seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 64a048f1b880..50922610758b 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -88,7 +88,7 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u:", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10); seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 5ed5095320e6..fa66f9c97d74 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -226,8 +226,7 @@ void show_ipi_stats(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < IPI_MAX; i++) { - seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, - prec >= 4 ? " " : ""); + seq_printf(p, "%*s%u:", prec - 1, "IPI", i); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); seq_printf(p, " %s\n", ipi_names[i]); diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index 9022d8af9d68..03c39b5da50f 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -46,7 +46,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) seq_printf(p, "%*s:", prec, "NMI"); for_each_online_cpu(j) seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat.__nmi_count, j), 10); - seq_printf(p, " Non-maskable interrupts\n"); + seq_printf(p, " Non-maskable interrupts\n"); seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c index 5210991429d5..22db727652ba 100644 --- a/arch/sparc/kernel/irq_32.c +++ b/arch/sparc/kernel/irq_32.c @@ -199,19 +199,19 @@ int arch_show_interrupts(struct seq_file *p, int prec) int j; #ifdef CONFIG_SMP - seq_printf(p, "RES:"); + seq_printf(p, "%*s:", prec, "RES"); for_each_online_cpu(j) seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_resched_count, 10); - seq_printf(p, " IPI rescheduling interrupts\n"); - seq_printf(p, "CAL:"); + seq_printf(p, " IPI rescheduling interrupts\n"); + seq_printf(p, "%*s:", prec, "CAL"); for_each_online_cpu(j) seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_call_count, 10); - seq_printf(p, " IPI function call interrupts\n"); + seq_printf(p, " IPI function call interrupts\n"); #endif - seq_printf(p, "NMI:"); + seq_printf(p, "%*s:", prec, "NMI"); for_each_online_cpu(j) seq_put_decimal_ull_width(p, " ", cpu_data(j).counter, 10); - seq_printf(p, " Non-maskable interrupts\n"); + seq_printf(p, " Non-maskable interrupts\n"); return 0; } diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index c5466a9fd560..3f55c69d5f3b 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -303,10 +303,10 @@ int arch_show_interrupts(struct seq_file *p, int prec) { int j; - seq_printf(p, "NMI:"); + seq_printf(p, "%*s:", prec, "NMI"); for_each_online_cpu(j) seq_put_decimal_ull_width(p, " ", cpu_data(j).__nmi_count, 10); - seq_printf(p, " Non-maskable interrupts\n"); + seq_printf(p, " Non-maskable interrupts\n"); return 0; } diff --git a/arch/um/kernel/irq.c b/arch/um/kernel/irq.c index 5929d498b65f..ddfd6e9bd8c7 100644 --- a/arch/um/kernel/irq.c +++ b/arch/um/kernel/irq.c @@ -716,12 +716,12 @@ int arch_show_interrupts(struct seq_file *p, int prec) seq_printf(p, "%*s: ", prec, "RES"); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_stats(cpu)->irq_resched_count); - seq_puts(p, " Rescheduling interrupts\n"); + seq_puts(p, " Rescheduling interrupts\n"); seq_printf(p, "%*s: ", prec, "CAL"); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_stats(cpu)->irq_call_count); - seq_puts(p, " Function call interrupts\n"); + seq_puts(p, " Function call interrupts\n"); #endif return 0; diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 0c92ed5f464b..305774b67995 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -1032,7 +1032,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) * Unmasking the LVTPC is not required as the Mask (M) bit of the LVT * PMI entry is not set by the local APIC when a PMC overflow occurs */ - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); done: cpuc->enabled = pmu_enabled; diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index e0bd5051db2a..ed18a6d7e1a8 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1600,7 +1600,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) handled += perf_ibs_handle_irq(&perf_ibs_op, regs); if (handled) - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); perf_sample_event_took(sched_clock() - stamp); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 4b9e105309c6..d1af33d96d0a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1753,7 +1753,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) } if (handled) - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); return handled; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index dd1e3aa75ee9..1b5ec78076ae 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3531,7 +3531,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) int bit; int handled = 0; - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); /* * Ignore a range of extra bits in status that do not indicate diff --git a/arch/x86/events/intel/knc.c b/arch/x86/events/intel/knc.c index e614baf42926..e887adc108ac 100644 --- a/arch/x86/events/intel/knc.c +++ b/arch/x86/events/intel/knc.c @@ -238,7 +238,7 @@ static int knc_pmu_handle_irq(struct pt_regs *regs) goto done; } - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) { struct perf_event *event = cpuc->events[bit]; diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index 02bfdb77158b..5368dc31787c 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -1077,7 +1077,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) } if (handled) - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); /* * When dealing with the unmasking of the LVTPC on P4 perf hw, it has diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index 4bdfcf091200..4bc177badac2 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -373,7 +373,7 @@ static int zhaoxin_pmu_handle_irq(struct pt_regs *regs) else zhaoxin_pmu_ack_status(status); - inc_irq_stat(apic_perf_irqs); + inc_perf_irq_stat(); /* * CondChgd bit 63 doesn't mean any overflow status. Ignore diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 323adc93f2dc..55a8b6de2865 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -219,7 +219,7 @@ static inline bool hv_reenlightenment_available(void) DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment) { apic_eoi(); - inc_irq_stat(irq_hv_reenlightenment_count); + inc_irq_stat(HYPERV_REENLIGHTENMENT); schedule_delayed_work(&hv_reenlightenment_work, HZ/10); } diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h index 9314642ae93c..dea60d66d976 100644 --- a/arch/x86/include/asm/hardirq.h +++ b/arch/x86/include/asm/hardirq.h @@ -4,51 +4,64 @@ #include +enum irq_stat_counts { + IRQ_COUNT_NMI, +#ifdef CONFIG_X86_LOCAL_APIC + IRQ_COUNT_APIC_TIMER, + IRQ_COUNT_SPURIOUS, + IRQ_COUNT_APIC_PERF, + IRQ_COUNT_IRQ_WORK, + IRQ_COUNT_ICR_READ_RETRY, + IRQ_COUNT_X86_PLATFORM_IPI, +#endif +#ifdef CONFIG_SMP + IRQ_COUNT_RESCHEDULE, + IRQ_COUNT_CALL_FUNCTION, +#endif + IRQ_COUNT_TLB, +#ifdef CONFIG_X86_THERMAL_VECTOR + IRQ_COUNT_THERMAL_APIC, +#endif +#ifdef CONFIG_X86_MCE_THRESHOLD + IRQ_COUNT_THRESHOLD_APIC, +#endif +#ifdef CONFIG_X86_MCE_AMD + IRQ_COUNT_DEFERRED_ERROR, +#endif +#ifdef CONFIG_X86_MCE + IRQ_COUNT_MCE_EXCEPTION, + IRQ_COUNT_MCE_POLL, +#endif +#ifdef CONFIG_X86_HV_CALLBACK_VECTOR + IRQ_COUNT_HYPERVISOR_CALLBACK, +#endif +#if IS_ENABLED(CONFIG_HYPERV) + IRQ_COUNT_HYPERV_REENLIGHTENMENT, + IRQ_COUNT_HYPERV_STIMER0, +#endif +#if IS_ENABLED(CONFIG_KVM) + IRQ_COUNT_POSTED_INTR, + IRQ_COUNT_POSTED_INTR_NESTED, + IRQ_COUNT_POSTED_INTR_WAKEUP, +#endif +#ifdef CONFIG_GUEST_PERF_EVENTS + IRQ_COUNT_PERF_GUEST_MEDIATED_PMI, +#endif +#ifdef CONFIG_X86_POSTED_MSI + IRQ_COUNT_POSTED_MSI_NOTIFICATION, +#endif + IRQ_COUNT_PIC_APIC_ERROR, +#ifdef CONFIG_X86_IO_APIC + IRQ_COUNT_IOAPIC_MISROUTED, +#endif + IRQ_COUNT_MAX, +}; + typedef struct { #if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL) u8 kvm_cpu_l1tf_flush_l1d; #endif - unsigned int __nmi_count; /* arch dependent */ -#ifdef CONFIG_X86_LOCAL_APIC - unsigned int apic_timer_irqs; /* arch dependent */ - unsigned int irq_spurious_count; - unsigned int icr_read_retry_count; -#endif -#if IS_ENABLED(CONFIG_KVM) - unsigned int kvm_posted_intr_ipis; - unsigned int kvm_posted_intr_wakeup_ipis; - unsigned int kvm_posted_intr_nested_ipis; -#endif -#ifdef CONFIG_GUEST_PERF_EVENTS - unsigned int perf_guest_mediated_pmis; -#endif - unsigned int x86_platform_ipis; /* arch dependent */ - unsigned int apic_perf_irqs; - unsigned int apic_irq_work_irqs; -#ifdef CONFIG_SMP - unsigned int irq_resched_count; - unsigned int irq_call_count; -#endif - unsigned int irq_tlb_count; -#ifdef CONFIG_X86_THERMAL_VECTOR - unsigned int irq_thermal_count; -#endif -#ifdef CONFIG_X86_MCE_THRESHOLD - unsigned int irq_threshold_count; -#endif -#ifdef CONFIG_X86_MCE_AMD - unsigned int irq_deferred_error_count; -#endif -#ifdef CONFIG_X86_HV_CALLBACK_VECTOR - unsigned int irq_hv_callback_count; -#endif -#if IS_ENABLED(CONFIG_HYPERV) - unsigned int irq_hv_reenlightenment_count; - unsigned int hyperv_stimer0_count; -#endif -#ifdef CONFIG_X86_POSTED_MSI - unsigned int posted_msi_notification_count; -#endif + unsigned int counts[IRQ_COUNT_MAX]; } ____cacheline_aligned irq_cpustat_t; DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); @@ -58,15 +71,21 @@ DECLARE_PER_CPU_ALIGNED(struct pi_desc, posted_msi_pi_desc); #endif #define __ARCH_IRQ_STAT -#define inc_irq_stat(member) this_cpu_inc(irq_stat.member) +#define inc_irq_stat(index) this_cpu_inc(irq_stat.counts[IRQ_COUNT_##index]) +void irq_stat_inc_and_enable(enum irq_stat_counts which); + +#ifdef CONFIG_X86_LOCAL_APIC +#define inc_perf_irq_stat() inc_irq_stat(APIC_PERF) +#else +#define inc_perf_irq_stat() do { } while (0) +#endif extern void ack_bad_irq(unsigned int irq); +#ifdef CONFIG_PROC_FS extern u64 arch_irq_stat_cpu(unsigned int cpu); #define arch_irq_stat_cpu arch_irq_stat_cpu - -extern u64 arch_irq_stat(void); -#define arch_irq_stat arch_irq_stat +#endif DECLARE_PER_CPU_CACHE_HOT(u16, __softirq_pending); #define local_softirq_pending_ref __softirq_pending diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index cbe19e669080..47727d0b540b 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -110,10 +110,6 @@ static inline void lock_vector_lock(void) {} static inline void unlock_vector_lock(void) {} #endif -/* Statistics */ -extern atomic_t irq_err_count; -extern atomic_t irq_mis_count; - extern void elcr_set_level_irq(unsigned int irq); extern char irq_entries_start[]; diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 0175d39a5856..e575b702063d 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -291,9 +291,6 @@ bool mce_is_memory_error(struct mce *m); bool mce_is_correctable(struct mce *m); bool mce_usable_address(struct mce *m); -DECLARE_PER_CPU(unsigned, mce_exception_count); -DECLARE_PER_CPU(unsigned, mce_poll_count); - typedef DECLARE_BITMAP(mce_banks_t, MAX_NR_BANKS); DECLARE_PER_CPU(mce_banks_t, mce_poll_banks); diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 639904911444..3eeebc2c5a1d 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1045,7 +1045,7 @@ static void local_apic_timer_interrupt(void) /* * the NMI deadlock-detector uses this. */ - inc_irq_stat(apic_timer_irqs); + inc_irq_stat(APIC_TIMER); evt->event_handler(evt); } @@ -2114,7 +2114,7 @@ static noinline void handle_spurious_interrupt(u8 vector) trace_spurious_apic_entry(vector); - inc_irq_stat(irq_spurious_count); + irq_stat_inc_and_enable(IRQ_COUNT_SPURIOUS); /* * If this is a spurious interrupt then do not acknowledge @@ -2186,7 +2186,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) apic_write(APIC_ESR, 0); v = apic_read(APIC_ESR); apic_eoi(); - atomic_inc(&irq_err_count); + irq_stat_inc_and_enable(IRQ_COUNT_PIC_APIC_ERROR); apic_pr_debug("APIC error on CPU%d: %02x", smp_processor_id(), v); diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 352ed5558cbc..7d7175d01228 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1575,8 +1575,6 @@ static unsigned int startup_ioapic_irq(struct irq_data *data) return was_pending; } -atomic_t irq_mis_count; - #ifdef CONFIG_GENERIC_PENDING_IRQ static bool io_apic_level_ack_pending(struct mp_chip_data *data) { @@ -1713,7 +1711,7 @@ static void ioapic_ack_level(struct irq_data *irq_data) * at the cpu. */ if (!(v & (1 << (i & 0x1f)))) { - atomic_inc(&irq_mis_count); + irq_stat_inc_and_enable(IRQ_COUNT_IOAPIC_MISROUTED); eoi_ioapic_pin(cfg->vector, irq_data->chip_data); } diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c index 98a57cb4aa86..c627bee3b14f 100644 --- a/arch/x86/kernel/apic/ipi.c +++ b/arch/x86/kernel/apic/ipi.c @@ -120,7 +120,7 @@ u32 apic_mem_wait_icr_idle_timeout(void) for (cnt = 0; cnt < 1000; cnt++) { if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY)) return 0; - inc_irq_stat(icr_read_retry_count); + irq_stat_inc_and_enable(IRQ_COUNT_ICR_READ_RETRY); udelay(100); } return APIC_ICR_BUSY; diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 2c5b51aad91a..dc119af83524 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -52,7 +52,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) * HYPERVISOR_CALLBACK_VECTOR. */ apic_eoi(); - inc_irq_stat(irq_hv_callback_count); + inc_irq_stat(HYPERVISOR_CALLBACK); if (acrn_intr_handler) acrn_intr_handler(); diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 6605a0224659..222fa9cb181b 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -850,7 +850,7 @@ bool amd_mce_usable_address(struct mce *m) DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error) { trace_deferred_error_apic_entry(DEFERRED_ERROR_VECTOR); - inc_irq_stat(irq_deferred_error_count); + inc_irq_stat(DEFERRED_ERROR); deferred_error_int_vector(); trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR); apic_eoi(); diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index f3a793e3a6c8..3c1faface683 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -67,8 +67,6 @@ static DEFINE_MUTEX(mce_sysfs_mutex); #define SPINUNIT 100 /* 100ns */ -DEFINE_PER_CPU(unsigned, mce_exception_count); - DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); @@ -693,8 +691,6 @@ static noinstr void mce_read_aux(struct mce_hw_err *err, int i) } } -DEFINE_PER_CPU(unsigned, mce_poll_count); - /* * We have three scenarios for checking for Deferred errors: * @@ -797,7 +793,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b) struct mce *m; int i; - this_cpu_inc(mce_poll_count); + inc_irq_stat(MCE_POLL); mce_gather_info(&err, NULL); m = &err.m; @@ -1572,7 +1568,7 @@ noinstr void do_machine_check(struct pt_regs *regs) */ lmce = 1; - this_cpu_inc(mce_exception_count); + inc_irq_stat(MCE_EXCEPTION); mce_gather_info(&err, regs); m = &err.m; diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c index 0d13c9ffcba0..6c370d5af5bd 100644 --- a/arch/x86/kernel/cpu/mce/threshold.c +++ b/arch/x86/kernel/cpu/mce/threshold.c @@ -37,7 +37,7 @@ void (*mce_threshold_vector)(void) = default_threshold_interrupt; DEFINE_IDTENTRY_SYSVEC(sysvec_threshold) { trace_threshold_apic_entry(THRESHOLD_APIC_VECTOR); - inc_irq_stat(irq_threshold_count); + inc_irq_stat(THRESHOLD_APIC); mce_threshold_vector(); trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR); apic_eoi(); diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index b5b6a58b67b0..9381102e884a 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -154,7 +154,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback) { struct pt_regs *old_regs = set_irq_regs(regs); - inc_irq_stat(irq_hv_callback_count); + inc_irq_stat(HYPERVISOR_CALLBACK); if (mshv_handler) mshv_handler(); @@ -193,7 +193,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) { struct pt_regs *old_regs = set_irq_regs(regs); - inc_irq_stat(hyperv_stimer0_count); + inc_irq_stat(HYPERV_STIMER0); if (hv_stimer0_handler) hv_stimer0_handler(); add_interrupt_randomness(HYPERV_STIMER0_VECTOR); diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c index f67063df6723..f7a86b94a0dd 100644 --- a/arch/x86/kernel/i8259.c +++ b/arch/x86/kernel/i8259.c @@ -214,7 +214,7 @@ static void mask_and_ack_8259A(struct irq_data *data) "spurious 8259A interrupt: IRQ%d.\n", irq); spurious_irq_mask |= irqmask; } - atomic_inc(&irq_err_count); + irq_stat_inc_and_enable(IRQ_COUNT_PIC_APIC_ERROR); /* * Theoretically we do not have to handle this IRQ, * but in Linux this does not cause problems and is diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index ec77be217eaf..30122f0b3af9 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -39,8 +39,6 @@ EXPORT_PER_CPU_SYMBOL(__softirq_pending); DEFINE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr); -atomic_t irq_err_count; - /* * 'what should we do if we get a hw irq event on an illegal vector'. * each architecture has to answer this themselves. @@ -62,150 +60,131 @@ void ack_bad_irq(unsigned int irq) apic_eoi(); } -#define irq_stats(x) (&per_cpu(irq_stat, x)) +struct irq_stat_info { + unsigned int skip_vector; + const char *symbol; + const char *text; +}; + +#define DEFAULT_SUPPRESSED_VECTOR UINT_MAX + +#define ISS(idx, sym, txt) [IRQ_COUNT_##idx] = { .symbol = sym, .text = txt } + +#define ITS(idx, sym, txt) [IRQ_COUNT_##idx] = \ + { .skip_vector = idx## _VECTOR, .symbol = sym, .text = txt } + +#define IDS(idx, sym, txt) [IRQ_COUNT_##idx] = \ + { .skip_vector = DEFAULT_SUPPRESSED_VECTOR, .symbol = sym, .text = txt } + +static const struct irq_stat_info irq_stat_info[IRQ_COUNT_MAX] = { + ISS(NMI, "NMI", " Non-maskable interrupts\n"), +#ifdef CONFIG_X86_LOCAL_APIC + ISS(APIC_TIMER, "LOC", " Local timer interrupts\n"), + IDS(SPURIOUS, "SPU", " Spurious interrupts\n"), + ISS(APIC_PERF, "PMI", " Performance monitoring interrupts\n"), + ISS(IRQ_WORK, "IWI", " IRQ work interrupts\n"), + IDS(ICR_READ_RETRY, "RTR", " APIC ICR read retries\n"), + ISS(X86_PLATFORM_IPI, "PLT", " Platform interrupts\n"), +#endif +#ifdef CONFIG_SMP + ISS(RESCHEDULE, "RES", " Rescheduling interrupts\n"), + ISS(CALL_FUNCTION, "CAL", " Function call interrupts\n"), +#endif + ISS(TLB, "TLB", " TLB shootdowns\n"), +#ifdef CONFIG_X86_THERMAL_VECTOR + ISS(THERMAL_APIC, "TRM", " Thermal event interrupts\n"), +#endif +#ifdef CONFIG_X86_MCE_THRESHOLD + ISS(THRESHOLD_APIC, "THR", " Threshold APIC interrupts\n"), +#endif +#ifdef CONFIG_X86_MCE_AMD + ISS(DEFERRED_ERROR, "DFR", " Deferred Error APIC interrupts\n"), +#endif +#ifdef CONFIG_X86_MCE + ISS(MCE_EXCEPTION, "MCE", " Machine check exceptions\n"), + ISS(MCE_POLL, "MCP", " Machine check polls\n"), +#endif +#ifdef CONFIG_X86_HV_CALLBACK_VECTOR + ITS(HYPERVISOR_CALLBACK, "HYP", " Hypervisor callback interrupts\n"), +#endif +#if IS_ENABLED(CONFIG_HYPERV) + ITS(HYPERV_REENLIGHTENMENT, "HRE", " Hyper-V reenlightenment interrupts\n"), + ITS(HYPERV_STIMER0, "HVS", " Hyper-V stimer0 interrupts\n"), +#endif +#if IS_ENABLED(CONFIG_KVM) + ITS(POSTED_INTR, "PIN", " Posted-interrupt notification event\n"), + ITS(POSTED_INTR_NESTED, "NPI", " Nested posted-interrupt event\n"), + ITS(POSTED_INTR_WAKEUP, "PIW", " Posted-interrupt wakeup event\n"), +#endif +#ifdef CONFIG_GUEST_PERF_EVENTS + ISS(PERF_GUEST_MEDIATED_PMI, "VPMI", " Perf Guest Mediated PMI\n"), +#endif +#ifdef CONFIG_X86_POSTED_MSI + ISS(POSTED_MSI_NOTIFICATION, "PMN", " Posted MSI notification event\n"), +#endif + IDS(PIC_APIC_ERROR, "ERR", " PIC/APIC error interrupts\n"), +#ifdef CONFIG_X86_IO_APIC + IDS(IOAPIC_MISROUTED, "MIS", " Misrouted IO/APIC interrupts\n"), +#endif +}; + +static DECLARE_BITMAP(irq_stat_count_show, IRQ_COUNT_MAX) __read_mostly; + +static int __init irq_init_stats(void) +{ + const struct irq_stat_info *info = irq_stat_info; + + for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) { + if (!info->skip_vector || (info->skip_vector != DEFAULT_SUPPRESSED_VECTOR && + test_bit(info->skip_vector, system_vectors))) + set_bit(i, irq_stat_count_show); + } + +#ifdef CONFIG_X86_LOCAL_APIC + if (!x86_platform_ipi_callback) + clear_bit(IRQ_COUNT_X86_PLATFORM_IPI, irq_stat_count_show); +#endif + +#ifdef CONFIG_X86_POSTED_MSI + if (!posted_msi_enabled()) + clear_bit(IRQ_COUNT_POSTED_MSI_NOTIFICATION, irq_stat_count_show); +#endif + +#ifdef CONFIG_X86_MCE_AMD + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && + boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) + clear_bit(IRQ_COUNT_DEFERRED_ERROR, irq_stat_count_show); +#endif + return 0; +} +late_initcall(irq_init_stats); + +/* + * Used for default disabled counters to increment the stats and to enable the + * entry for /proc/interrupts output. + */ +void irq_stat_inc_and_enable(enum irq_stat_counts which) +{ + this_cpu_inc(irq_stat.counts[which]); + set_bit(which, irq_stat_count_show); +} + +#ifdef CONFIG_PROC_FS /* * /proc/interrupts printing for arch specific interrupts */ int arch_show_interrupts(struct seq_file *p, int prec) { - int j; + const struct irq_stat_info *info = irq_stat_info; - seq_printf(p, "%*s: ", prec, "NMI"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->__nmi_count); - seq_puts(p, " Non-maskable interrupts\n"); -#ifdef CONFIG_X86_LOCAL_APIC - seq_printf(p, "%*s: ", prec, "LOC"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs); - seq_puts(p, " Local timer interrupts\n"); + for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) { + if (!test_bit(i, irq_stat_count_show)) + continue; - seq_printf(p, "%*s: ", prec, "SPU"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count); - seq_puts(p, " Spurious interrupts\n"); - seq_printf(p, "%*s: ", prec, "PMI"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs); - seq_puts(p, " Performance monitoring interrupts\n"); - seq_printf(p, "%*s: ", prec, "IWI"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs); - seq_puts(p, " IRQ work interrupts\n"); - seq_printf(p, "%*s: ", prec, "RTR"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count); - seq_puts(p, " APIC ICR read retries\n"); - if (x86_platform_ipi_callback) { - seq_printf(p, "%*s: ", prec, "PLT"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis); - seq_puts(p, " Platform interrupts\n"); + seq_printf(p, "%*s:", prec, info->symbol); + irq_proc_emit_counts(p, &irq_stat.counts[i]); + seq_puts(p, info->text); } -#endif -#ifdef CONFIG_SMP - seq_printf(p, "%*s: ", prec, "RES"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count); - seq_puts(p, " Rescheduling interrupts\n"); - seq_printf(p, "%*s: ", prec, "CAL"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_call_count); - seq_puts(p, " Function call interrupts\n"); - seq_printf(p, "%*s: ", prec, "TLB"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count); - seq_puts(p, " TLB shootdowns\n"); -#endif -#ifdef CONFIG_X86_THERMAL_VECTOR - seq_printf(p, "%*s: ", prec, "TRM"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count); - seq_puts(p, " Thermal event interrupts\n"); -#endif -#ifdef CONFIG_X86_MCE_THRESHOLD - seq_printf(p, "%*s: ", prec, "THR"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count); - seq_puts(p, " Threshold APIC interrupts\n"); -#endif -#ifdef CONFIG_X86_MCE_AMD - seq_printf(p, "%*s: ", prec, "DFR"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count); - seq_puts(p, " Deferred Error APIC interrupts\n"); -#endif -#ifdef CONFIG_X86_MCE - seq_printf(p, "%*s: ", prec, "MCE"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", per_cpu(mce_exception_count, j)); - seq_puts(p, " Machine check exceptions\n"); - seq_printf(p, "%*s: ", prec, "MCP"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", per_cpu(mce_poll_count, j)); - seq_puts(p, " Machine check polls\n"); -#endif -#ifdef CONFIG_X86_HV_CALLBACK_VECTOR - if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) { - seq_printf(p, "%*s: ", prec, "HYP"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->irq_hv_callback_count); - seq_puts(p, " Hypervisor callback interrupts\n"); - } -#endif -#if IS_ENABLED(CONFIG_HYPERV) - if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) { - seq_printf(p, "%*s: ", prec, "HRE"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->irq_hv_reenlightenment_count); - seq_puts(p, " Hyper-V reenlightenment interrupts\n"); - } - if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) { - seq_printf(p, "%*s: ", prec, "HVS"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->hyperv_stimer0_count); - seq_puts(p, " Hyper-V stimer0 interrupts\n"); - } -#endif - seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); -#if defined(CONFIG_X86_IO_APIC) - seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count)); -#endif -#if IS_ENABLED(CONFIG_KVM) - seq_printf(p, "%*s: ", prec, "PIN"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis); - seq_puts(p, " Posted-interrupt notification event\n"); - - seq_printf(p, "%*s: ", prec, "NPI"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->kvm_posted_intr_nested_ipis); - seq_puts(p, " Nested posted-interrupt event\n"); - - seq_printf(p, "%*s: ", prec, "PIW"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->kvm_posted_intr_wakeup_ipis); - seq_puts(p, " Posted-interrupt wakeup event\n"); -#endif -#ifdef CONFIG_GUEST_PERF_EVENTS - seq_printf(p, "%*s: ", prec, "VPMI"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->perf_guest_mediated_pmis); - seq_puts(p, " Perf Guest Mediated PMI\n"); -#endif -#ifdef CONFIG_X86_POSTED_MSI - seq_printf(p, "%*s: ", prec, "PMN"); - for_each_online_cpu(j) - seq_printf(p, "%10u ", - irq_stats(j)->posted_msi_notification_count); - seq_puts(p, " Posted MSI notification event\n"); -#endif return 0; } @@ -214,46 +193,14 @@ int arch_show_interrupts(struct seq_file *p, int prec) */ u64 arch_irq_stat_cpu(unsigned int cpu) { - u64 sum = irq_stats(cpu)->__nmi_count; + irq_cpustat_t *p = per_cpu_ptr(&irq_stat, cpu); + u64 sum = 0; -#ifdef CONFIG_X86_LOCAL_APIC - sum += irq_stats(cpu)->apic_timer_irqs; - sum += irq_stats(cpu)->irq_spurious_count; - sum += irq_stats(cpu)->apic_perf_irqs; - sum += irq_stats(cpu)->apic_irq_work_irqs; - sum += irq_stats(cpu)->icr_read_retry_count; - if (x86_platform_ipi_callback) - sum += irq_stats(cpu)->x86_platform_ipis; -#endif -#ifdef CONFIG_SMP - sum += irq_stats(cpu)->irq_resched_count; - sum += irq_stats(cpu)->irq_call_count; -#endif -#ifdef CONFIG_X86_THERMAL_VECTOR - sum += irq_stats(cpu)->irq_thermal_count; -#endif -#ifdef CONFIG_X86_MCE_THRESHOLD - sum += irq_stats(cpu)->irq_threshold_count; -#endif -#ifdef CONFIG_X86_HV_CALLBACK_VECTOR - sum += irq_stats(cpu)->irq_hv_callback_count; -#endif -#if IS_ENABLED(CONFIG_HYPERV) - sum += irq_stats(cpu)->irq_hv_reenlightenment_count; - sum += irq_stats(cpu)->hyperv_stimer0_count; -#endif -#ifdef CONFIG_X86_MCE - sum += per_cpu(mce_exception_count, cpu); - sum += per_cpu(mce_poll_count, cpu); -#endif - return sum; -} - -u64 arch_irq_stat(void) -{ - u64 sum = atomic_read(&irq_err_count); + for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++) + sum += p->counts[i]; return sum; } +#endif /* CONFIG_PROC_FS */ static __always_inline void handle_irq(struct irq_desc *desc, struct pt_regs *regs) @@ -338,7 +285,7 @@ DEFINE_IDTENTRY_IRQ(common_interrupt) #ifdef CONFIG_X86_LOCAL_APIC /* Function pointer for generic interrupt vector handling */ -void (*x86_platform_ipi_callback)(void) = NULL; +void (*x86_platform_ipi_callback)(void) __ro_after_init = NULL; /* * Handler for X86_PLATFORM_IPI_VECTOR. */ @@ -348,7 +295,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) apic_eoi(); trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); - inc_irq_stat(x86_platform_ipis); + inc_irq_stat(X86_PLATFORM_IPI); if (x86_platform_ipi_callback) x86_platform_ipi_callback(); trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); @@ -363,7 +310,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) DEFINE_IDTENTRY_SYSVEC(sysvec_perf_guest_mediated_pmi_handler) { apic_eoi(); - inc_irq_stat(perf_guest_mediated_pmis); + inc_irq_stat(PERF_GUEST_MEDIATED_PMI); perf_guest_handle_mediated_pmi(); } #endif @@ -389,7 +336,7 @@ EXPORT_SYMBOL_FOR_KVM(kvm_set_posted_intr_wakeup_handler); DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) { apic_eoi(); - inc_irq_stat(kvm_posted_intr_ipis); + inc_irq_stat(POSTED_INTR); } /* @@ -398,7 +345,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) { apic_eoi(); - inc_irq_stat(kvm_posted_intr_wakeup_ipis); + inc_irq_stat(POSTED_INTR_WAKEUP); kvm_posted_intr_wakeup_handler(); } @@ -408,7 +355,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) { apic_eoi(); - inc_irq_stat(kvm_posted_intr_nested_ipis); + inc_irq_stat(POSTED_INTR_NESTED); } #endif @@ -482,7 +429,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification) /* Mark the handler active for intel_ack_posted_msi_irq() */ __this_cpu_write(posted_msi_handler_active, true); - inc_irq_stat(posted_msi_notification_count); + inc_irq_stat(POSTED_MSI_NOTIFICATION); irq_enter(); /* @@ -577,7 +524,7 @@ static void smp_thermal_vector(void) DEFINE_IDTENTRY_SYSVEC(sysvec_thermal) { trace_thermal_apic_entry(THERMAL_APIC_VECTOR); - inc_irq_stat(irq_thermal_count); + inc_irq_stat(THERMAL_APIC); smp_thermal_vector(); trace_thermal_apic_exit(THERMAL_APIC_VECTOR); apic_eoi(); diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index b0a24deab4a1..308c62411ff4 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c @@ -18,7 +18,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work) { apic_eoi(); trace_irq_work_entry(IRQ_WORK_VECTOR); - inc_irq_stat(apic_irq_work_irqs); + inc_irq_stat(IRQ_WORK); irq_work_run(); trace_irq_work_exit(IRQ_WORK_VECTOR); } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 29226d112029..d1f3f320168c 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -304,7 +304,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) apic_eoi(); - inc_irq_stat(irq_hv_callback_count); + inc_irq_stat(HYPERVISOR_CALLBACK); if (__this_cpu_read(async_pf_enabled)) { token = __this_cpu_read(apf_reason.token); diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index 52a3afb1b79e..3c9f60d6ca5a 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -576,7 +576,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi) irq_state = irqentry_nmi_enter(regs); - inc_irq_stat(__nmi_count); + inc_irq_stat(NMI); if (IS_ENABLED(CONFIG_NMI_CHECK_CPU) && ignore_nmis) { WRITE_ONCE(nsp->idt_ignored, nsp->idt_ignored + 1); @@ -724,7 +724,7 @@ DEFINE_FREDENTRY_NMI(exc_nmi) irq_state = irqentry_nmi_enter(regs); - inc_irq_stat(__nmi_count); + inc_irq_stat(NMI); default_do_nmi(regs); irqentry_nmi_exit(regs, irq_state); diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index cbf95fe2b207..985103cab16c 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -250,7 +250,7 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi) { apic_eoi(); trace_reschedule_entry(RESCHEDULE_VECTOR); - inc_irq_stat(irq_resched_count); + inc_irq_stat(RESCHEDULE); scheduler_ipi(); trace_reschedule_exit(RESCHEDULE_VECTOR); } @@ -259,7 +259,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function) { apic_eoi(); trace_call_function_entry(CALL_FUNCTION_VECTOR); - inc_irq_stat(irq_call_count); + inc_irq_stat(CALL_FUNCTION); generic_smp_call_function_interrupt(); trace_call_function_exit(CALL_FUNCTION_VECTOR); } @@ -268,7 +268,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single) { apic_eoi(); trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); - inc_irq_stat(irq_call_count); + inc_irq_stat(CALL_FUNCTION); generic_smp_call_function_single_interrupt(); trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR); } diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index af43d177087e..4c045f844492 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1123,7 +1123,7 @@ static void flush_tlb_func(void *info) VM_WARN_ON(!irqs_disabled()); if (!local) { - inc_irq_stat(irq_tlb_count); + inc_irq_stat(TLB); count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); } diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 2f9fa27e5a3c..6c4eac4ff13a 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -125,7 +125,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback) if (xen_percpu_upcall) apic_eoi(); - inc_irq_stat(irq_hv_callback_count); + inc_irq_stat(HYPERVISOR_CALLBACK); xen_evtchn_do_upcall(); diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index b310b91f2994..54a714077eee 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -722,7 +722,7 @@ static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); - inc_irq_stat(irq_hv_callback_count); + inc_irq_stat(HYPERVISOR_CALLBACK); xen_evtchn_do_upcall(); diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 05f92c812ac8..05ee0d3b0874 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -23,7 +23,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); */ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) { - inc_irq_stat(irq_resched_count); + inc_irq_stat(RESCHEDULE); scheduler_ipi(); return IRQ_HANDLED; @@ -254,7 +254,7 @@ void xen_send_IPI_allbutself(int vector) static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) { generic_smp_call_function_interrupt(); - inc_irq_stat(irq_call_count); + inc_irq_stat(CALL_FUNCTION); return IRQ_HANDLED; } @@ -262,7 +262,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) { generic_smp_call_function_single_interrupt(); - inc_irq_stat(irq_call_count); + inc_irq_stat(CALL_FUNCTION); return IRQ_HANDLED; } diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index db9b8e222b38..c2812f8177bb 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -400,7 +400,7 @@ static void xen_pv_stop_other_cpus(int wait) static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id) { irq_work_run(); - inc_irq_stat(apic_irq_work_irqs); + inc_irq_stat(IRQ_WORK); return IRQ_HANDLED; } diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c index b1e410f6b5ab..6f01f530868b 100644 --- a/arch/xtensa/kernel/irq.c +++ b/arch/xtensa/kernel/irq.c @@ -59,7 +59,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) seq_printf(p, "%*s:", prec, "NMI"); for_each_online_cpu(cpu) seq_printf(p, " %10lu", per_cpu(nmi_count, cpu)); - seq_puts(p, " Non-maskable interrupts\n"); + seq_puts(p, " Non-maskable interrupts\n"); #endif return 0; } diff --git a/fs/proc/Makefile b/fs/proc/Makefile index 7b4db9c56e6a..8bc615ff84e5 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -16,7 +16,9 @@ proc-y += cmdline.o proc-y += consoles.o proc-y += cpuinfo.o proc-y += devices.o -proc-y += interrupts.o +ifneq ($(CONFIG_GENERIC_IRQ_SHOW),y) +proc-y += interrupts.o +endif proc-y += loadavg.o proc-y += meminfo.o proc-y += stat.o diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 8b444e862319..20c3df9a9b80 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -18,9 +18,6 @@ #ifndef arch_irq_stat_cpu #define arch_irq_stat_cpu(cpu) 0 #endif -#ifndef arch_irq_stat -#define arch_irq_stat() 0 -#endif u64 get_idle_time(struct kernel_cpustat *kcs, int cpu) { @@ -122,7 +119,6 @@ static int show_stat(struct seq_file *p, void *v) sum_softirq += softirq_stat; } } - sum += arch_irq_stat(); seq_put_decimal_ull(p, "cpu ", nsec_to_clock_t(user)); seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 6cd26ffb0505..3bf969ad8fe0 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -864,6 +864,7 @@ static inline void init_irq_proc(void) struct seq_file; int show_interrupts(struct seq_file *p, void *v); int arch_show_interrupts(struct seq_file *p, int prec); +void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts); extern int early_irq_init(void); extern int arch_probe_nr_irqs(void); diff --git a/include/linux/irq.h b/include/linux/irq.h index efa514ee562f..f485369b1b4f 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -103,6 +103,7 @@ enum { IRQ_DISABLE_UNLAZY = (1 << 19), IRQ_HIDDEN = (1 << 20), IRQ_NO_DEBUG = (1 << 21), + IRQ_RESERVED = (1 << 22), }; #define IRQF_MODIFY_MASK \ diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index dae9a9b93665..8080db17c1b1 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -52,8 +52,8 @@ struct irq_redirect { * @depth: disable-depth, for nested irq_disable() calls * @wake_depth: enable depth, for multiple irq_set_irq_wake() callers * @tot_count: stats field for non-percpu irqs - * @irq_count: stats field to detect stalled irqs * @last_unhandled: aging timer for unhandled count + * @irq_count: stats field to detect stalled irqs * @irqs_unhandled: stats field for spurious unhandled interrupts * @threads_handled: stats field for deferred spurious detection of threaded handlers * @threads_handled_last: comparator field for deferred spurious detection of threaded handlers @@ -70,6 +70,7 @@ struct irq_redirect { * IRQF_NO_SUSPEND set * @force_resume_depth: number of irqactions on a irq descriptor with * IRQF_FORCE_RESUME set + * @refcnt: Reference count mainly for /proc/interrupts * @rcu: rcu head for delayed free * @kobj: kobject used to represent this struct in sysfs * @request_mutex: mutex to protect request/free before locking desc->lock @@ -87,9 +88,9 @@ struct irq_desc { unsigned int core_internal_state__do_not_mess_with_it; unsigned int depth; /* nested irq disables */ unsigned int wake_depth; /* nested wake enables */ - unsigned int tot_count; - unsigned int irq_count; /* For detecting broken IRQs */ + unsigned long tot_count; unsigned long last_unhandled; /* Aging timer for unhandled count */ + unsigned int irq_count; /* For detecting broken IRQs */ unsigned int irqs_unhandled; atomic_t threads_handled; int threads_handled_last; @@ -119,6 +120,7 @@ struct irq_desc { struct dentry *debugfs_file; const char *dev_name; #endif + rcuref_t refcnt; #ifdef CONFIG_SPARSE_IRQ struct rcu_head rcu; struct kobject kobj; diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index b635e3c5d5b6..de754db414d1 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -48,9 +48,11 @@ int irq_set_chip(unsigned int irq, const struct irq_chip *chip) scoped_irqdesc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip); ret = 0; } - /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */ - if (!ret) + if (!ret) { + /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */ irq_mark_irq(irq); + irq_proc_update_chip(chip); + } return ret; } EXPORT_SYMBOL(irq_set_chip); @@ -1012,6 +1014,7 @@ __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle, WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc))); irq_activate_and_startup(desc, IRQ_RESEND); } + irq_proc_update_valid(desc); } void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, @@ -1072,6 +1075,7 @@ void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set) trigger = tmp; irqd_set(&desc->irq_data, trigger); + irq_proc_update_valid(desc); } } EXPORT_SYMBOL_GPL(irq_modify_status); diff --git a/kernel/irq/debugfs.h b/kernel/irq/debugfs.h new file mode 100644 index 000000000000..8a9360d5fefb --- /dev/null +++ b/kernel/irq/debugfs.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_IRQ_DEBUGFS_H +#define _KERNEL_IRQ_DEBUGFS_H + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +#include + +struct irq_bit_descr { + unsigned int mask; + char *name; +}; + +#define BIT_MASK_DESCR(m) { .mask = m, .name = #m } + +void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state, + const struct irq_bit_descr *sd, int size); + +void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc); +static inline void irq_remove_debugfs_entry(struct irq_desc *desc) +{ + debugfs_remove(desc->debugfs_file); + kfree(desc->dev_name); +} +void irq_debugfs_copy_devname(int irq, struct device *dev); +# ifdef CONFIG_IRQ_DOMAIN +void irq_domain_debugfs_init(struct dentry *root); +# else +static inline void irq_domain_debugfs_init(struct dentry *root) +{ +} +# endif +#else /* CONFIG_GENERIC_IRQ_DEBUGFS */ +static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d) +{ +} +static inline void irq_remove_debugfs_entry(struct irq_desc *d) +{ +} +static inline void irq_debugfs_copy_devname(int irq, struct device *dev) +{ +} +#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */ + +#endif diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 9412e57056f5..f9c099d45a64 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -9,8 +9,12 @@ #include #include #include +#include #include +#include "debugfs.h" +#include "proc.h" + #ifdef CONFIG_SPARSE_IRQ # define MAX_SPARSE_IRQS INT_MAX #else @@ -21,6 +25,7 @@ extern bool noirqdebug; extern int irq_poll_cpu; +extern unsigned int total_nr_irqs; extern struct irqaction chained_action; @@ -100,9 +105,23 @@ extern void unmask_irq(struct irq_desc *desc); extern void unmask_threaded_irq(struct irq_desc *desc); #ifdef CONFIG_SPARSE_IRQ -static inline void irq_mark_irq(unsigned int irq) { } +static __always_inline void irq_mark_irq(unsigned int irq) { } +void irq_desc_free_rcu(struct irq_desc *desc); + +static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) +{ + return rcuref_get(&desc->refcnt); +} + +static __always_inline void irq_desc_put_ref(struct irq_desc *desc) +{ + if (rcuref_put(&desc->refcnt)) + irq_desc_free_rcu(desc); +} #else extern void irq_mark_irq(unsigned int irq); +static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; } +static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { } #endif irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc); @@ -122,6 +141,7 @@ extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); extern void register_handler_proc(unsigned int irq, struct irqaction *action); extern void unregister_handler_proc(unsigned int irq, struct irqaction *action); +void irq_proc_update_valid(struct irq_desc *desc); #else static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { } static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { } @@ -129,8 +149,11 @@ static inline void register_handler_proc(unsigned int irq, struct irqaction *action) { } static inline void unregister_handler_proc(unsigned int irq, struct irqaction *action) { } +static inline void irq_proc_update_valid(struct irq_desc *desc) { } #endif +struct irq_desc *irq_find_desc_at_or_after(unsigned int offset); + extern bool irq_can_set_affinity_usr(unsigned int irq); extern int irq_do_set_affinity(struct irq_data *data, @@ -372,42 +395,3 @@ static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd) return NULL; #endif } - -#ifdef CONFIG_GENERIC_IRQ_DEBUGFS -#include - -struct irq_bit_descr { - unsigned int mask; - char *name; -}; - -#define BIT_MASK_DESCR(m) { .mask = m, .name = #m } - -void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state, - const struct irq_bit_descr *sd, int size); - -void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc); -static inline void irq_remove_debugfs_entry(struct irq_desc *desc) -{ - debugfs_remove(desc->debugfs_file); - kfree(desc->dev_name); -} -void irq_debugfs_copy_devname(int irq, struct device *dev); -# ifdef CONFIG_IRQ_DOMAIN -void irq_domain_debugfs_init(struct dentry *root); -# else -static inline void irq_domain_debugfs_init(struct dentry *root) -{ -} -# endif -#else /* CONFIG_GENERIC_IRQ_DEBUGFS */ -static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d) -{ -} -static inline void irq_remove_debugfs_entry(struct irq_desc *d) -{ -} -static inline void irq_debugfs_copy_devname(int irq, struct device *dev) -{ -} -#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */ diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 7173b8b634f2..80ef4e27dcf4 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -137,17 +137,18 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, desc->tot_count = 0; desc->name = NULL; desc->owner = owner; + rcuref_init(&desc->refcnt, 1); desc_smp_init(desc, node, affinity); } -static unsigned int nr_irqs = NR_IRQS; +unsigned int total_nr_irqs __read_mostly = NR_IRQS; /** * irq_get_nr_irqs() - Number of interrupts supported by the system. */ unsigned int irq_get_nr_irqs(void) { - return nr_irqs; + return total_nr_irqs; } EXPORT_SYMBOL_GPL(irq_get_nr_irqs); @@ -157,13 +158,12 @@ EXPORT_SYMBOL_GPL(irq_get_nr_irqs); * * Return: @nr. */ -unsigned int irq_set_nr_irqs(unsigned int nr) +unsigned int __init irq_set_nr_irqs(unsigned int nr) { - nr_irqs = nr; - + total_nr_irqs = nr; + irq_proc_calc_prec(); return nr; } -EXPORT_SYMBOL_GPL(irq_set_nr_irqs); static DEFINE_MUTEX(sparse_irq_lock); static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs, @@ -181,15 +181,12 @@ static int irq_find_free_area(unsigned int from, unsigned int cnt) return mas.index; } -static unsigned int irq_find_at_or_after(unsigned int offset) +struct irq_desc *irq_find_desc_at_or_after(unsigned int offset) { unsigned long index = offset; - struct irq_desc *desc; - guard(rcu)(); - desc = mt_find(&sparse_irqs, &index, nr_irqs); - - return desc ? irq_desc_get_irq(desc) : nr_irqs; + lockdep_assert_in_rcu_read_lock(); + return mt_find(&sparse_irqs, &index, total_nr_irqs); } static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) @@ -466,6 +463,17 @@ static void delayed_free_desc(struct rcu_head *rhp) kobject_put(&desc->kobj); } +void irq_desc_free_rcu(struct irq_desc *desc) +{ + /* + * We free the descriptor, masks and stat fields via RCU. That + * allows demultiplex interrupts to do rcu based management of + * the child interrupts. + * This also allows us to use rcu in kstat_irqs_usr(). + */ + call_rcu(&desc->rcu, delayed_free_desc); +} + static void free_desc(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); @@ -484,14 +492,7 @@ static void free_desc(unsigned int irq) */ irq_sysfs_del(desc); delete_irq_desc(irq); - - /* - * We free the descriptor, masks and stat fields via RCU. That - * allows demultiplex interrupts to do rcu based management of - * the child interrupts. - * This also allows us to use rcu in kstat_irqs_usr(). - */ - call_rcu(&desc->rcu, delayed_free_desc); + irq_desc_put_ref(desc); } static int alloc_descs(unsigned int start, unsigned int cnt, int node, @@ -543,7 +544,8 @@ static bool irq_expand_nr_irqs(unsigned int nr) { if (nr > MAX_SPARSE_IRQS) return false; - nr_irqs = nr; + total_nr_irqs = nr; + irq_proc_calc_prec(); return true; } @@ -557,21 +559,22 @@ int __init early_irq_init(void) /* Let arch update nr_irqs and return the nr of preallocated irqs */ initcnt = arch_probe_nr_irqs(); printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n", - NR_IRQS, nr_irqs, initcnt); + NR_IRQS, total_nr_irqs, initcnt); - if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS)) - nr_irqs = MAX_SPARSE_IRQS; + if (WARN_ON(total_nr_irqs > MAX_SPARSE_IRQS)) + total_nr_irqs = MAX_SPARSE_IRQS; if (WARN_ON(initcnt > MAX_SPARSE_IRQS)) initcnt = MAX_SPARSE_IRQS; - if (initcnt > nr_irqs) - nr_irqs = initcnt; + if (initcnt > total_nr_irqs) + total_nr_irqs = initcnt; for (i = 0; i < initcnt; i++) { desc = alloc_desc(i, node, 0, NULL, NULL); irq_insert_desc(i, desc); } + irq_proc_calc_prec(); return arch_early_irq_init(); } @@ -592,7 +595,7 @@ int __init early_irq_init(void) init_irq_default_affinity(); - printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS); + pr_info("NR_IRQS: %d\n", NR_IRQS); count = ARRAY_SIZE(irq_desc); @@ -602,6 +605,7 @@ int __init early_irq_init(void) goto __free_desc_res; } + irq_proc_calc_prec(); return arch_early_irq_init(); __free_desc_res: @@ -862,7 +866,7 @@ void irq_free_descs(unsigned int from, unsigned int cnt) { int i; - if (from >= nr_irqs || (from + cnt) > nr_irqs) + if (from >= total_nr_irqs || (from + cnt) > total_nr_irqs) return; guard(mutex)(&sparse_irq_lock); @@ -911,7 +915,7 @@ int __ref __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int no if (irq >=0 && start != irq) return -EEXIST; - if (start + cnt > nr_irqs) { + if (start + cnt > total_nr_irqs) { if (!irq_expand_nr_irqs(start + cnt)) return -ENOMEM; } @@ -923,11 +927,15 @@ EXPORT_SYMBOL_GPL(__irq_alloc_descs); * irq_get_next_irq - get next allocated irq number * @offset: where to start the search * - * Returns next irq number after offset or nr_irqs if none is found. + * Returns next irq number after offset or total_nr_irqs if none is found. */ unsigned int irq_get_next_irq(unsigned int offset) { - return irq_find_at_or_after(offset); + struct irq_desc *desc; + + guard(rcu)(); + desc = irq_find_desc_at_or_after(offset); + return desc ? irq_desc_get_irq(desc) : total_nr_irqs; } struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index cc93abf009e8..f15c9f1223bb 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -20,6 +20,8 @@ #include #include +#include "proc.h" + static LIST_HEAD(irq_domain_list); static DEFINE_MUTEX(irq_domain_mutex); @@ -1532,6 +1534,7 @@ int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, unsigned int virq, irq_data->chip = (struct irq_chip *)(chip ? chip : &no_irq_chip); irq_data->chip_data = chip_data; + irq_proc_update_chip(chip); return 0; } EXPORT_SYMBOL_GPL(irq_domain_set_hwirq_and_chip); @@ -2081,7 +2084,7 @@ static void irq_domain_free_one_irq(struct irq_domain *domain, unsigned int virq #endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */ #ifdef CONFIG_GENERIC_IRQ_DEBUGFS -#include "internals.h" +#include "debugfs.h" static struct dentry *domain_dir; diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 2e8072437826..7eb07e3bdb4c 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1802,6 +1802,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) __enable_irq(desc); } + irq_proc_update_valid(desc); raw_spin_unlock_irqrestore(&desc->lock, flags); chip_bus_sync_unlock(desc); mutex_unlock(&desc->request_mutex); @@ -1906,6 +1907,7 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) desc->affinity_hint = NULL; #endif + irq_proc_update_valid(desc); raw_spin_unlock_irqrestore(&desc->lock, flags); /* * Drop bus_lock here so the changes which were done in the chip @@ -2026,24 +2028,32 @@ const void *free_irq(unsigned int irq, void *dev_id) } EXPORT_SYMBOL(free_irq); -/* This function must be called with desc->lock held */ static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc) { + struct irqaction *action = NULL; const char *devname = NULL; - desc->istate &= ~IRQS_NMI; + scoped_guard(raw_spinlock_irqsave, &desc->lock) { + irq_nmi_teardown(desc); - if (!WARN_ON(desc->action == NULL)) { - irq_pm_remove_action(desc, desc->action); - devname = desc->action->name; - unregister_handler_proc(irq, desc->action); + desc->istate &= ~IRQS_NMI; - kfree(desc->action); + if (!WARN_ON(desc->action == NULL)) { + action = desc->action; + irq_pm_remove_action(desc, action); + devname = action->name; + } desc->action = NULL; + + irq_settings_clr_disable_unlazy(desc); + irq_shutdown_and_deactivate(desc); } - irq_settings_clr_disable_unlazy(desc); - irq_shutdown_and_deactivate(desc); + irq_proc_update_valid(desc); + + if (action) + unregister_handler_proc(irq, action); + kfree(action); irq_release_resources(desc); @@ -2067,8 +2077,6 @@ const void *free_nmi(unsigned int irq, void *dev_id) if (WARN_ON(desc->depth == 0)) disable_nmi_nosync(irq); - guard(raw_spinlock_irqsave)(&desc->lock); - irq_nmi_teardown(desc); return __cleanup_nmi(irq, desc); } @@ -2318,13 +2326,14 @@ int request_nmi(unsigned int irq, irq_handler_t handler, /* Setup NMI state */ desc->istate |= IRQS_NMI; retval = irq_nmi_setup(desc); - if (retval) { - __cleanup_nmi(irq, desc); - return -EINVAL; - } - return 0; } + if (retval) { + __cleanup_nmi(irq, desc); + return -EINVAL; + } + return 0; + err_irq_setup: irq_chip_pm_put(&desc->irq_data); err_out: @@ -2428,8 +2437,10 @@ static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_ *action_ptr = action->next; /* Demote from NMI if we killed the last action */ - if (!desc->action) + if (!desc->action) { desc->istate &= ~IRQS_NMI; + irq_proc_update_valid(desc); + } } unregister_handler_proc(irq, action); diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index b0999a4f1f68..1b835725f7b1 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -326,7 +327,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action) #undef MAX_NAMELEN -#define MAX_NAMELEN 10 +#define MAX_NAMELEN 11 void register_irq_proc(unsigned int irq, struct irq_desc *desc) { @@ -348,7 +349,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) return; /* create /proc/irq/1234 */ - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); desc->dir = proc_mkdir(name, root_irq_dir); if (!desc->dir) return; @@ -401,7 +402,7 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) #endif remove_proc_entry("spurious", desc->dir); - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); remove_proc_entry(name, root_irq_dir); } @@ -439,77 +440,159 @@ void init_irq_proc(void) register_irq_proc(irq, desc); } +void irq_proc_update_valid(struct irq_desc *desc) +{ + u32 set = _IRQ_PROC_VALID; + + if (irq_settings_is_hidden(desc) || irq_desc_is_chained(desc) || !desc->action) + set = 0; + + irq_settings_update_proc_valid(desc, set); +} + #ifdef CONFIG_GENERIC_IRQ_SHOW +#define ARCH_PROC_IRQDESC ((void *)0x00001111) + int __weak arch_show_interrupts(struct seq_file *p, int prec) { return 0; } +static DEFINE_RAW_SPINLOCK(irq_proc_constraints_lock); + +static struct irq_proc_constraints { + bool print_header; + unsigned int num_prec; + unsigned int chip_width; +} irq_proc_constraints __read_mostly = { + .num_prec = 4, + .chip_width = 8, +}; + #ifndef ACTUAL_NR_IRQS -# define ACTUAL_NR_IRQS irq_get_nr_irqs() +# define ACTUAL_NR_IRQS total_nr_irqs #endif -int show_interrupts(struct seq_file *p, void *v) +void irq_proc_calc_prec(void) { - const unsigned int nr_irqs = irq_get_nr_irqs(); - static int prec; + unsigned int prec, n; - int i = *(loff_t *) v, j; + for (prec = 4, n = 10000; prec < 10 && n <= total_nr_irqs; ++prec) + n *= 10; + + guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock); + if (prec > irq_proc_constraints.num_prec) + WRITE_ONCE(irq_proc_constraints.num_prec, prec); +} + +void irq_proc_update_chip(const struct irq_chip *chip) +{ + unsigned int len = chip && chip->name ? strlen(chip->name) : 0; + + if (!len || len <= READ_ONCE(irq_proc_constraints.chip_width)) + return; + + /* Can be invoked from interrupt disabled contexts */ + guard(raw_spinlock_irqsave)(&irq_proc_constraints_lock); + if (len > irq_proc_constraints.chip_width) + WRITE_ONCE(irq_proc_constraints.chip_width, len); +} + +/* Same as seq_put_decimal_ull_width(p, " ", cnt, 10) */ +#define ZSTR1 " 0" +#define ZSTR1_LEN (sizeof(ZSTR1) - 1) +#define ZSTR16 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 \ + ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 ZSTR1 +#define ZSTR256 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 \ + ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 ZSTR16 + +static inline void irq_proc_emit_zero_counts(struct seq_file *p, unsigned int zeros) +{ + if (!zeros) + return; + + for (unsigned int n = min(zeros, 256); n; zeros -= n, n = min(zeros, 256)) + seq_write(p, ZSTR256, n * ZSTR1_LEN); +} + +static inline unsigned int irq_proc_emit_count(struct seq_file *p, unsigned int cnt, + unsigned int zeros) +{ + if (!cnt) + return zeros + 1; + + irq_proc_emit_zero_counts(p, zeros); + seq_put_decimal_ull_width(p, " ", cnt, 10); + return 0; +} + +void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts) +{ + unsigned int cpu, zeros = 0; + + for_each_online_cpu(cpu) + zeros = irq_proc_emit_count(p, per_cpu(*cnts, cpu), zeros); + irq_proc_emit_zero_counts(p, zeros); +} + +static int irq_seq_show(struct seq_file *p, void *v) +{ + struct irq_proc_constraints *constr = p->private; + struct irq_desc *desc = v; struct irqaction *action; - struct irq_desc *desc; - if (i > ACTUAL_NR_IRQS) - return 0; + /* Print header for the first interrupt? */ + if (constr->print_header) { + unsigned int cpu; - if (i == ACTUAL_NR_IRQS) - return arch_show_interrupts(p, prec); - - /* print header and calculate the width of the first column */ - if (i == 0) { - for (prec = 3, j = 1000; prec < 10 && j <= nr_irqs; ++prec) - j *= 10; - - seq_printf(p, "%*s", prec + 8, ""); - for_each_online_cpu(j) - seq_printf(p, "CPU%-8d", j); + seq_printf(p, "%*s", constr->num_prec + 8, ""); + for_each_online_cpu(cpu) + seq_printf(p, "CPU%-8d", cpu); seq_putc(p, '\n'); + constr->print_header = false; } - guard(rcu)(); - desc = irq_to_desc(i); - if (!desc || irq_settings_is_hidden(desc)) - return 0; + if (desc == ARCH_PROC_IRQDESC) + return arch_show_interrupts(p, constr->num_prec); - if (!desc->action || irq_desc_is_chained(desc) || !desc->kstat_irqs) - return 0; + seq_put_decimal_ull_width(p, "", irq_desc_get_irq(desc), constr->num_prec); + seq_putc(p, ':'); - seq_printf(p, "%*d:", prec, i); - for_each_online_cpu(j) { - unsigned int cnt = desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, j) : 0; + /* + * Always output per CPU interrupts. Output device interrupts only when + * desc::tot_count is not zero. + */ + if (irq_settings_is_per_cpu(desc) || irq_settings_is_per_cpu_devid(desc) || + data_race(desc->tot_count)) + irq_proc_emit_counts(p, &desc->kstat_irqs->cnt); + else + irq_proc_emit_zero_counts(p, num_online_cpus()); - seq_put_decimal_ull_width(p, " ", cnt, 10); - } - seq_putc(p, ' '); + /* Enforce a visual gap */ + seq_write(p, " ", 2); guard(raw_spinlock_irq)(&desc->lock); if (desc->irq_data.chip) { if (desc->irq_data.chip->irq_print_chip) desc->irq_data.chip->irq_print_chip(&desc->irq_data, p); else if (desc->irq_data.chip->name) - seq_printf(p, "%8s", desc->irq_data.chip->name); + seq_printf(p, "%-*s", constr->chip_width, desc->irq_data.chip->name); else - seq_printf(p, "%8s", "-"); + seq_printf(p, "%-*s", constr->chip_width, "-"); } else { - seq_printf(p, "%8s", "None"); + seq_printf(p, "%-*s", constr->chip_width, "None"); } + + seq_putc(p, ' '); if (desc->irq_data.domain) - seq_printf(p, " %*lu", prec, desc->irq_data.hwirq); + seq_put_decimal_ull_width(p, "", desc->irq_data.hwirq, constr->num_prec); else - seq_printf(p, " %*s", prec, ""); -#ifdef CONFIG_GENERIC_IRQ_SHOW_LEVEL - seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge"); -#endif + seq_printf(p, " %*s", constr->num_prec, ""); + + if (IS_ENABLED(CONFIG_GENERIC_IRQ_SHOW_LEVEL)) + seq_printf(p, " %-8s", irqd_is_level_type(&desc->irq_data) ? "Level" : "Edge"); + if (desc->name) seq_printf(p, "-%-8s", desc->name); @@ -523,4 +606,73 @@ int show_interrupts(struct seq_file *p, void *v) seq_putc(p, '\n'); return 0; } + +static void *irq_seq_next_desc(loff_t *pos) +{ + if (*pos > total_nr_irqs) + return NULL; + + guard(rcu)(); + for (;;) { + struct irq_desc *desc = irq_find_desc_at_or_after((unsigned int) *pos); + + if (desc) { + *pos = irq_desc_get_irq(desc); + /* + * If valid for output then try to acquire a reference + * count on the descriptor so that it can't be freed + * after dropping RCU read lock on return. + */ + if (irq_settings_proc_valid(desc) && irq_desc_get_ref(desc)) + return desc; + (*pos)++; + } else { + *pos = total_nr_irqs; + return ARCH_PROC_IRQDESC; + } + } +} + +static void *irq_seq_start(struct seq_file *f, loff_t *pos) +{ + if (!*pos) { + struct irq_proc_constraints *constr = f->private; + + constr->num_prec = READ_ONCE(irq_proc_constraints.num_prec); + constr->chip_width = READ_ONCE(irq_proc_constraints.chip_width); + constr->print_header = true; + } + return irq_seq_next_desc(pos); +} + +static void *irq_seq_next(struct seq_file *f, void *v, loff_t *pos) +{ + if (v && v != ARCH_PROC_IRQDESC) + irq_desc_put_ref(v); + + (*pos)++; + return irq_seq_next_desc(pos); +} + +static void irq_seq_stop(struct seq_file *f, void *v) +{ + if (v && v != ARCH_PROC_IRQDESC) + irq_desc_put_ref(v); +} + +static const struct seq_operations irq_seq_ops = { + .start = irq_seq_start, + .next = irq_seq_next, + .stop = irq_seq_stop, + .show = irq_seq_show, +}; + +static int __init irq_proc_init(void) +{ + proc_create_seq_private("interrupts", 0, NULL, &irq_seq_ops, + sizeof(irq_proc_constraints), NULL); + return 0; +} +fs_initcall(irq_proc_init); + #endif diff --git a/kernel/irq/proc.h b/kernel/irq/proc.h new file mode 100644 index 000000000000..0631d57fbfb7 --- /dev/null +++ b/kernel/irq/proc.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_IRQ_PROC_H +#define _KERNEL_IRQ_PROC_H + +#if defined(CONFIG_PROC_FS) && defined(CONFIG_GENERIC_IRQ_SHOW) +void irq_proc_calc_prec(void); +void irq_proc_update_chip(const struct irq_chip *chip); +#else +static inline void irq_proc_calc_prec(void) { } +static inline void irq_proc_update_chip(const struct irq_chip *chip) { } +#endif + +#endif diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h index 00b3bd127692..0a0c027a5d34 100644 --- a/kernel/irq/settings.h +++ b/kernel/irq/settings.h @@ -18,6 +18,7 @@ enum { _IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY, _IRQ_HIDDEN = IRQ_HIDDEN, _IRQ_NO_DEBUG = IRQ_NO_DEBUG, + _IRQ_PROC_VALID = IRQ_RESERVED, _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, }; @@ -34,6 +35,7 @@ enum { #define IRQ_DISABLE_UNLAZY GOT_YOU_MORON #define IRQ_HIDDEN GOT_YOU_MORON #define IRQ_NO_DEBUG GOT_YOU_MORON +#define IRQ_RESERVED GOT_YOU_MORON #undef IRQF_MODIFY_MASK #define IRQF_MODIFY_MASK GOT_YOU_MORON @@ -180,3 +182,14 @@ static inline bool irq_settings_no_debug(struct irq_desc *desc) { return desc->status_use_accessors & _IRQ_NO_DEBUG; } + +static inline bool irq_settings_proc_valid(struct irq_desc *desc) +{ + return desc->status_use_accessors & _IRQ_PROC_VALID; +} + +static inline void irq_settings_update_proc_valid(struct irq_desc *desc, u32 set) +{ + desc->status_use_accessors &= ~_IRQ_PROC_VALID; + desc->status_use_accessors |= (set & _IRQ_PROC_VALID); +} diff --git a/scripts/gdb/linux/interrupts.py b/scripts/gdb/linux/interrupts.py index f4f715a8f0e3..a68ae91b4531 100644 --- a/scripts/gdb/linux/interrupts.py +++ b/scripts/gdb/linux/interrupts.py @@ -20,7 +20,7 @@ def irq_desc_is_chained(desc): def irqd_is_level(desc): return desc['irq_data']['common']['state_use_accessors'] & constants.LX_IRQD_LEVEL -def show_irq_desc(prec, irq): +def show_irq_desc(prec, chip_width, irq): text = "" desc = mapletree.mtree_load(gdb.parse_and_eval("&sparse_irqs"), irq) @@ -48,7 +48,7 @@ def show_irq_desc(prec, irq): count = cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt'] else: count = 0 - text += "%10u" % (count) + text += "%10u " % (count) name = "None" if desc['irq_data']['chip']: @@ -58,7 +58,7 @@ def show_irq_desc(prec, irq): else: name = "-" - text += " %8s" % (name) + text += " %-*s" % (chip_width, name) if desc['irq_data']['domain']: text += " %*lu" % (prec, desc['irq_data']['hwirq']) @@ -97,64 +97,29 @@ def show_irq_err_count(prec): text += "%*s: %10u\n" % (prec, "ERR", cnt['counter']) return text -def x86_show_irqstat(prec, pfx, field, desc): - irq_stat = gdb.parse_and_eval("&irq_stat") +def x86_show_irqstat(prec, pfx, idx, desc): + irq_stat = gdb.parse_and_eval("&irq_stat.counts[%d]" %idx) text = "%*s: " % (prec, pfx) for cpu in cpus.each_online_cpu(): stat = cpus.per_cpu(irq_stat, cpu) - text += "%10u " % (stat[field]) - text += " %s\n" % (desc) - return text - -def x86_show_mce(prec, var, pfx, desc): - pvar = gdb.parse_and_eval(var) - text = "%*s: " % (prec, pfx) - for cpu in cpus.each_online_cpu(): - text += "%10u " % (cpus.per_cpu(pvar, cpu).dereference()) - text += " %s\n" % (desc) + text += "%10u " % (stat.dereference()) + text += desc return text def x86_show_interupts(prec): - text = x86_show_irqstat(prec, "NMI", '__nmi_count', 'Non-maskable interrupts') + info_type = gdb.lookup_type('struct irq_stat_info') + info = gdb.parse_and_eval('irq_stat_info') + bitmap = gdb.parse_and_eval('irq_stat_count_show') + bitsperlong = 8 * int(bitmap.type.target().sizeof) - if constants.LX_CONFIG_X86_LOCAL_APIC: - text += x86_show_irqstat(prec, "LOC", 'apic_timer_irqs', "Local timer interrupts") - text += x86_show_irqstat(prec, "SPU", 'irq_spurious_count', "Spurious interrupts") - text += x86_show_irqstat(prec, "PMI", 'apic_perf_irqs', "Performance monitoring interrupts") - text += x86_show_irqstat(prec, "IWI", 'apic_irq_work_irqs', "IRQ work interrupts") - text += x86_show_irqstat(prec, "RTR", 'icr_read_retry_count', "APIC ICR read retries") - if utils.gdb_eval_or_none("x86_platform_ipi_callback") is not None: - text += x86_show_irqstat(prec, "PLT", 'x86_platform_ipis', "Platform interrupts") - - if constants.LX_CONFIG_SMP: - text += x86_show_irqstat(prec, "RES", 'irq_resched_count', "Rescheduling interrupts") - text += x86_show_irqstat(prec, "CAL", 'irq_call_count', "Function call interrupts") - text += x86_show_irqstat(prec, "TLB", 'irq_tlb_count', "TLB shootdowns") - - if constants.LX_CONFIG_X86_THERMAL_VECTOR: - text += x86_show_irqstat(prec, "TRM", 'irq_thermal_count', "Thermal events interrupts") - - if constants.LX_CONFIG_X86_MCE_THRESHOLD: - text += x86_show_irqstat(prec, "THR", 'irq_threshold_count', "Threshold APIC interrupts") - - if constants.LX_CONFIG_X86_MCE_AMD: - text += x86_show_irqstat(prec, "DFR", 'irq_deferred_error_count', "Deferred Error APIC interrupts") - - if constants.LX_CONFIG_X86_MCE: - text += x86_show_mce(prec, "&mce_exception_count", "MCE", "Machine check exceptions") - text += x86_show_mce(prec, "&mce_poll_count", "MCP", "Machine check polls") - - text += show_irq_err_count(prec) - - if constants.LX_CONFIG_X86_IO_APIC: - cnt = utils.gdb_eval_or_none("irq_mis_count") - if cnt is not None: - text += "%*s: %10u\n" % (prec, "MIS", cnt['counter']) - - if constants.LX_CONFIG_KVM: - text += x86_show_irqstat(prec, "PIN", 'kvm_posted_intr_ipis', 'Posted-interrupt notification event') - text += x86_show_irqstat(prec, "NPI", 'kvm_posted_intr_nested_ipis', 'Nested posted-interrupt event') - text += x86_show_irqstat(prec, "PIW", 'kvm_posted_intr_wakeup_ipis', 'Posted-interrupt wakeup event') + text = "" + for idx in range(int(info.type.sizeof / info_type.sizeof)): + show = bitmap[int(idx / bitsperlong)] + if not show & 1 << int(idx % bitsperlong): + continue + pfx = info[idx]['symbol'].string() + desc = info[idx]['text'].string() + text += x86_show_irqstat(prec, pfx, idx, desc) return text @@ -166,23 +131,19 @@ def arm_common_show_interrupts(prec): if nr_ipi is None or ipi_desc is None or ipi_types is None: return text - if prec >= 4: - sep = " " - else: - sep = "" - for ipi in range(nr_ipi): - text += "%*s%u:%s" % (prec - 1, "IPI", ipi, sep) + text += "%*s%u: " % (prec - 1, "IPI", ipi) desc = ipi_desc[ipi].cast(irq_desc_type.get_type().pointer()) if desc == 0: continue for cpu in cpus.each_online_cpu(): - text += "%10u" % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt']) - text += " %s" % (ipi_types[ipi].string()) + text += "%10u " % (cpus.per_cpu(desc['kstat_irqs'], cpu)['cnt']) + text += "%s" % (ipi_types[ipi].string()) text += "\n" return text def aarch64_show_interrupts(prec): + # Does not work for ARM64 as "ipi_desc" is not available there text = arm_common_show_interrupts(prec) text += "%*s: %10lu\n" % (prec, "ERR", gdb.parse_and_eval("irq_err_count")) return text @@ -209,12 +170,19 @@ class LxInterruptList(gdb.Command): super(LxInterruptList, self).__init__("lx-interruptlist", gdb.COMMAND_DATA) def invoke(self, arg, from_tty): - nr_irqs = gdb.parse_and_eval("nr_irqs") - prec = 3 - j = 1000 - while prec < 10 and j <= nr_irqs: - prec += 1 - j *= 10 + nr_irqs = gdb.parse_and_eval("total_nr_irqs") + constr = utils.gdb_eval_or_none('irq_proc_constraints') + + if constr: + prec = int(constr['num_prec']) + chip_width = int(constr['chip_width']) + else: + prec = 4 + j = 10000 + while prec < 10 and j <= nr_irqs: + prec += 1 + j *= 10 + chip_width = 8 gdb.write("%*s" % (prec + 8, "")) for cpu in cpus.each_online_cpu(): @@ -225,7 +193,7 @@ class LxInterruptList(gdb.Command): raise gdb.GdbError("Unable to find the sparse IRQ tree, is CONFIG_SPARSE_IRQ enabled?") for irq in range(nr_irqs): - gdb.write(show_irq_desc(prec, irq)) + gdb.write(show_irq_desc(prec, chip_width, irq)) gdb.write(arch_show_interrupts(prec))