From cc475e5023b416817196ecb3c0af1acc21cd5d8c Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:03:12 +0000 Subject: [PATCH 01/31] duplicate AIC driver as irq-apple-fiq.c --- drivers/irqchip/irq-apple-fiq.c | 861 ++++++++++++++++++++++++++++++++ 1 file changed, 861 insertions(+) create mode 100644 drivers/irqchip/irq-apple-fiq.c diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c new file mode 100644 index 00000000000000..b8c06bd8659e91 --- /dev/null +++ b/drivers/irqchip/irq-apple-fiq.c @@ -0,0 +1,861 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +/* + * AIC is a fairly simple interrupt controller with the following features: + * + * - 896 level-triggered hardware IRQs + * - Single mask bit per IRQ + * - Per-IRQ affinity setting + * - Automatic masking on event delivery (auto-ack) + * - Software triggering (ORed with hw line) + * - 2 per-CPU IPIs (meant as "self" and "other", but they are + * interchangeable if not symmetric) + * - Automatic prioritization (single event/ack register per CPU, lower IRQs = + * higher priority) + * - Automatic masking on ack + * - Default "this CPU" register view and explicit per-CPU views + * + * In addition, this driver also handles FIQs, as these are routed to the same + * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and + * performance counters (TODO). + * + * Implementation notes: + * + * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, + * and one for IPIs. + * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller + * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). + * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - DT bindings use 3-cell form (like GIC): + * - <0 nr flags> - hwirq #nr + * - <1 nr flags> - FIQ #nr + * - nr=0 Physical HV timer + * - nr=1 Virtual HV timer + * - nr=2 Physical guest timer + * - nr=3 Virtual guest timer + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * AIC registers (MMIO) + */ + +#define AIC_INFO 0x0004 +#define AIC_INFO_NR_HW GENMASK(15, 0) + +#define AIC_CONFIG 0x0010 + +#define AIC_WHOAMI 0x2000 +#define AIC_EVENT 0x2004 +#define AIC_EVENT_TYPE GENMASK(31, 16) +#define AIC_EVENT_NUM GENMASK(15, 0) + +#define AIC_EVENT_TYPE_HW 1 +#define AIC_EVENT_TYPE_IPI 4 +#define AIC_EVENT_IPI_OTHER 1 +#define AIC_EVENT_IPI_SELF 2 + +#define AIC_IPI_SEND 0x2008 +#define AIC_IPI_ACK 0x200c +#define AIC_IPI_MASK_SET 0x2024 +#define AIC_IPI_MASK_CLR 0x2028 + +#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) + +#define AIC_IPI_OTHER BIT(0) +#define AIC_IPI_SELF BIT(31) + +#define AIC_TARGET_CPU 0x3000 +#define AIC_SW_SET 0x4000 +#define AIC_SW_CLR 0x4080 +#define AIC_MASK_SET 0x4100 +#define AIC_MASK_CLR 0x4180 + +#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) +#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) + +#define MASK_REG(x) (4 * ((x) >> 5)) +#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) + +/* + * IMP-DEF sysregs that control FIQ sources + * Note: sysreg-based IPIs are not supported yet. + */ + +/* Core PMC control register */ +#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) +#define PMCR0_IMODE GENMASK(10, 8) +#define PMCR0_IMODE_OFF 0 +#define PMCR0_IMODE_PMI 1 +#define PMCR0_IMODE_AIC 2 +#define PMCR0_IMODE_HALT 3 +#define PMCR0_IMODE_FIQ 4 +#define PMCR0_IACT BIT(11) + +/* IPI request registers */ +#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) +#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) +#define IPI_RR_CPU GENMASK(7, 0) +/* Cluster only used for the GLOBAL register */ +#define IPI_RR_CLUSTER GENMASK(23, 16) +#define IPI_RR_TYPE GENMASK(29, 28) +#define IPI_RR_IMMEDIATE 0 +#define IPI_RR_RETRACT 1 +#define IPI_RR_DEFERRED 2 +#define IPI_RR_NOWAKE 3 + +/* IPI status register */ +#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) +#define IPI_SR_PENDING BIT(0) + +/* Guest timer FIQ enable register */ +#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) +#define VM_TMR_FIQ_ENABLE_V BIT(0) +#define VM_TMR_FIQ_ENABLE_P BIT(1) + +/* Deferred IPI countdown register */ +#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) + +/* Uncore PMC control register */ +#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) +#define UPMCR0_IMODE GENMASK(18, 16) +#define UPMCR0_IMODE_OFF 0 +#define UPMCR0_IMODE_AIC 2 +#define UPMCR0_IMODE_HALT 3 +#define UPMCR0_IMODE_FIQ 4 + +/* Uncore PMC status register */ +#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) +#define UPMSR_IACT BIT(0) + +#define AIC_NR_FIQ 4 +#define AIC_NR_SWIPI 32 + +/* + * FIQ hwirq index definitions: FIQ sources use the DT binding defines + * directly, except that timers are special. At the irqchip level, the + * two timer types are represented by their access method: _EL0 registers + * or _EL02 registers. In the DT binding, the timers are represented + * by their purpose (HV or guest). This mapping is for when the kernel is + * running at EL2 (with VHE). When the kernel is running at EL1, the + * mapping differs and aic_irq_domain_translate() performs the remapping. + */ + +#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS +#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT +#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS +#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT + +struct aic_irq_chip { + void __iomem *base; + struct irq_domain *hw_domain; + struct irq_domain *ipi_domain; + int nr_hw; + int ipi_hwirq; +}; + +static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); + +static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); +static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); + +static struct aic_irq_chip *aic_irqc; + +static void aic_handle_ipi(struct pt_regs *regs); + +static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) +{ + return readl_relaxed(ic->base + reg); +} + +static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) +{ + writel_relaxed(val, ic->base + reg); +} + +/* + * IRQ irqchip + */ + +static void aic_irq_mask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(d->hwirq), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_eoi(struct irq_data *d) +{ + /* + * Reading the interrupt reason automatically acknowledges and masks + * the IRQ, so we just unmask it here if needed. + */ + if (!irqd_irq_disabled(d) && !irqd_irq_masked(d)) + aic_irq_unmask(d); +} + +static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 event, type, irq; + + do { + /* + * We cannot use a relaxed read here, as reads from DMA buffers + * need to be ordered after the IRQ fires. + */ + event = readl(ic->base + AIC_EVENT); + type = FIELD_GET(AIC_EVENT_TYPE, event); + irq = FIELD_GET(AIC_EVENT_NUM, event); + + if (type == AIC_EVENT_TYPE_HW) + handle_domain_irq(aic_irqc->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI && irq == 1) + aic_handle_ipi(regs); + else if (event != 0) + pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); + } while (event); + + /* + * vGIC maintenance interrupts end up here too, so we need to check + * for them separately. This should never trigger if KVM is working + * properly, because it will have already taken care of clearing it + * on guest exit before this handler runs. + */ + if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && + read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { + pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } +} + +static int aic_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, bool force) +{ + irq_hw_number_t hwirq = irqd_to_hwirq(d); + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + int cpu; + + if (force) + cpu = cpumask_first(mask_val); + else + cpu = cpumask_any_and(mask_val, cpu_online_mask); + + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK; +} + +static int aic_irq_set_type(struct irq_data *d, unsigned int type) +{ + /* + * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't + * have a way to find out the type of any given IRQ, so just allow both. + */ + return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; +} + +static struct irq_chip aic_chip = { + .name = "AIC", + .irq_mask = aic_irq_mask, + .irq_unmask = aic_irq_unmask, + .irq_eoi = aic_irq_eoi, + .irq_set_affinity = aic_irq_set_affinity, + .irq_set_type = aic_irq_set_type, +}; + +/* + * FIQ irqchip + */ + +static unsigned long aic_fiq_get_idx(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + return irqd_to_hwirq(d) - ic->nr_hw; +} + +static void aic_fiq_set_mask(struct irq_data *d) +{ + /* Only the guest timers have real mask bits, unfortunately. */ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_clear_mask(struct irq_data *d) +{ + switch (aic_fiq_get_idx(d)) { + case AIC_TMR_EL02_PHYS: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); + isb(); + break; + case AIC_TMR_EL02_VIRT: + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); + isb(); + break; + default: + break; + } +} + +static void aic_fiq_mask(struct irq_data *d) +{ + aic_fiq_set_mask(d); + __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_unmask(struct irq_data *d) +{ + aic_fiq_clear_mask(d); + __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); +} + +static void aic_fiq_eoi(struct irq_data *d) +{ + /* We mask to ack (where we can), so we need to unmask at EOI. */ + if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) + aic_fiq_clear_mask(d); +} + +#define TIMER_FIRING(x) \ + (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ + ARCH_TIMER_CTRL_IT_STAT)) == \ + (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) + +static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) +{ + /* + * It would be really nice if we had a system register that lets us get + * the FIQ source state without having to peek down into sources... + * but such a register does not seem to exist. + * + * So, we have these potential sources to test for: + * - Fast IPIs (not yet used) + * - The 4 timers (CNTP, CNTV for each of HV and guest) + * - Per-core PMCs (not yet supported) + * - Per-cluster uncore PMCs (not yet supported) + * + * Since not dealing with any of these results in a FIQ storm, + * we check for everything here, even things we don't support yet. + */ + + if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { + pr_err_ratelimited("Fast IPI fired. Acking.\n"); + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + } + + if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); + + if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); + + if (is_kernel_in_hyp_mode()) { + uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); + + if ((enabled & VM_TMR_FIQ_ENABLE_P) && + TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); + + if ((enabled & VM_TMR_FIQ_ENABLE_V) && + TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) + handle_domain_irq(aic_irqc->hw_domain, + aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); + } + + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { + /* + * Not supported yet, let's figure out how to handle this when + * we implement these proprietary performance counters. For now, + * just mask it and move on. + */ + pr_err_ratelimited("PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + } + + if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + /* Same story with uncore PMCs */ + pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } +} + +static int aic_fiq_set_type(struct irq_data *d, unsigned int type) +{ + return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; +} + +static struct irq_chip fiq_chip = { + .name = "AIC-FIQ", + .irq_mask = aic_fiq_mask, + .irq_unmask = aic_fiq_unmask, + .irq_ack = aic_fiq_set_mask, + .irq_eoi = aic_fiq_eoi, + .irq_set_type = aic_fiq_set_type, +}; + +/* + * Main IRQ domain + */ + +static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) +{ + struct aic_irq_chip *ic = id->host_data; + + if (hw < ic->nr_hw) { + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); + } else { + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static int aic_irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + struct aic_irq_chip *ic = id->host_data; + + if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) + return -EINVAL; + + switch (fwspec->param[0]) { + case AIC_IRQ: + if (fwspec->param[1] >= ic->nr_hw) + return -EINVAL; + *hwirq = fwspec->param[1]; + break; + case AIC_FIQ: + if (fwspec->param[1] >= AIC_NR_FIQ) + return -EINVAL; + *hwirq = ic->nr_hw + fwspec->param[1]; + + /* + * In EL1 the non-redirected registers are the guest's, + * not EL2's, so remap the hwirqs to match. + */ + if (!is_kernel_in_hyp_mode()) { + switch (fwspec->param[1]) { + case AIC_TMR_GUEST_PHYS: + *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; + break; + case AIC_TMR_GUEST_VIRT: + *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; + break; + case AIC_TMR_HV_PHYS: + case AIC_TMR_HV_VIRT: + return -ENOENT; + default: + break; + } + } + break; + default: + return -EINVAL; + } + + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int i, ret; + + ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + if (ret) + return ret; + } + + return 0; +} + +static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops aic_irq_domain_ops = { + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, +}; + +/* + * IPI irqchip + */ + +static void aic_ipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +} + +static void aic_ipi_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) + aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); +} + +static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + u32 send = 0; + int cpu; + unsigned long pending; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in aic_ipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in aic_handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing aic_ipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) + send |= AIC_IPI_SEND_CPU(cpu); + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + aic_ic_write(ic, AIC_IPI_SEND, send); +} + +static struct irq_chip ipi_chip = { + .name = "AIC-IPI", + .irq_mask = aic_ipi_mask, + .irq_unmask = aic_ipi_unmask, + .ipi_send_mask = aic_ipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void aic_handle_ipi(struct pt_regs *regs) +{ + int i; + unsigned long enabled, firing; + + /* + * Ack the IPI. We need to order this after the AIC event read, but + * that is enforced by normal MMIO ordering guarantees. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (which is covered by + * the barrier that is part of the top-level AIC handler's readl()). + */ + enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be + * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, AIC_NR_SWIPI) + handle_domain_irq(aic_irqc->ipi_domain, i, regs); + + /* + * No ordering needed here; at worst this just changes the timing of + * when the next IPI will be delivered. + */ + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); +} + +static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops aic_ipi_domain_ops = { + .alloc = aic_ipi_alloc, + .free = aic_ipi_free, +}; + +static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +{ + struct irq_domain *ipi_domain; + int base_ipi; + + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + &aic_ipi_domain_ops, irqc); + if (WARN_ON(!ipi_domain)) + return -ENODEV; + + ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(!base_ipi)) { + irq_domain_remove(ipi_domain); + return -ENODEV; + } + + set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + + irqc->ipi_domain = ipi_domain; + + return 0; +} + +static int aic_init_cpu(unsigned int cpu) +{ + /* Mask all hard-wired per-CPU IRQ/FIQ sources */ + + /* Pending Fast IPI FIQs */ + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + + /* Timer FIQs */ + sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + + /* EL2-only (VHE mode) IRQ sources */ + if (is_kernel_in_hyp_mode()) { + /* Guest timers */ + sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, + VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); + + /* vGIC maintenance IRQ */ + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } + + /* PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, + FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); + + /* Uncore PMC FIQ */ + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + + /* Commit all of the above */ + isb(); + + /* + * Make sure the kernel's idea of logical CPU order is the same as AIC's + * If we ever end up with a mismatch here, we will have to introduce + * a mapping table similar to what other irqchip drivers do. + */ + WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); + + /* + * Always keep IPIs unmasked at the hardware level (except auto-masking + * by AIC during processing). We manage masks at the vIPI level. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + + /* Initialize the local mask state */ + __this_cpu_write(aic_fiq_unmasked, 0); + + return 0; +} + +static struct gic_kvm_info vgic_info __initdata = { + .type = GIC_V3, + .no_maint_irq_mask = true, + .no_hw_deactivation = true, +}; + +static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +{ + int i; + void __iomem *regs; + u32 info; + struct aic_irq_chip *irqc; + + regs = of_iomap(node, 0); + if (WARN_ON(!regs)) + return -EIO; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + aic_irqc = irqc; + irqc->base = regs; + + info = aic_ic_read(irqc, AIC_INFO); + irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + + irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), + irqc->nr_hw + AIC_NR_FIQ, + &aic_irq_domain_ops, irqc); + if (WARN_ON(!irqc->hw_domain)) { + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); + + if (aic_init_smp(irqc, node)) { + irq_domain_remove(irqc->hw_domain); + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + set_handle_irq(aic_handle_irq); + set_handle_fiq(aic_handle_fiq); + + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); + for (i = 0; i < irqc->nr_hw; i++) + aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); + + if (!is_kernel_in_hyp_mode()) + pr_info("Kernel running in EL1, mapping interrupts"); + + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, + "irqchip/apple-aic/ipi:starting", + aic_init_cpu, NULL); + + vgic_set_kvm_info(&vgic_info); + + pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", + irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + + return 0; +} + +IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); From 99e3e01d4cc03009c0e31dafc18a11d5d6b75988 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:08:36 +0000 Subject: [PATCH 02/31] delete IRQ stuff from FIQ driver --- drivers/irqchip/irq-apple-fiq.c | 320 -------------------------------- 1 file changed, 320 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index b8c06bd8659e91..ff53d983f5cb86 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -50,7 +50,6 @@ #include #include #include -#include #include #include #include @@ -59,48 +58,6 @@ #include #include -#include - -/* - * AIC registers (MMIO) - */ - -#define AIC_INFO 0x0004 -#define AIC_INFO_NR_HW GENMASK(15, 0) - -#define AIC_CONFIG 0x0010 - -#define AIC_WHOAMI 0x2000 -#define AIC_EVENT 0x2004 -#define AIC_EVENT_TYPE GENMASK(31, 16) -#define AIC_EVENT_NUM GENMASK(15, 0) - -#define AIC_EVENT_TYPE_HW 1 -#define AIC_EVENT_TYPE_IPI 4 -#define AIC_EVENT_IPI_OTHER 1 -#define AIC_EVENT_IPI_SELF 2 - -#define AIC_IPI_SEND 0x2008 -#define AIC_IPI_ACK 0x200c -#define AIC_IPI_MASK_SET 0x2024 -#define AIC_IPI_MASK_CLR 0x2028 - -#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) - -#define AIC_IPI_OTHER BIT(0) -#define AIC_IPI_SELF BIT(31) - -#define AIC_TARGET_CPU 0x3000 -#define AIC_SW_SET 0x4000 -#define AIC_SW_CLR 0x4080 -#define AIC_MASK_SET 0x4100 -#define AIC_MASK_CLR 0x4180 - -#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) -#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) - #define MASK_REG(x) (4 * ((x) >> 5)) #define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) @@ -183,124 +140,8 @@ struct aic_irq_chip { static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); - static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs); - -static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) -{ - return readl_relaxed(ic->base + reg); -} - -static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) -{ - writel_relaxed(val, ic->base + reg); -} - -/* - * IRQ irqchip - */ - -static void aic_irq_mask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(d->hwirq), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_eoi(struct irq_data *d) -{ - /* - * Reading the interrupt reason automatically acknowledges and masks - * the IRQ, so we just unmask it here if needed. - */ - if (!irqd_irq_disabled(d) && !irqd_irq_masked(d)) - aic_irq_unmask(d); -} - -static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) -{ - struct aic_irq_chip *ic = aic_irqc; - u32 event, type, irq; - - do { - /* - * We cannot use a relaxed read here, as reads from DMA buffers - * need to be ordered after the IRQ fires. - */ - event = readl(ic->base + AIC_EVENT); - type = FIELD_GET(AIC_EVENT_TYPE, event); - irq = FIELD_GET(AIC_EVENT_NUM, event); - - if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(aic_irqc->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs); - else if (event != 0) - pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); - } while (event); - - /* - * vGIC maintenance interrupts end up here too, so we need to check - * for them separately. This should never trigger if KVM is working - * properly, because it will have already taken care of clearing it - * on guest exit before this handler runs. - */ - if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && - read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { - pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); - } -} - -static int aic_irq_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) -{ - irq_hw_number_t hwirq = irqd_to_hwirq(d); - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - int cpu; - - if (force) - cpu = cpumask_first(mask_val); - else - cpu = cpumask_any_and(mask_val, cpu_online_mask); - - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); - - return IRQ_SET_MASK_OK; -} - -static int aic_irq_set_type(struct irq_data *d, unsigned int type) -{ - /* - * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't - * have a way to find out the type of any given IRQ, so just allow both. - */ - return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; -} - -static struct irq_chip aic_chip = { - .name = "AIC", - .irq_mask = aic_irq_mask, - .irq_unmask = aic_irq_unmask, - .irq_eoi = aic_irq_eoi, - .irq_set_affinity = aic_irq_set_affinity, - .irq_set_type = aic_irq_set_type, -}; - /* * FIQ irqchip */ @@ -481,11 +322,6 @@ static int aic_irq_domain_translate(struct irq_domain *id, return -EINVAL; switch (fwspec->param[0]) { - case AIC_IRQ: - if (fwspec->param[1] >= ic->nr_hw) - return -EINVAL; - *hwirq = fwspec->param[1]; - break; case AIC_FIQ: if (fwspec->param[1] >= AIC_NR_FIQ) return -EINVAL; @@ -560,153 +396,6 @@ static const struct irq_domain_ops aic_irq_domain_ops = { .free = aic_irq_domain_free, }; -/* - * IPI irqchip - */ - -static void aic_ipi_mask(struct irq_data *d) -{ - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); -} - -static void aic_ipi_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); -} - -static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - u32 send = 0; - int cpu; - unsigned long pending; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); - - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); - - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ - if (send) - aic_ic_write(ic, AIC_IPI_SEND, send); -} - -static struct irq_chip ipi_chip = { - .name = "AIC-IPI", - .irq_mask = aic_ipi_mask, - .irq_unmask = aic_ipi_unmask, - .ipi_send_mask = aic_ipi_send_mask, -}; - -/* - * IPI IRQ domain - */ - -static void aic_handle_ipi(struct pt_regs *regs) -{ - int i; - unsigned long enabled, firing; - - /* - * Ack the IPI. We need to order this after the AIC event read, but - * that is enforced by normal MMIO ordering guarantees. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); - - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); - - /* - * No ordering needed here; at worst this just changes the timing of - * when the next IPI will be delivered. - */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); -} - -static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) -{ - int i; - - for (i = 0; i < nr_irqs; i++) { - irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, - handle_percpu_devid_irq, NULL, NULL); - } - - return 0; -} - -static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) -{ - /* Not freeing IPIs */ -} - -static const struct irq_domain_ops aic_ipi_domain_ops = { - .alloc = aic_ipi_alloc, - .free = aic_ipi_free, -}; - static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) { struct irq_domain *ipi_domain; @@ -788,12 +477,6 @@ static int aic_init_cpu(unsigned int cpu) return 0; } -static struct gic_kvm_info vgic_info __initdata = { - .type = GIC_V3, - .no_maint_irq_mask = true, - .no_hw_deactivation = true, -}; - static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i; @@ -833,7 +516,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p return -ENODEV; } - set_handle_irq(aic_handle_irq); set_handle_fiq(aic_handle_fiq); for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) @@ -850,8 +532,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); - vgic_set_kvm_info(&vgic_info); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); From 5d693042e25982984057e946bb6f87626d3d44b3 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:14:26 +0000 Subject: [PATCH 03/31] delete FIQ stuff from IRQ driver --- drivers/irqchip/irq-apple-aic.c | 297 +------------------------------- 1 file changed, 8 insertions(+), 289 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index b8c06bd8659e91..d085a51e931e63 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -23,24 +23,13 @@ * - Automatic masking on ack * - Default "this CPU" register view and explicit per-CPU views * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). - * * Implementation notes: * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. + * - This driver creates two IRQ domains, one for HW IRQs, and one for IPIs. * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. * - DT bindings use 3-cell form (like GIC): * - <0 nr flags> - hwirq #nr - * - <1 nr flags> - FIQ #nr - * - nr=0 Physical HV timer - * - nr=1 Virtual HV timer - * - nr=2 Physical guest timer - * - nr=3 Virtual guest timer */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -104,75 +93,8 @@ #define MASK_REG(x) (4 * ((x) >> 5)) #define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) -/* - * IMP-DEF sysregs that control FIQ sources - * Note: sysreg-based IPIs are not supported yet. - */ - -/* Core PMC control register */ -#define SYS_IMP_APL_PMCR0_EL1 sys_reg(3, 1, 15, 0, 0) -#define PMCR0_IMODE GENMASK(10, 8) -#define PMCR0_IMODE_OFF 0 -#define PMCR0_IMODE_PMI 1 -#define PMCR0_IMODE_AIC 2 -#define PMCR0_IMODE_HALT 3 -#define PMCR0_IMODE_FIQ 4 -#define PMCR0_IACT BIT(11) - -/* IPI request registers */ -#define SYS_IMP_APL_IPI_RR_LOCAL_EL1 sys_reg(3, 5, 15, 0, 0) -#define SYS_IMP_APL_IPI_RR_GLOBAL_EL1 sys_reg(3, 5, 15, 0, 1) -#define IPI_RR_CPU GENMASK(7, 0) -/* Cluster only used for the GLOBAL register */ -#define IPI_RR_CLUSTER GENMASK(23, 16) -#define IPI_RR_TYPE GENMASK(29, 28) -#define IPI_RR_IMMEDIATE 0 -#define IPI_RR_RETRACT 1 -#define IPI_RR_DEFERRED 2 -#define IPI_RR_NOWAKE 3 - -/* IPI status register */ -#define SYS_IMP_APL_IPI_SR_EL1 sys_reg(3, 5, 15, 1, 1) -#define IPI_SR_PENDING BIT(0) - -/* Guest timer FIQ enable register */ -#define SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2 sys_reg(3, 5, 15, 1, 3) -#define VM_TMR_FIQ_ENABLE_V BIT(0) -#define VM_TMR_FIQ_ENABLE_P BIT(1) - -/* Deferred IPI countdown register */ -#define SYS_IMP_APL_IPI_CR_EL1 sys_reg(3, 5, 15, 3, 1) - -/* Uncore PMC control register */ -#define SYS_IMP_APL_UPMCR0_EL1 sys_reg(3, 7, 15, 0, 4) -#define UPMCR0_IMODE GENMASK(18, 16) -#define UPMCR0_IMODE_OFF 0 -#define UPMCR0_IMODE_AIC 2 -#define UPMCR0_IMODE_HALT 3 -#define UPMCR0_IMODE_FIQ 4 - -/* Uncore PMC status register */ -#define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) -#define UPMSR_IACT BIT(0) - -#define AIC_NR_FIQ 4 #define AIC_NR_SWIPI 32 -/* - * FIQ hwirq index definitions: FIQ sources use the DT binding defines - * directly, except that timers are special. At the irqchip level, the - * two timer types are represented by their access method: _EL0 registers - * or _EL02 registers. In the DT binding, the timers are represented - * by their purpose (HV or guest). This mapping is for when the kernel is - * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. - */ - -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT - struct aic_irq_chip { void __iomem *base; struct irq_domain *hw_domain; @@ -181,8 +103,6 @@ struct aic_irq_chip { int ipi_hwirq; }; -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); - static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); @@ -301,153 +221,6 @@ static struct irq_chip aic_chip = { .irq_set_type = aic_irq_set_type, }; -/* - * FIQ irqchip - */ - -static unsigned long aic_fiq_get_idx(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - return irqd_to_hwirq(d) - ic->nr_hw; -} - -static void aic_fiq_set_mask(struct irq_data *d) -{ - /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_clear_mask(struct irq_data *d) -{ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); - isb(); - break; - case AIC_TMR_EL02_VIRT: - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); - isb(); - break; - default: - break; - } -} - -static void aic_fiq_mask(struct irq_data *d) -{ - aic_fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_unmask(struct irq_data *d) -{ - aic_fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); -} - -static void aic_fiq_eoi(struct irq_data *d) -{ - /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); -} - -#define TIMER_FIRING(x) \ - (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ - ARCH_TIMER_CTRL_IT_STAT)) == \ - (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) - -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) -{ - /* - * It would be really nice if we had a system register that lets us get - * the FIQ source state without having to peek down into sources... - * but such a register does not seem to exist. - * - * So, we have these potential sources to test for: - * - Fast IPIs (not yet used) - * - The 4 timers (CNTP, CNTV for each of HV and guest) - * - Per-core PMCs (not yet supported) - * - Per-cluster uncore PMCs (not yet supported) - * - * Since not dealing with any of these results in a FIQ storm, - * we check for everything here, even things we don't support yet. - */ - - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - - if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); - - if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); - - if (is_kernel_in_hyp_mode()) { - uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); - - if ((enabled & VM_TMR_FIQ_ENABLE_P) && - TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); - - if ((enabled & VM_TMR_FIQ_ENABLE_V) && - TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); - } - - if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == - (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { - /* - * Not supported yet, let's figure out how to handle this when - * we implement these proprietary performance counters. For now, - * just mask it and move on. - */ - pr_err_ratelimited("PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - } - - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { - /* Same story with uncore PMCs */ - pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - } -} - -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) -{ - return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; -} - -static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, -}; - /* * Main IRQ domain */ @@ -457,15 +230,9 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, { struct aic_irq_chip *ic = id->host_data; - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); return 0; } @@ -486,31 +253,6 @@ static int aic_irq_domain_translate(struct irq_domain *id, return -EINVAL; *hwirq = fwspec->param[1]; break; - case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) - return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; - - /* - * In EL1 the non-redirected registers are the guest's, - * not EL2's, so remap the hwirqs to match. - */ - if (!is_kernel_in_hyp_mode()) { - switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; - break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; - break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: - return -ENOENT; - default: - break; - } - } - break; default: return -EINVAL; } @@ -737,33 +479,14 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ - - /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - - /* Timer FIQs */ - sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); - sysreg_clear_set(cntv_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); + /* Mask all hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { - /* Guest timers */ - sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, - VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - /* vGIC maintenance IRQ */ sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } - /* PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_PMCR0_EL1, PMCR0_IMODE | PMCR0_IACT, - FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); - - /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); - /* Commit all of the above */ isb(); @@ -782,9 +505,6 @@ static int aic_init_cpu(unsigned int cpu) aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); - return 0; } @@ -816,7 +536,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, + irqc->nr_hw, &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) { iounmap(irqc->base); @@ -834,7 +554,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p } set_handle_irq(aic_handle_irq); - set_handle_fiq(aic_handle_fiq); for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); @@ -852,8 +571,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p vgic_set_kvm_info(&vgic_info); - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d IRQs, %d vIPIs\n", + irqc->nr_hw, AIC_NR_SWIPI); return 0; } From 5e3e99ad462437a22d1f281f6fa73e6461b2326f Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:22:04 +0000 Subject: [PATCH 04/31] remove more IRQ stuff from FIQ driver --- drivers/irqchip/irq-apple-fiq.c | 222 +++++++++----------------------- 1 file changed, 60 insertions(+), 162 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index ff53d983f5cb86..f705472960d5c0 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -9,33 +9,13 @@ */ /* - * AIC is a fairly simple interrupt controller with the following features: - * - * - 896 level-triggered hardware IRQs - * - Single mask bit per IRQ - * - Per-IRQ affinity setting - * - Automatic masking on event delivery (auto-ack) - * - Software triggering (ORed with hw line) - * - 2 per-CPU IPIs (meant as "self" and "other", but they are - * interchangeable if not symmetric) - * - Automatic prioritization (single event/ack register per CPU, lower IRQs = - * higher priority) - * - Automatic masking on ack - * - Default "this CPU" register view and explicit per-CPU views - * - * In addition, this driver also handles FIQs, as these are routed to the same - * IRQ vector. These are used for Fast IPIs (TODO), the ARMv8 timer IRQs, and - * performance counters (TODO). + * This driver handles FIQs. These are used for Fast IPIs (TODO), the + * ARMv8 timer IRQs, and performance counters (TODO). * * Implementation notes: * - * - This driver creates two IRQ domains, one for HW IRQs and internal FIQs, - * and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - FIQ hwirq numbers are assigned after true hwirqs, and are per-cpu. + * - This driver creates one IRQ domain, for FIQs. * - DT bindings use 3-cell form (like GIC): - * - <0 nr flags> - hwirq #nr * - <1 nr flags> - FIQ #nr * - nr=0 Physical HV timer * - nr=1 Virtual HV timer @@ -58,9 +38,6 @@ #include #include -#define MASK_REG(x) (4 * ((x) >> 5)) -#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) - /* * IMP-DEF sysregs that control FIQ sources * Note: sysreg-based IPIs are not supported yet. @@ -112,8 +89,7 @@ #define SYS_IMP_APL_UPMSR_EL1 sys_reg(3, 7, 15, 6, 4) #define UPMSR_IACT BIT(0) -#define AIC_NR_FIQ 4 -#define AIC_NR_SWIPI 32 +#define NR_FIQ 4 /* * FIQ hwirq index definitions: FIQ sources use the DT binding defines @@ -125,10 +101,10 @@ * mapping differs and aic_irq_domain_translate() performs the remapping. */ -#define AIC_TMR_EL0_PHYS AIC_TMR_HV_PHYS -#define AIC_TMR_EL0_VIRT AIC_TMR_HV_VIRT -#define AIC_TMR_EL02_PHYS AIC_TMR_GUEST_PHYS -#define AIC_TMR_EL02_VIRT AIC_TMR_GUEST_VIRT +#define TMR_EL0_PHYS TMR_HV_PHYS +#define TMR_EL0_VIRT TMR_HV_VIRT +#define TMR_EL02_PHYS TMR_GUEST_PHYS +#define TMR_EL02_VIRT TMR_GUEST_VIRT struct aic_irq_chip { void __iomem *base; @@ -146,22 +122,22 @@ static struct aic_irq_chip *aic_irqc; * FIQ irqchip */ -static unsigned long aic_fiq_get_idx(struct irq_data *d) +static unsigned long fiq_get_idx(struct irq_data *d) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - return irqd_to_hwirq(d) - ic->nr_hw; + return irqd_to_hwirq(d); } -static void aic_fiq_set_mask(struct irq_data *d) +static void fiq_set_mask(struct irq_data *d) { /* Only the guest timers have real mask bits, unfortunately. */ - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: + switch (fiq_get_idx(d)) { + case TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); isb(); break; - case AIC_TMR_EL02_VIRT: + case TMR_EL02_VIRT: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V, 0); isb(); break; @@ -170,14 +146,14 @@ static void aic_fiq_set_mask(struct irq_data *d) } } -static void aic_fiq_clear_mask(struct irq_data *d) +static void fiq_clear_mask(struct irq_data *d) { - switch (aic_fiq_get_idx(d)) { - case AIC_TMR_EL02_PHYS: + switch (fiq_get_idx(d)) { + case TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); isb(); break; - case AIC_TMR_EL02_VIRT: + case TMR_EL02_VIRT: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_V); isb(); break; @@ -186,22 +162,22 @@ static void aic_fiq_clear_mask(struct irq_data *d) } } -static void aic_fiq_mask(struct irq_data *d) +static void fiq_mask(struct irq_data *d) { - aic_fiq_set_mask(d); + fiq_set_mask(d); __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); } static void aic_fiq_unmask(struct irq_data *d) { - aic_fiq_clear_mask(d); + fiq_clear_mask(d); __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); } -static void aic_fiq_eoi(struct irq_data *d) +static void fiq_eoi(struct irq_data *d) { /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(aic_fiq_unmasked) & BIT(aic_fiq_get_idx(d))) + if (__this_cpu_read(fiq_unmasked) & BIT(aic_fiq_get_idx(d))) aic_fiq_clear_mask(d); } @@ -210,7 +186,7 @@ static void aic_fiq_eoi(struct irq_data *d) ARCH_TIMER_CTRL_IT_STAT)) == \ (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) -static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) +static void __exception_irq_entry handle_fiq(struct pt_regs *regs) { /* * It would be really nice if we had a system register that lets us get @@ -233,25 +209,21 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) } if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_PHYS, regs); + handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL0_PHYS, regs); if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL0_VIRT, regs); + handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL0_VIRT, regs); if (is_kernel_in_hyp_mode()) { uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); if ((enabled & VM_TMR_FIQ_ENABLE_P) && TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_PHYS, regs); + handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL02_PHYS, regs); if ((enabled & VM_TMR_FIQ_ENABLE_V) && TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, - aic_irqc->nr_hw + AIC_TMR_EL02_VIRT, regs); + handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL02_VIRT, regs); } if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == @@ -275,43 +247,37 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) } } -static int aic_fiq_set_type(struct irq_data *d, unsigned int type) +static int fiq_set_type(struct irq_data *d, unsigned int type) { return (type == IRQ_TYPE_LEVEL_HIGH) ? 0 : -EINVAL; } static struct irq_chip fiq_chip = { .name = "AIC-FIQ", - .irq_mask = aic_fiq_mask, - .irq_unmask = aic_fiq_unmask, - .irq_ack = aic_fiq_set_mask, - .irq_eoi = aic_fiq_eoi, - .irq_set_type = aic_fiq_set_type, + .irq_mask = fiq_mask, + .irq_unmask = fiq_unmask, + .irq_ack = fiq_set_mask, + .irq_eoi = fiq_eoi, + .irq_set_type = fiq_set_type, }; /* * Main IRQ domain */ -static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, +static int irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { struct aic_irq_chip *ic = id->host_data; - if (hw < ic->nr_hw) { - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - } else { - irq_set_percpu_devid(irq); - irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, - handle_percpu_devid_irq, NULL, NULL); - } + irq_set_percpu_devid(irq); + irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, + handle_percpu_devid_irq, NULL, NULL); return 0; } -static int aic_irq_domain_translate(struct irq_domain *id, +static int irq_domain_translate(struct irq_domain *id, struct irq_fwspec *fwspec, unsigned long *hwirq, unsigned int *type) @@ -323,7 +289,7 @@ static int aic_irq_domain_translate(struct irq_domain *id, switch (fwspec->param[0]) { case AIC_FIQ: - if (fwspec->param[1] >= AIC_NR_FIQ) + if (fwspec->param[1] >= NR_FIQ) return -EINVAL; *hwirq = ic->nr_hw + fwspec->param[1]; @@ -333,14 +299,14 @@ static int aic_irq_domain_translate(struct irq_domain *id, */ if (!is_kernel_in_hyp_mode()) { switch (fwspec->param[1]) { - case AIC_TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + AIC_TMR_EL0_PHYS; + case TMR_GUEST_PHYS: + *hwirq = ic->nr_hw + TMR_EL0_PHYS; break; - case AIC_TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + AIC_TMR_EL0_VIRT; + case TMR_GUEST_VIRT: + *hwirq = ic->nr_hw + TMR_EL0_VIRT; break; - case AIC_TMR_HV_PHYS: - case AIC_TMR_HV_VIRT: + case TMR_HV_PHYS: + case TMR_HV_VIRT: return -ENOENT; default: break; @@ -356,7 +322,7 @@ static int aic_irq_domain_translate(struct irq_domain *id, return 0; } -static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, +static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { unsigned int type = IRQ_TYPE_NONE; @@ -364,12 +330,12 @@ static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, irq_hw_number_t hwirq; int i, ret; - ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + ret = irq_domain_translate(domain, fwspec, &hwirq, &type); if (ret) return ret; for (i = 0; i < nr_irqs; i++) { - ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + ret = irq_domain_map(domain, virq + i, hwirq + i); if (ret) return ret; } @@ -377,7 +343,7 @@ static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, +static void irq_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { int i; @@ -390,40 +356,12 @@ static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, } } -static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, +static const struct irq_domain_ops irq_domain_ops = { + .translate = irq_domain_translate, + .alloc = irq_domain_alloc, + .free = irq_domain_free, }; -static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) -{ - struct irq_domain *ipi_domain; - int base_ipi; - - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, - &aic_ipi_domain_ops, irqc); - if (WARN_ON(!ipi_domain)) - return -ENODEV; - - ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, - NUMA_NO_NODE, NULL, false, NULL); - - if (WARN_ON(!base_ipi)) { - irq_domain_remove(ipi_domain); - return -ENODEV; - } - - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); - - irqc->ipi_domain = ipi_domain; - - return 0; -} - static int aic_init_cpu(unsigned int cpu) { /* Mask all hard-wired per-CPU IRQ/FIQ sources */ @@ -440,9 +378,6 @@ static int aic_init_cpu(unsigned int cpu) /* Guest timers */ sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); - - /* vGIC maintenance IRQ */ - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } /* PMC FIQ */ @@ -456,28 +391,13 @@ static int aic_init_cpu(unsigned int cpu) /* Commit all of the above */ isb(); - /* - * Make sure the kernel's idea of logical CPU order is the same as AIC's - * If we ever end up with a mismatch here, we will have to introduce - * a mapping table similar to what other irqchip drivers do. - */ - WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); - - /* - * Always keep IPIs unmasked at the hardware level (except auto-masking - * by AIC during processing). We manage masks at the vIPI level. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); - aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); - /* Initialize the local mask state */ - __this_cpu_write(aic_fiq_unmasked, 0); + __this_cpu_write(fiq_unmasked, 0); return 0; } -static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +static int __init fiq_of_ic_init(struct device_node *node, struct device_node *parent) { int i; void __iomem *regs; @@ -495,12 +415,9 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p aic_irqc = irqc; irqc->base = regs; - info = aic_ic_read(irqc, AIC_INFO); - irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); - irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw + AIC_NR_FIQ, - &aic_irq_domain_ops, irqc); + AIC_NR_FIQ, + &irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) { iounmap(irqc->base); kfree(irqc); @@ -509,33 +426,14 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - if (aic_init_smp(irqc, node)) { - irq_domain_remove(irqc->hw_domain); - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; - } - set_handle_fiq(aic_handle_fiq); - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); - for (i = 0; i < irqc->nr_hw; i++) - aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); - if (!is_kernel_in_hyp_mode()) pr_info("Kernel running in EL1, mapping interrupts"); - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, - "irqchip/apple-aic/ipi:starting", - aic_init_cpu, NULL); - - pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); + pr_info("Initialized with %d FIQs\n", AIC_NR_FIQ); return 0; } -IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); +IRQCHIP_DECLARE(apple_m1_fiq, "apple,fiq", fiq_of_ic_init); From 77896b20f8130206a3e1ae5f869cf8b37265bc0b Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:23:53 +0000 Subject: [PATCH 05/31] remove more IRQ stuff from FIQ driver --- drivers/irqchip/irq-apple-fiq.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index f705472960d5c0..1f7a81d3e479bb 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -107,11 +107,7 @@ #define TMR_EL02_VIRT TMR_GUEST_VIRT struct aic_irq_chip { - void __iomem *base; - struct irq_domain *hw_domain; - struct irq_domain *ipi_domain; - int nr_hw; - int ipi_hwirq; + struct irq_domain *fiq_domain; }; static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); @@ -209,21 +205,21 @@ static void __exception_irq_entry handle_fiq(struct pt_regs *regs) } if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL0_PHYS, regs); + handle_domain_irq(aic_irqc->domain, AIC_TMR_EL0_PHYS, regs); if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL0_VIRT, regs); + handle_domain_irq(aic_irqc->domain, AIC_TMR_EL0_VIRT, regs); if (is_kernel_in_hyp_mode()) { uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); if ((enabled & VM_TMR_FIQ_ENABLE_P) && TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL02_PHYS, regs); + handle_domain_irq(aic_irqc->domain, AIC_TMR_EL02_PHYS, regs); if ((enabled & VM_TMR_FIQ_ENABLE_V) && TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->hw_domain, AIC_TMR_EL02_VIRT, regs); + handle_domain_irq(aic_irqc->domain, AIC_TMR_EL02_VIRT, regs); } if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == @@ -291,7 +287,7 @@ static int irq_domain_translate(struct irq_domain *id, case AIC_FIQ: if (fwspec->param[1] >= NR_FIQ) return -EINVAL; - *hwirq = ic->nr_hw + fwspec->param[1]; + *hwirq = fwspec->param[1]; /* * In EL1 the non-redirected registers are the guest's, @@ -300,10 +296,10 @@ static int irq_domain_translate(struct irq_domain *id, if (!is_kernel_in_hyp_mode()) { switch (fwspec->param[1]) { case TMR_GUEST_PHYS: - *hwirq = ic->nr_hw + TMR_EL0_PHYS; + *hwirq = TMR_EL0_PHYS; break; case TMR_GUEST_VIRT: - *hwirq = ic->nr_hw + TMR_EL0_VIRT; + *hwirq = TMR_EL0_VIRT; break; case TMR_HV_PHYS: case TMR_HV_VIRT: @@ -415,23 +411,22 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p aic_irqc = irqc; irqc->base = regs; - irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - AIC_NR_FIQ, - &irq_domain_ops, irqc); - if (WARN_ON(!irqc->hw_domain)) { + irqc->domain = irq_domain_create_linear(of_node_to_fwnode(node), + NR_FIQ, &irq_domain_ops, irqc); + if (WARN_ON(!irqc->domain)) { iounmap(irqc->base); kfree(irqc); return -ENODEV; } - irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); + irq_domain_update_bus_token(irqc->domain, DOMAIN_BUS_WIRED); - set_handle_fiq(aic_handle_fiq); + set_handle_fiq(handle_fiq); if (!is_kernel_in_hyp_mode()) pr_info("Kernel running in EL1, mapping interrupts"); - pr_info("Initialized with %d FIQs\n", AIC_NR_FIQ); + pr_info("Initialized with %d FIQs\n", NR_FIQ); return 0; } From 911a0b3c47d033a0973415e2c7c59308af1235e9 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:28:42 +0000 Subject: [PATCH 06/31] remove more IRQ stuff from FIQ driver --- drivers/irqchip/irq-apple-fiq.c | 41 +++++++++++++++------------------ 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 1f7a81d3e479bb..4d4b4b35c4d69d 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -98,7 +98,7 @@ * or _EL02 registers. In the DT binding, the timers are represented * by their purpose (HV or guest). This mapping is for when the kernel is * running at EL2 (with VHE). When the kernel is running at EL1, the - * mapping differs and aic_irq_domain_translate() performs the remapping. + * mapping differs and irq_domain_translate() performs the remapping. */ #define TMR_EL0_PHYS TMR_HV_PHYS @@ -106,13 +106,13 @@ #define TMR_EL02_PHYS TMR_GUEST_PHYS #define TMR_EL02_VIRT TMR_GUEST_VIRT -struct aic_irq_chip { - struct irq_domain *fiq_domain; +struct fiq_irq_chip { + struct irq_domain *domain; }; -static DEFINE_PER_CPU(uint32_t, aic_fiq_unmasked); +static DEFINE_PER_CPU(uint32_t, fiq_unmasked); -static struct aic_irq_chip *aic_irqc; +static struct fiq_irq_chip *fiq_irqc; /* * FIQ irqchip @@ -120,8 +120,6 @@ static struct aic_irq_chip *aic_irqc; static unsigned long fiq_get_idx(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - return irqd_to_hwirq(d); } @@ -161,20 +159,20 @@ static void fiq_clear_mask(struct irq_data *d) static void fiq_mask(struct irq_data *d) { fiq_set_mask(d); - __this_cpu_and(aic_fiq_unmasked, ~BIT(aic_fiq_get_idx(d))); + __this_cpu_and(fiq_unmasked, ~BIT(fiq_get_idx(d))); } -static void aic_fiq_unmask(struct irq_data *d) +static void fiq_unmask(struct irq_data *d) { fiq_clear_mask(d); - __this_cpu_or(aic_fiq_unmasked, BIT(aic_fiq_get_idx(d))); + __this_cpu_or(fiq_unmasked, BIT(fiq_get_idx(d))); } static void fiq_eoi(struct irq_data *d) { /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(fiq_unmasked) & BIT(aic_fiq_get_idx(d))) - aic_fiq_clear_mask(d); + if (__this_cpu_read(fiq_unmasked) & BIT(fiq_get_idx(d))) + fiq_clear_mask(d); } #define TIMER_FIRING(x) \ @@ -205,21 +203,21 @@ static void __exception_irq_entry handle_fiq(struct pt_regs *regs) } if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) - handle_domain_irq(aic_irqc->domain, AIC_TMR_EL0_PHYS, regs); + handle_domain_irq(fiq_irqc->domain, TMR_EL0_PHYS, regs); if (TIMER_FIRING(read_sysreg(cntv_ctl_el0))) - handle_domain_irq(aic_irqc->domain, AIC_TMR_EL0_VIRT, regs); + handle_domain_irq(fiq_irqc->domain, TMR_EL0_VIRT, regs); if (is_kernel_in_hyp_mode()) { uint64_t enabled = read_sysreg_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2); if ((enabled & VM_TMR_FIQ_ENABLE_P) && TIMER_FIRING(read_sysreg_s(SYS_CNTP_CTL_EL02))) - handle_domain_irq(aic_irqc->domain, AIC_TMR_EL02_PHYS, regs); + handle_domain_irq(fiq_irqc->domain, TMR_EL02_PHYS, regs); if ((enabled & VM_TMR_FIQ_ENABLE_V) && TIMER_FIRING(read_sysreg_s(SYS_CNTV_CTL_EL02))) - handle_domain_irq(aic_irqc->domain, AIC_TMR_EL02_VIRT, regs); + handle_domain_irq(fiq_irqc->domain, TMR_EL02_VIRT, regs); } if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == @@ -264,7 +262,7 @@ static struct irq_chip fiq_chip = { static int irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct aic_irq_chip *ic = id->host_data; + struct fiq_irq_chip *ic = id->host_data; irq_set_percpu_devid(irq); irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, @@ -278,7 +276,7 @@ static int irq_domain_translate(struct irq_domain *id, unsigned long *hwirq, unsigned int *type) { - struct aic_irq_chip *ic = id->host_data; + struct fiq_irq_chip *ic = id->host_data; if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) return -EINVAL; @@ -358,7 +356,7 @@ static const struct irq_domain_ops irq_domain_ops = { .free = irq_domain_free, }; -static int aic_init_cpu(unsigned int cpu) +static int fiq_init_cpu(unsigned int cpu) { /* Mask all hard-wired per-CPU IRQ/FIQ sources */ @@ -398,7 +396,7 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p int i; void __iomem *regs; u32 info; - struct aic_irq_chip *irqc; + struct fiq_irq_chip *irqc; regs = of_iomap(node, 0); if (WARN_ON(!regs)) @@ -408,8 +406,7 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p if (!irqc) return -ENOMEM; - aic_irqc = irqc; - irqc->base = regs; + fiq_irqc = irqc; irqc->domain = irq_domain_create_linear(of_node_to_fwnode(node), NR_FIQ, &irq_domain_ops, irqc); From 4be21898ac391b3325acb71d53819014fdcdb34b Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:31:44 +0000 Subject: [PATCH 07/31] FIQ cleanup --- drivers/irqchip/irq-apple-fiq.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 4d4b4b35c4d69d..47fdf355d1c17b 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -118,15 +118,10 @@ static struct fiq_irq_chip *fiq_irqc; * FIQ irqchip */ -static unsigned long fiq_get_idx(struct irq_data *d) -{ - return irqd_to_hwirq(d); -} - static void fiq_set_mask(struct irq_data *d) { /* Only the guest timers have real mask bits, unfortunately. */ - switch (fiq_get_idx(d)) { + switch (irqd_to_hwirq(d)) { case TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, VM_TMR_FIQ_ENABLE_P, 0); isb(); @@ -142,7 +137,7 @@ static void fiq_set_mask(struct irq_data *d) static void fiq_clear_mask(struct irq_data *d) { - switch (fiq_get_idx(d)) { + switch (irqd_to_hwirq(d)) { case TMR_EL02_PHYS: sysreg_clear_set_s(SYS_IMP_APL_VM_TMR_FIQ_ENA_EL2, 0, VM_TMR_FIQ_ENABLE_P); isb(); @@ -159,19 +154,19 @@ static void fiq_clear_mask(struct irq_data *d) static void fiq_mask(struct irq_data *d) { fiq_set_mask(d); - __this_cpu_and(fiq_unmasked, ~BIT(fiq_get_idx(d))); + __this_cpu_and(fiq_unmasked, ~BIT(irqd_to_hwirq(d))); } static void fiq_unmask(struct irq_data *d) { fiq_clear_mask(d); - __this_cpu_or(fiq_unmasked, BIT(fiq_get_idx(d))); + __this_cpu_or(fiq_unmasked, BIT(irqd_to_hwirq(d))); } static void fiq_eoi(struct irq_data *d) { /* We mask to ack (where we can), so we need to unmask at EOI. */ - if (__this_cpu_read(fiq_unmasked) & BIT(fiq_get_idx(d))) + if (__this_cpu_read(fiq_unmasked) & BIT(irqd_to_hwirq(d))) fiq_clear_mask(d); } @@ -247,7 +242,7 @@ static int fiq_set_type(struct irq_data *d, unsigned int type) } static struct irq_chip fiq_chip = { - .name = "AIC-FIQ", + .name = "FIQ", .irq_mask = fiq_mask, .irq_unmask = fiq_unmask, .irq_ack = fiq_set_mask, From 8b437d311229a6e46624ec208654bca908d5befd Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:32:06 +0000 Subject: [PATCH 08/31] TEMPORARY hack to build FIQ driver --- drivers/irqchip/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index f88cbf36a9d28d..3d4e246b7e4547 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -115,4 +115,4 @@ obj-$(CONFIG_SL28CPLD_INTC) += irq-sl28cpld.o obj-$(CONFIG_MACH_REALTEK_RTL) += irq-realtek-rtl.o obj-$(CONFIG_WPCM450_AIC) += irq-wpcm450-aic.o obj-$(CONFIG_IRQ_IDT3243X) += irq-idt3243x.o -obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o +obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o irq-apple-fiq.o From e76788d25bb1a4e3196794a17008a04364ebf271 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:41:58 +0000 Subject: [PATCH 09/31] compilation fixes --- drivers/irqchip/irq-apple-fiq.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 47fdf355d1c17b..6581058b5f7bae 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -38,6 +38,8 @@ #include #include +#include + /* * IMP-DEF sysregs that control FIQ sources * Note: sysreg-based IPIs are not supported yet. @@ -101,6 +103,11 @@ * mapping differs and irq_domain_translate() performs the remapping. */ +#define TMR_HV_PHYS AIC_TMR_HV_PHYS +#define TMR_HV_VIRT AIC_TMR_HV_VIRT +#define TMR_GUEST_PHYS AIC_TMR_GUEST_PHYS +#define TMR_GUEST_VIRT AIC_TMR_GUEST_VIRT + #define TMR_EL0_PHYS TMR_HV_PHYS #define TMR_EL0_VIRT TMR_HV_VIRT #define TMR_EL02_PHYS TMR_GUEST_PHYS @@ -257,8 +264,6 @@ static struct irq_chip fiq_chip = { static int irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct fiq_irq_chip *ic = id->host_data; - irq_set_percpu_devid(irq); irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, handle_percpu_devid_irq, NULL, NULL); @@ -271,8 +276,6 @@ static int irq_domain_translate(struct irq_domain *id, unsigned long *hwirq, unsigned int *type) { - struct fiq_irq_chip *ic = id->host_data; - if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) return -EINVAL; @@ -388,15 +391,8 @@ static int fiq_init_cpu(unsigned int cpu) static int __init fiq_of_ic_init(struct device_node *node, struct device_node *parent) { - int i; - void __iomem *regs; - u32 info; struct fiq_irq_chip *irqc; - regs = of_iomap(node, 0); - if (WARN_ON(!regs)) - return -EIO; - irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); if (!irqc) return -ENOMEM; @@ -406,7 +402,6 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p irqc->domain = irq_domain_create_linear(of_node_to_fwnode(node), NR_FIQ, &irq_domain_ops, irqc); if (WARN_ON(!irqc->domain)) { - iounmap(irqc->base); kfree(irqc); return -ENODEV; } From ae64c3ec48f25a98562c4fc90cb6c80afc475ebe Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 15:42:18 +0000 Subject: [PATCH 10/31] TEMPORARY DT entries --- arch/arm64/boot/dts/apple/t8103.dtsi | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index f0c10d1c6cf2ff..d290fde187f826 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -85,9 +85,15 @@ }; }; + fiq: interrupt-controller { + compatible = "apple,fiq"; + #interrupt-cells = <3>; + interrupt-controller; + }; + timer { compatible = "arm,armv8-timer"; - interrupt-parent = <&aic>; + interrupt-parent = <&fiq>; interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; interrupts = , , From 566b0baaf2f83aea3ddba880c487b6a6df5dc889 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:06:27 +0000 Subject: [PATCH 11/31] minor things --- drivers/irqchip/irq-apple-aic.c | 4 +--- drivers/irqchip/irq-apple-fiq.c | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index d085a51e931e63..0833a1be8157ec 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -136,7 +136,7 @@ static void aic_irq_unmask(struct irq_data *d) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(d->hwirq), + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), MASK_BIT(irqd_to_hwirq(d))); } @@ -228,8 +228,6 @@ static struct irq_chip aic_chip = { static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, irq_hw_number_t hw) { - struct aic_irq_chip *ic = id->host_data; - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, handle_fasteoi_irq, NULL, NULL); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 6581058b5f7bae..d4ca9a59976cea 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -182,7 +182,7 @@ static void fiq_eoi(struct irq_data *d) ARCH_TIMER_CTRL_IT_STAT)) == \ (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) -static void __exception_irq_entry handle_fiq(struct pt_regs *regs) +void __exception_irq_entry handle_fiq(struct pt_regs *regs) { /* * It would be really nice if we had a system register that lets us get @@ -406,8 +406,6 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p return -ENODEV; } - irq_domain_update_bus_token(irqc->domain, DOMAIN_BUS_WIRED); - set_handle_fiq(handle_fiq); if (!is_kernel_in_hyp_mode()) From 58d1214c132eb2723a8231e3f0bb72368e399efb Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:06:52 +0000 Subject: [PATCH 12/31] TEMPORARY hack since FIQs aren't masked properly yet. --- drivers/irqchip/irq-apple-aic.c | 3 +++ drivers/irqchip/irq-apple-fiq.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 0833a1be8157ec..afee310d40b623 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -475,8 +475,11 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) return 0; } +extern int fiq_init_cpu(unsigned int cpu); + static int aic_init_cpu(unsigned int cpu) { + fiq_init_cpu(cpu); /* Mask all hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index d4ca9a59976cea..41b1524340fe40 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -354,7 +354,7 @@ static const struct irq_domain_ops irq_domain_ops = { .free = irq_domain_free, }; -static int fiq_init_cpu(unsigned int cpu) +int fiq_init_cpu(unsigned int cpu) { /* Mask all hard-wired per-CPU IRQ/FIQ sources */ From de52b6b946d41517aff2a787c73bea241178f653 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:13:32 +0000 Subject: [PATCH 13/31] initialize CPUs properly --- drivers/irqchip/irq-apple-fiq.c | 4 ++++ include/linux/cpuhotplug.h | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 41b1524340fe40..1a510674367332 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -411,6 +411,10 @@ static int __init fiq_of_ic_init(struct device_node *node, struct device_node *p if (!is_kernel_in_hyp_mode()) pr_info("Kernel running in EL1, mapping interrupts"); + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_FIQ_STARTING, + "irqchip/apple-fiq/fiq:starting", + fiq_init_cpu, NULL); + pr_info("Initialized with %d FIQs\n", NR_FIQ); return 0; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f39b34b1387109..0477ff4603ec50 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -100,6 +100,7 @@ enum cpuhp_state { CPUHP_AP_CPU_PM_STARTING, CPUHP_AP_IRQ_GIC_STARTING, CPUHP_AP_IRQ_HIP04_STARTING, + CPUHP_AP_IRQ_APPLE_FIQ_STARTING, CPUHP_AP_IRQ_APPLE_AIC_STARTING, CPUHP_AP_IRQ_ARMADA_XP_STARTING, CPUHP_AP_IRQ_BCM2836_STARTING, From 1dabd80679226b23f8197c9ea0587cce58acf0c7 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:13:54 +0000 Subject: [PATCH 14/31] undo temporary hack --- drivers/irqchip/irq-apple-aic.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index afee310d40b623..0833a1be8157ec 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -475,11 +475,8 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) return 0; } -extern int fiq_init_cpu(unsigned int cpu); - static int aic_init_cpu(unsigned int cpu) { - fiq_init_cpu(cpu); /* Mask all hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ From 1df1de827315ba33a738e6490436f452ce2a6a8a Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:14:10 +0000 Subject: [PATCH 15/31] rm inappropriate message --- drivers/irqchip/irq-apple-aic.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 0833a1be8157ec..1f1155d488b079 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -560,9 +560,6 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p for (i = 0; i < irqc->nr_hw; i++) aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); - if (!is_kernel_in_hyp_mode()) - pr_info("Kernel running in EL1, mapping interrupts"); - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); From 8332d9f1d2996f03f452ec6f441aaa4774495016 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:20:24 +0000 Subject: [PATCH 16/31] fix the bug @svenpeter42 found. --- drivers/irqchip/irq-apple-aic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 1f1155d488b079..023fec7f9fe328 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -146,7 +146,7 @@ static void aic_irq_eoi(struct irq_data *d) * Reading the interrupt reason automatically acknowledges and masks * the IRQ, so we just unmask it here if needed. */ - if (!irqd_irq_disabled(d) && !irqd_irq_masked(d)) + if (!irqd_irq_masked(d)) aic_irq_unmask(d); } From c47660c2923b9abfa141de58ca0bb4dc0353de55 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 18:28:00 +0000 Subject: [PATCH 17/31] duplicate code --- arch/arm64/kernel/ipi-funnel.c | 575 +++++++++++++++++++++++++++++++++ 1 file changed, 575 insertions(+) create mode 100644 arch/arm64/kernel/ipi-funnel.c diff --git a/arch/arm64/kernel/ipi-funnel.c b/arch/arm64/kernel/ipi-funnel.c new file mode 100644 index 00000000000000..023fec7f9fe328 --- /dev/null +++ b/arch/arm64/kernel/ipi-funnel.c @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +/* + * AIC is a fairly simple interrupt controller with the following features: + * + * - 896 level-triggered hardware IRQs + * - Single mask bit per IRQ + * - Per-IRQ affinity setting + * - Automatic masking on event delivery (auto-ack) + * - Software triggering (ORed with hw line) + * - 2 per-CPU IPIs (meant as "self" and "other", but they are + * interchangeable if not symmetric) + * - Automatic prioritization (single event/ack register per CPU, lower IRQs = + * higher priority) + * - Automatic masking on ack + * - Default "this CPU" register view and explicit per-CPU views + * + * Implementation notes: + * + * - This driver creates two IRQ domains, one for HW IRQs, and one for IPIs. + * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller + * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). + * - DT bindings use 3-cell form (like GIC): + * - <0 nr flags> - hwirq #nr + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * AIC registers (MMIO) + */ + +#define AIC_INFO 0x0004 +#define AIC_INFO_NR_HW GENMASK(15, 0) + +#define AIC_CONFIG 0x0010 + +#define AIC_WHOAMI 0x2000 +#define AIC_EVENT 0x2004 +#define AIC_EVENT_TYPE GENMASK(31, 16) +#define AIC_EVENT_NUM GENMASK(15, 0) + +#define AIC_EVENT_TYPE_HW 1 +#define AIC_EVENT_TYPE_IPI 4 +#define AIC_EVENT_IPI_OTHER 1 +#define AIC_EVENT_IPI_SELF 2 + +#define AIC_IPI_SEND 0x2008 +#define AIC_IPI_ACK 0x200c +#define AIC_IPI_MASK_SET 0x2024 +#define AIC_IPI_MASK_CLR 0x2028 + +#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) + +#define AIC_IPI_OTHER BIT(0) +#define AIC_IPI_SELF BIT(31) + +#define AIC_TARGET_CPU 0x3000 +#define AIC_SW_SET 0x4000 +#define AIC_SW_CLR 0x4080 +#define AIC_MASK_SET 0x4100 +#define AIC_MASK_CLR 0x4180 + +#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) +#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) +#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) + +#define MASK_REG(x) (4 * ((x) >> 5)) +#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) + +#define AIC_NR_SWIPI 32 + +struct aic_irq_chip { + void __iomem *base; + struct irq_domain *hw_domain; + struct irq_domain *ipi_domain; + int nr_hw; + int ipi_hwirq; +}; + +static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); +static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); + +static struct aic_irq_chip *aic_irqc; + +static void aic_handle_ipi(struct pt_regs *regs); + +static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) +{ + return readl_relaxed(ic->base + reg); +} + +static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) +{ + writel_relaxed(val, ic->base + reg); +} + +/* + * IRQ irqchip + */ + +static void aic_irq_mask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + + aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), + MASK_BIT(irqd_to_hwirq(d))); +} + +static void aic_irq_eoi(struct irq_data *d) +{ + /* + * Reading the interrupt reason automatically acknowledges and masks + * the IRQ, so we just unmask it here if needed. + */ + if (!irqd_irq_masked(d)) + aic_irq_unmask(d); +} + +static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) +{ + struct aic_irq_chip *ic = aic_irqc; + u32 event, type, irq; + + do { + /* + * We cannot use a relaxed read here, as reads from DMA buffers + * need to be ordered after the IRQ fires. + */ + event = readl(ic->base + AIC_EVENT); + type = FIELD_GET(AIC_EVENT_TYPE, event); + irq = FIELD_GET(AIC_EVENT_NUM, event); + + if (type == AIC_EVENT_TYPE_HW) + handle_domain_irq(aic_irqc->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI && irq == 1) + aic_handle_ipi(regs); + else if (event != 0) + pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); + } while (event); + + /* + * vGIC maintenance interrupts end up here too, so we need to check + * for them separately. This should never trigger if KVM is working + * properly, because it will have already taken care of clearing it + * on guest exit before this handler runs. + */ + if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && + read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { + pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } +} + +static int aic_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, bool force) +{ + irq_hw_number_t hwirq = irqd_to_hwirq(d); + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + int cpu; + + if (force) + cpu = cpumask_first(mask_val); + else + cpu = cpumask_any_and(mask_val, cpu_online_mask); + + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK; +} + +static int aic_irq_set_type(struct irq_data *d, unsigned int type) +{ + /* + * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't + * have a way to find out the type of any given IRQ, so just allow both. + */ + return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; +} + +static struct irq_chip aic_chip = { + .name = "AIC", + .irq_mask = aic_irq_mask, + .irq_unmask = aic_irq_unmask, + .irq_eoi = aic_irq_eoi, + .irq_set_affinity = aic_irq_set_affinity, + .irq_set_type = aic_irq_set_type, +}; + +/* + * Main IRQ domain + */ + +static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, + irq_hw_number_t hw) +{ + irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); + + return 0; +} + +static int aic_irq_domain_translate(struct irq_domain *id, + struct irq_fwspec *fwspec, + unsigned long *hwirq, + unsigned int *type) +{ + struct aic_irq_chip *ic = id->host_data; + + if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) + return -EINVAL; + + switch (fwspec->param[0]) { + case AIC_IRQ: + if (fwspec->param[1] >= ic->nr_hw) + return -EINVAL; + *hwirq = fwspec->param[1]; + break; + default: + return -EINVAL; + } + + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int i, ret; + + ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = aic_irq_domain_map(domain, virq + i, hwirq + i); + if (ret) + return ret; + } + + return 0; +} + +static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops aic_irq_domain_ops = { + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, +}; + +/* + * IPI irqchip + */ + +static void aic_ipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); +} + +static void aic_ipi_unmask(struct irq_data *d) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) + aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); +} + +static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + u32 send = 0; + int cpu; + unsigned long pending; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in aic_ipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in aic_handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing aic_ipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) + send |= AIC_IPI_SEND_CPU(cpu); + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + aic_ic_write(ic, AIC_IPI_SEND, send); +} + +static struct irq_chip ipi_chip = { + .name = "AIC-IPI", + .irq_mask = aic_ipi_mask, + .irq_unmask = aic_ipi_unmask, + .ipi_send_mask = aic_ipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void aic_handle_ipi(struct pt_regs *regs) +{ + int i; + unsigned long enabled, firing; + + /* + * Ack the IPI. We need to order this after the AIC event read, but + * that is enforced by normal MMIO ordering guarantees. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (which is covered by + * the barrier that is part of the top-level AIC handler's readl()). + */ + enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be + * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, AIC_NR_SWIPI) + handle_domain_irq(aic_irqc->ipi_domain, i, regs); + + /* + * No ordering needed here; at worst this just changes the timing of + * when the next IPI will be delivered. + */ + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); +} + +static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops aic_ipi_domain_ops = { + .alloc = aic_ipi_alloc, + .free = aic_ipi_free, +}; + +static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +{ + struct irq_domain *ipi_domain; + int base_ipi; + + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + &aic_ipi_domain_ops, irqc); + if (WARN_ON(!ipi_domain)) + return -ENODEV; + + ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(!base_ipi)) { + irq_domain_remove(ipi_domain); + return -ENODEV; + } + + set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + + irqc->ipi_domain = ipi_domain; + + return 0; +} + +static int aic_init_cpu(unsigned int cpu) +{ + /* Mask all hard-wired per-CPU IRQ sources */ + + /* EL2-only (VHE mode) IRQ sources */ + if (is_kernel_in_hyp_mode()) { + /* vGIC maintenance IRQ */ + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + } + + /* Commit all of the above */ + isb(); + + /* + * Make sure the kernel's idea of logical CPU order is the same as AIC's + * If we ever end up with a mismatch here, we will have to introduce + * a mapping table similar to what other irqchip drivers do. + */ + WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); + + /* + * Always keep IPIs unmasked at the hardware level (except auto-masking + * by AIC during processing). We manage masks at the vIPI level. + */ + aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + + return 0; +} + +static struct gic_kvm_info vgic_info __initdata = { + .type = GIC_V3, + .no_maint_irq_mask = true, + .no_hw_deactivation = true, +}; + +static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +{ + int i; + void __iomem *regs; + u32 info; + struct aic_irq_chip *irqc; + + regs = of_iomap(node, 0); + if (WARN_ON(!regs)) + return -EIO; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + aic_irqc = irqc; + irqc->base = regs; + + info = aic_ic_read(irqc, AIC_INFO); + irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + + irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), + irqc->nr_hw, + &aic_irq_domain_ops, irqc); + if (WARN_ON(!irqc->hw_domain)) { + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); + + if (aic_init_smp(irqc, node)) { + irq_domain_remove(irqc->hw_domain); + iounmap(irqc->base); + kfree(irqc); + return -ENODEV; + } + + set_handle_irq(aic_handle_irq); + + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); + for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) + aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); + for (i = 0; i < irqc->nr_hw; i++) + aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); + + cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, + "irqchip/apple-aic/ipi:starting", + aic_init_cpu, NULL); + + vgic_set_kvm_info(&vgic_info); + + pr_info("Initialized with %d IRQs, %d vIPIs\n", + irqc->nr_hw, AIC_NR_SWIPI); + + return 0; +} + +IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); From 4820a22c511858c58118642459590721da94ef2a Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 19:38:28 +0000 Subject: [PATCH 18/31] prepare the funneling driver --- arch/arm64/Kconfig | 3 + arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/ipi-funnel.c | 497 ++++++--------------------------- arch/arm64/kernel/smp.c | 13 + 4 files changed, 95 insertions(+), 419 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5b13a932561fc..9a7b03cbd6ae1e 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -314,6 +314,9 @@ config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE config SMP def_bool y +config IPI_FUNNELING + def_bool y + config KERNEL_MODE_NEON def_bool y diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3f1490bfb938a0..fe0ca11b354cc5 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o +obj-$(CONFIG_IPI_FUNNELING) += ipi-funnel.o obj-y += probes/ head-y := head.o diff --git a/arch/arm64/kernel/ipi-funnel.c b/arch/arm64/kernel/ipi-funnel.c index 023fec7f9fe328..a6be85246ccb32 100644 --- a/arch/arm64/kernel/ipi-funnel.c +++ b/arch/arm64/kernel/ipi-funnel.c @@ -8,30 +8,6 @@ * Copyright 2015 Broadcom */ -/* - * AIC is a fairly simple interrupt controller with the following features: - * - * - 896 level-triggered hardware IRQs - * - Single mask bit per IRQ - * - Per-IRQ affinity setting - * - Automatic masking on event delivery (auto-ack) - * - Software triggering (ORed with hw line) - * - 2 per-CPU IPIs (meant as "self" and "other", but they are - * interchangeable if not symmetric) - * - Automatic prioritization (single event/ack register per CPU, lower IRQs = - * higher priority) - * - Automatic masking on ack - * - Default "this CPU" register view and explicit per-CPU views - * - * Implementation notes: - * - * - This driver creates two IRQ domains, one for HW IRQs, and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). - * - DT bindings use 3-cell form (like GIC): - * - <0 nr flags> - hwirq #nr - */ - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include @@ -39,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -48,276 +23,38 @@ #include #include -#include - -/* - * AIC registers (MMIO) - */ - -#define AIC_INFO 0x0004 -#define AIC_INFO_NR_HW GENMASK(15, 0) - -#define AIC_CONFIG 0x0010 - -#define AIC_WHOAMI 0x2000 -#define AIC_EVENT 0x2004 -#define AIC_EVENT_TYPE GENMASK(31, 16) -#define AIC_EVENT_NUM GENMASK(15, 0) - -#define AIC_EVENT_TYPE_HW 1 -#define AIC_EVENT_TYPE_IPI 4 -#define AIC_EVENT_IPI_OTHER 1 -#define AIC_EVENT_IPI_SELF 2 - -#define AIC_IPI_SEND 0x2008 -#define AIC_IPI_ACK 0x200c -#define AIC_IPI_MASK_SET 0x2024 -#define AIC_IPI_MASK_CLR 0x2028 - -#define AIC_IPI_SEND_CPU(cpu) BIT(cpu) - -#define AIC_IPI_OTHER BIT(0) -#define AIC_IPI_SELF BIT(31) - -#define AIC_TARGET_CPU 0x3000 -#define AIC_SW_SET 0x4000 -#define AIC_SW_CLR 0x4080 -#define AIC_MASK_SET 0x4100 -#define AIC_MASK_CLR 0x4180 - -#define AIC_CPU_IPI_SET(cpu) (0x5008 + ((cpu) << 7)) -#define AIC_CPU_IPI_CLR(cpu) (0x500c + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_SET(cpu) (0x5024 + ((cpu) << 7)) -#define AIC_CPU_IPI_MASK_CLR(cpu) (0x5028 + ((cpu) << 7)) - -#define MASK_REG(x) (4 * ((x) >> 5)) -#define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) - -#define AIC_NR_SWIPI 32 - -struct aic_irq_chip { - void __iomem *base; - struct irq_domain *hw_domain; - struct irq_domain *ipi_domain; - int nr_hw; - int ipi_hwirq; -}; - -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); - -static struct aic_irq_chip *aic_irqc; - -static void aic_handle_ipi(struct pt_regs *regs); - -static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) -{ - return readl_relaxed(ic->base + reg); -} - -static void aic_ic_write(struct aic_irq_chip *ic, u32 reg, u32 val) -{ - writel_relaxed(val, ic->base + reg); -} - -/* - * IRQ irqchip - */ - -static void aic_irq_mask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_SET + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_unmask(struct irq_data *d) -{ - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - - aic_ic_write(ic, AIC_MASK_CLR + MASK_REG(irqd_to_hwirq(d)), - MASK_BIT(irqd_to_hwirq(d))); -} - -static void aic_irq_eoi(struct irq_data *d) -{ - /* - * Reading the interrupt reason automatically acknowledges and masks - * the IRQ, so we just unmask it here if needed. - */ - if (!irqd_irq_masked(d)) - aic_irq_unmask(d); -} - -static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) -{ - struct aic_irq_chip *ic = aic_irqc; - u32 event, type, irq; - - do { - /* - * We cannot use a relaxed read here, as reads from DMA buffers - * need to be ordered after the IRQ fires. - */ - event = readl(ic->base + AIC_EVENT); - type = FIELD_GET(AIC_EVENT_TYPE, event); - irq = FIELD_GET(AIC_EVENT_NUM, event); - - if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(aic_irqc->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs); - else if (event != 0) - pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); - } while (event); - - /* - * vGIC maintenance interrupts end up here too, so we need to check - * for them separately. This should never trigger if KVM is working - * properly, because it will have already taken care of clearing it - * on guest exit before this handler runs. - */ - if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && - read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { - pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); - } -} - -static int aic_irq_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) -{ - irq_hw_number_t hwirq = irqd_to_hwirq(d); - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - int cpu; - - if (force) - cpu = cpumask_first(mask_val); - else - cpu = cpumask_any_and(mask_val, cpu_online_mask); - - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); - - return IRQ_SET_MASK_OK; -} - -static int aic_irq_set_type(struct irq_data *d, unsigned int type) -{ - /* - * Some IRQs (e.g. MSIs) implicitly have edge semantics, and we don't - * have a way to find out the type of any given IRQ, so just allow both. - */ - return (type == IRQ_TYPE_LEVEL_HIGH || type == IRQ_TYPE_EDGE_RISING) ? 0 : -EINVAL; -} - -static struct irq_chip aic_chip = { - .name = "AIC", - .irq_mask = aic_irq_mask, - .irq_unmask = aic_irq_unmask, - .irq_eoi = aic_irq_eoi, - .irq_set_affinity = aic_irq_set_affinity, - .irq_set_type = aic_irq_set_type, +struct ipi_funnel_irq_chip { + struct irq_domain *ipi_funnel_domain; + struct irq_data *hwirq; + void (*send_hwipi)(struct cpumask *); }; -/* - * Main IRQ domain - */ - -static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, - irq_hw_number_t hw) -{ - irq_domain_set_info(id, irq, hw, &aic_chip, id->host_data, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); - - return 0; -} - -static int aic_irq_domain_translate(struct irq_domain *id, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) -{ - struct aic_irq_chip *ic = id->host_data; - - if (fwspec->param_count != 3 || !is_of_node(fwspec->fwnode)) - return -EINVAL; - - switch (fwspec->param[0]) { - case AIC_IRQ: - if (fwspec->param[1] >= ic->nr_hw) - return -EINVAL; - *hwirq = fwspec->param[1]; - break; - default: - return -EINVAL; - } +#define NR_SWIPI 32 - *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; +static DEFINE_PER_CPU(atomic_t, vipi_flag); +static DEFINE_PER_CPU(atomic_t, vipi_enable); - return 0; -} - -static int aic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) -{ - unsigned int type = IRQ_TYPE_NONE; - struct irq_fwspec *fwspec = arg; - irq_hw_number_t hwirq; - int i, ret; - - ret = aic_irq_domain_translate(domain, fwspec, &hwirq, &type); - if (ret) - return ret; - - for (i = 0; i < nr_irqs; i++) { - ret = aic_irq_domain_map(domain, virq + i, hwirq + i); - if (ret) - return ret; - } - - return 0; -} - -static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - int i; - - for (i = 0; i < nr_irqs; i++) { - struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); - - irq_set_handler(virq + i, NULL); - irq_domain_reset_irq_data(d); - } -} - -static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, -}; +static struct ipi_funnel_irq_chip *ipi_funnel_irqc; +static void handle_ipi(struct irq_desc *d); /* * IPI irqchip */ -static void aic_ipi_mask(struct irq_data *d) +static void ipi_funnel_mask(struct irq_data *d) { u32 irq_bit = BIT(irqd_to_hwirq(d)); /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); } -static void aic_ipi_unmask(struct irq_data *d) +static void ipi_funnel_unmask(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct ipi_funnel_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); /* * The atomic_or() above must complete before the atomic_read() @@ -329,37 +66,44 @@ static void aic_ipi_unmask(struct irq_data *d) * If a pending vIPI was unmasked, raise a HW IPI to ourselves. * No barriers needed here since this is a self-IPI. */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); + if (atomic_read(this_cpu_ptr(&vipi_flag)) & irq_bit) { + struct cpumask self_mask = { 0, }; + cpumask_set_cpu(smp_processor_id(), &self_mask); + ipi_send_mask(ic->hwirq->irq, &self_mask); + } } -static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) + +static void ipi_funnel_send_mask(struct irq_data *d, const struct cpumask *mask) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct ipi_funnel_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); - u32 send = 0; int cpu; + bool send; unsigned long pending; + struct cpumask sendmask = *mask; for_each_cpu(cpu, mask) { /* - * This sequence is the mirror of the one in aic_ipi_unmask(); + * This sequence is the mirror of the one in ipi_funnel_unmask(); * see the comment there. Additionally, release semantics * ensure that the vIPI flag set is ordered after any shared * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). + * with the atomic_fetch_andnot in handle_ipi(). */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&vipi_flag, cpu)); /* * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). + * atomic_read() below to avoid racing ipi_funnel_unmask(). */ smp_mb__after_atomic(); if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); + (atomic_read(per_cpu_ptr(&vipi_enable, cpu)) & irq_bit)) { + cpumask_set_cpu(cpu, &sendmask); + send = true; + } } /* @@ -369,207 +113,122 @@ static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) * already ordered after the vIPI flag write. */ if (send) - aic_ic_write(ic, AIC_IPI_SEND, send); + ipi_send_mask(ic->hwirq->irq, &sendmask); } -static struct irq_chip ipi_chip = { - .name = "AIC-IPI", - .irq_mask = aic_ipi_mask, - .irq_unmask = aic_ipi_unmask, - .ipi_send_mask = aic_ipi_send_mask, +static struct irq_chip ipi_funnel_chip = { + .name = "IPI", + .irq_mask = ipi_funnel_mask, + .irq_unmask = ipi_funnel_unmask, + .ipi_send_mask = ipi_funnel_send_mask, }; /* * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs) +static void handle_ipi(struct irq_desc *d) { int i; unsigned long enabled, firing; - /* - * Ack the IPI. We need to order this after the AIC event read, but - * that is enforced by normal MMIO ordering guarantees. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); - /* * The mask read does not need to be ordered. Only we can change * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). + * we are properly in the interrupt handler (XXX is this satisfied?). */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); + enabled = atomic_read(this_cpu_ptr(&vipi_enable)); /* * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and + * atomic_fetch_or_release() in ipi_funnel_send_mask(), and needs to be + * ordered after the ic_write() above (to avoid dropping vIPIs) and * before IPI handling code (to avoid races handling vIPIs before they * are signaled). The former is taken care of by the release semantics * of the write portion, while the latter is taken care of by the * acquire semantics of the read portion. */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); + for_each_set_bit(i, &firing, NR_SWIPI) { + struct irq_desc *nd = + irq_resolve_mapping(ipi_funnel_irqc->ipi_funnel_domain, i); - /* - * No ordering needed here; at worst this just changes the timing of - * when the next IPI will be delivered. - */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + handle_irq_desc(nd); + } } -static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) +static int ipi_funnel_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) { int i; for (i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_chip, d->host_data, + irq_domain_set_info(d, virq + i, i, &ipi_funnel_chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void aic_ipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +static void ipi_funnel_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { /* Not freeing IPIs */ + WARN_ON(1); } -static const struct irq_domain_ops aic_ipi_domain_ops = { - .alloc = aic_ipi_alloc, - .free = aic_ipi_free, +static const struct irq_domain_ops ipi_funnel_domain_ops = { + .alloc = ipi_funnel_alloc, + .free = ipi_funnel_free, }; -static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) +static int ipi_funnel_init_smp(struct ipi_funnel_irq_chip *irqc) { - struct irq_domain *ipi_domain; + struct irq_domain *ipi_funnel_domain; int base_ipi; - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, - &aic_ipi_domain_ops, irqc); - if (WARN_ON(!ipi_domain)) - return -ENODEV; + ipi_funnel_domain = irq_domain_create_linear(NULL, NR_SWIPI, + &ipi_funnel_domain_ops, irqc); + if (WARN_ON(!ipi_funnel_domain)) + return -ENOMEM; - ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + ipi_funnel_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_funnel_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(ipi_funnel_domain, -1, NR_SWIPI, NUMA_NO_NODE, NULL, false, NULL); if (WARN_ON(!base_ipi)) { - irq_domain_remove(ipi_domain); - return -ENODEV; - } - - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); - - irqc->ipi_domain = ipi_domain; - - return 0; -} - -static int aic_init_cpu(unsigned int cpu) -{ - /* Mask all hard-wired per-CPU IRQ sources */ - - /* EL2-only (VHE mode) IRQ sources */ - if (is_kernel_in_hyp_mode()) { - /* vGIC maintenance IRQ */ - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + irq_domain_remove(ipi_funnel_domain); + return -ENOMEM; } - /* Commit all of the above */ - isb(); - - /* - * Make sure the kernel's idea of logical CPU order is the same as AIC's - * If we ever end up with a mismatch here, we will have to introduce - * a mapping table similar to what other irqchip drivers do. - */ - WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); + set_smp_ipi_range(base_ipi, NR_SWIPI); - /* - * Always keep IPIs unmasked at the hardware level (except auto-masking - * by AIC during processing). We manage masks at the vIPI level. - */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); - aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + irqc->ipi_funnel_domain = ipi_funnel_domain; return 0; } -static struct gic_kvm_info vgic_info __initdata = { - .type = GIC_V3, - .no_maint_irq_mask = true, - .no_hw_deactivation = true, -}; - -static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) +int __init ipi_funnel_init(struct irq_data *hwirq) { - int i; - void __iomem *regs; - u32 info; - struct aic_irq_chip *irqc; - - regs = of_iomap(node, 0); - if (WARN_ON(!regs)) - return -EIO; + struct ipi_funnel_irq_chip *irqc; irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); if (!irqc) return -ENOMEM; - aic_irqc = irqc; - irqc->base = regs; - - info = aic_ic_read(irqc, AIC_INFO); - irqc->nr_hw = FIELD_GET(AIC_INFO_NR_HW, info); + irqc->hwirq = hwirq; - irqc->hw_domain = irq_domain_create_linear(of_node_to_fwnode(node), - irqc->nr_hw, - &aic_irq_domain_ops, irqc); - if (WARN_ON(!irqc->hw_domain)) { - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; - } - - irq_domain_update_bus_token(irqc->hw_domain, DOMAIN_BUS_WIRED); - - if (aic_init_smp(irqc, node)) { - irq_domain_remove(irqc->hw_domain); - iounmap(irqc->base); - kfree(irqc); - return -ENODEV; - } - - set_handle_irq(aic_handle_irq); - - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_MASK_SET + i * 4, U32_MAX); - for (i = 0; i < BITS_TO_U32(irqc->nr_hw); i++) - aic_ic_write(irqc, AIC_SW_CLR + i * 4, U32_MAX); - for (i = 0; i < irqc->nr_hw; i++) - aic_ic_write(irqc, AIC_TARGET_CPU + i * 4, 1); + if (ipi_funnel_init_smp(irqc)) + return -ENOMEM; - cpuhp_setup_state(CPUHP_AP_IRQ_APPLE_AIC_STARTING, - "irqchip/apple-aic/ipi:starting", - aic_init_cpu, NULL); + ipi_funnel_irqc = irqc; - vgic_set_kvm_info(&vgic_info); + irq_set_handler_locked(hwirq, handle_ipi); - pr_info("Initialized with %d IRQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_SWIPI); + pr_info("Initialized with %d vIPIs\n", NR_SWIPI); return 0; } - -IRQCHIP_DECLARE(apple_m1_aic, "apple,aic", aic_of_ic_init); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 6f6ff072acbde7..11fb7209551f93 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -979,10 +979,23 @@ static void ipi_teardown(int cpu) } #endif +#ifdef CONFIG_IPI_FUNNELING +extern int __init ipi_funnel_init(struct irq_data *hwirq); +#else +#define ipi_funnel_init(irqdata) (-EINVAL) +#endif + void __init set_smp_ipi_range(int ipi_base, int n) { int i; + if (n < NR_IPI) { + int ret; + BUG_ON(n < 1); + ret = ipi_funnel_init(irq_get_irq_data(ipi_base)); + if (ret >= 0) + return; + } WARN_ON(n < NR_IPI); nr_ipi = min(n, NR_IPI); From db0fca9c0d93ee53f22e42e3d407a9338a195acf Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 20:00:50 +0000 Subject: [PATCH 19/31] still hoping for a negative diffstat --- arch/arm64/kernel/ipi-funnel.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kernel/ipi-funnel.c b/arch/arm64/kernel/ipi-funnel.c index a6be85246ccb32..efce8aa7f5a39f 100644 --- a/arch/arm64/kernel/ipi-funnel.c +++ b/arch/arm64/kernel/ipi-funnel.c @@ -8,8 +8,6 @@ * Copyright 2015 Broadcom */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include #include #include From c7a80135a3fd841aa85c23afe8e707cbad6dc3a9 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Tue, 10 Aug 2021 20:01:26 +0000 Subject: [PATCH 20/31] remove the vIPI implementation --- drivers/irqchip/irq-apple-aic.c | 108 +++++--------------------------- 1 file changed, 16 insertions(+), 92 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 023fec7f9fe328..0aa3b9cc489140 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -26,8 +26,9 @@ * Implementation notes: * * - This driver creates two IRQ domains, one for HW IRQs, and one for IPIs. - * - Since Linux needs more than 2 IPIs, we implement a software IRQ controller - * and funnel all IPIs into one per-CPU IPI (the second "self" IPI is unused). + * - Since Linux needs more than 2 IPIs, we rely on the arch IRQ layer + * to funnel IPIs through its own implementation, using just one + * per-CPU real IPI (the second "self" IPI is unused). * - DT bindings use 3-cell form (like GIC): * - <0 nr flags> - hwirq #nr */ @@ -93,22 +94,18 @@ #define MASK_REG(x) (4 * ((x) >> 5)) #define MASK_BIT(x) BIT((x) & GENMASK(4, 0)) -#define AIC_NR_SWIPI 32 - struct aic_irq_chip { void __iomem *base; struct irq_domain *hw_domain; struct irq_domain *ipi_domain; int nr_hw; - int ipi_hwirq; }; -static DEFINE_PER_CPU(atomic_t, aic_vipi_flag); -static DEFINE_PER_CPU(atomic_t, aic_vipi_enable); +#define AIC_NR_IPI 2 static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs); +static void aic_handle_ipi(struct pt_regs *regs, int index); static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) { @@ -167,7 +164,7 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) if (type == AIC_EVENT_TYPE_HW) handle_domain_irq(aic_irqc->hw_domain, irq, regs); else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs); + aic_handle_ipi(regs, 0); else if (event != 0) pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); } while (event); @@ -306,68 +303,23 @@ static const struct irq_domain_ops aic_irq_domain_ops = { static void aic_ipi_mask(struct irq_data *d) { - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&aic_vipi_enable)); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_OTHER); } static void aic_ipi_unmask(struct irq_data *d) { - struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&aic_vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&aic_vipi_flag)) & irq_bit) - aic_ic_write(ic, AIC_IPI_SEND, AIC_IPI_SEND_CPU(smp_processor_id())); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); u32 send = 0; int cpu; - unsigned long pending; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in aic_ipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in aic_handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&aic_vipi_flag, cpu)); - - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing aic_ipi_unmask(). - */ - smp_mb__after_atomic(); - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&aic_vipi_enable, cpu)) & irq_bit)) - send |= AIC_IPI_SEND_CPU(cpu); - } + for_each_cpu(cpu, mask) + send |= AIC_IPI_SEND_CPU(cpu); - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ if (send) aic_ic_write(ic, AIC_IPI_SEND, send); } @@ -383,38 +335,15 @@ static struct irq_chip ipi_chip = { * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs) +static void aic_handle_ipi(struct pt_regs *regs, int index) { - int i; - unsigned long enabled, firing; - /* * Ack the IPI. We need to order this after the AIC event read, but * that is enforced by normal MMIO ordering guarantees. */ aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (which is covered by - * the barrier that is part of the top-level AIC handler's readl()). - */ - enabled = atomic_read(this_cpu_ptr(&aic_vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in aic_ipi_send_mask(), and needs to be - * ordered after the aic_ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&aic_vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, AIC_NR_SWIPI) - handle_domain_irq(aic_irqc->ipi_domain, i, regs); + handle_domain_irq(aic_irqc->ipi_domain, index, regs); /* * No ordering needed here; at worst this just changes the timing of @@ -452,7 +381,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) struct irq_domain *ipi_domain; int base_ipi; - ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_SWIPI, + ipi_domain = irq_domain_create_linear(irqc->hw_domain->fwnode, AIC_NR_IPI, &aic_ipi_domain_ops, irqc); if (WARN_ON(!ipi_domain)) return -ENODEV; @@ -460,7 +389,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, AIC_NR_IPI, NUMA_NO_NODE, NULL, false, NULL); if (WARN_ON(!base_ipi)) { @@ -468,7 +397,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) return -ENODEV; } - set_smp_ipi_range(base_ipi, AIC_NR_SWIPI); + set_smp_ipi_range(base_ipi, AIC_NR_IPI); irqc->ipi_domain = ipi_domain; @@ -495,10 +424,6 @@ static int aic_init_cpu(unsigned int cpu) */ WARN_ON(aic_ic_read(aic_irqc, AIC_WHOAMI) != smp_processor_id()); - /* - * Always keep IPIs unmasked at the hardware level (except auto-masking - * by AIC during processing). We manage masks at the vIPI level. - */ aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_SELF | AIC_IPI_OTHER); aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_SELF); aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); @@ -566,8 +491,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p vgic_set_kvm_info(&vgic_info); - pr_info("Initialized with %d IRQs, %d vIPIs\n", - irqc->nr_hw, AIC_NR_SWIPI); + pr_info("Initialized with %d IRQs, 1 IPI\n", irqc->nr_hw); return 0; } From 051e9f7894d4ddac693434a12f32e85816412622 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:44:27 +0000 Subject: [PATCH 21/31] minor --- drivers/irqchip/irq-apple-aic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 0aa3b9cc489140..c97c4c3d63eecc 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -162,7 +162,7 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) irq = FIELD_GET(AIC_EVENT_NUM, event); if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(aic_irqc->hw_domain, irq, regs); + handle_domain_irq(ic->hw_domain, irq, regs); else if (type == AIC_EVENT_TYPE_IPI && irq == 1) aic_handle_ipi(regs, 0); else if (event != 0) From 605c9f7a041593972e98ae6f878af0393cc7b336 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:45:11 +0000 Subject: [PATCH 22/31] comments --- drivers/irqchip/irq-apple-fiq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index 1a510674367332..cd313af16bb45b 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -315,7 +315,7 @@ static int irq_domain_translate(struct irq_domain *id, } static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) + unsigned int nr_irqs, void *arg) { unsigned int type = IRQ_TYPE_NONE; struct irq_fwspec *fwspec = arg; @@ -336,7 +336,7 @@ static int irq_domain_alloc(struct irq_domain *domain, unsigned int virq, } static void irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) + unsigned int nr_irqs) { int i; @@ -356,7 +356,7 @@ static const struct irq_domain_ops irq_domain_ops = { int fiq_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ/FIQ sources */ + /* Mask all hard-wired per-CPU FIQ sources */ /* Pending Fast IPI FIQs */ write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); From 7235b1bf0897c6d47e4c0cb43c2d490277c580cc Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:45:56 +0000 Subject: [PATCH 23/31] comments --- drivers/irqchip/irq-apple-aic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index c97c4c3d63eecc..433a46ea8269bb 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -406,7 +406,7 @@ static int aic_init_smp(struct aic_irq_chip *irqc, struct device_node *node) static int aic_init_cpu(unsigned int cpu) { - /* Mask all hard-wired per-CPU IRQ sources */ + /* Mask hard-wired per-CPU IRQ sources */ /* EL2-only (VHE mode) IRQ sources */ if (is_kernel_in_hyp_mode()) { @@ -414,7 +414,7 @@ static int aic_init_cpu(unsigned int cpu) sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); } - /* Commit all of the above */ + /* Commit the above */ isb(); /* From 88c6ef956f0f87bda2deeb91ba5837cdb8c45354 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:49:47 +0000 Subject: [PATCH 24/31] whitespace --- drivers/irqchip/irq-apple-fiq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-apple-fiq.c b/drivers/irqchip/irq-apple-fiq.c index cd313af16bb45b..c8c9f74ca465f8 100644 --- a/drivers/irqchip/irq-apple-fiq.c +++ b/drivers/irqchip/irq-apple-fiq.c @@ -177,9 +177,9 @@ static void fiq_eoi(struct irq_data *d) fiq_clear_mask(d); } -#define TIMER_FIRING(x) \ - (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ - ARCH_TIMER_CTRL_IT_STAT)) == \ +#define TIMER_FIRING(x) \ + (((x) & (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_MASK | \ + ARCH_TIMER_CTRL_IT_STAT)) == \ (ARCH_TIMER_CTRL_ENABLE | ARCH_TIMER_CTRL_IT_STAT)) void __exception_irq_entry handle_fiq(struct pt_regs *regs) From 7ccd8ecacd479a079f60138a15060ca1fa8afab0 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:55:05 +0000 Subject: [PATCH 25/31] rename ipi-funnel to vipi --- arch/arm64/kernel/vipi.c | 232 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 arch/arm64/kernel/vipi.c diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c new file mode 100644 index 00000000000000..00041fb29b23ca --- /dev/null +++ b/arch/arm64/kernel/vipi.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright The Asahi Linux Contributors + * + * Based on irq-lpc32xx: + * Copyright 2015-2016 Vladimir Zapolskiy + * Based on irq-bcm2836: + * Copyright 2015 Broadcom + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct vipi_irq_chip { + struct irq_domain *vipi_domain; + struct irq_data *hwirq; + void (*send_hwipi)(struct cpumask *); +}; + +#define NR_SWIPI 32 + +static DEFINE_PER_CPU(atomic_t, vipi_flag); +static DEFINE_PER_CPU(atomic_t, vipi_enable); + +static struct vipi_irq_chip *vipi_irqc; + +static void handle_ipi(struct irq_desc *d); +/* + * IPI irqchip + */ + +static void vipi_mask(struct irq_data *d) +{ + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + /* No specific ordering requirements needed here. */ + atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); +} + +static void vipi_unmask(struct irq_data *d) +{ + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + + atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); + + /* + * The atomic_or() above must complete before the atomic_read() + * below to avoid racing aic_ipi_send_mask(). + */ + smp_mb__after_atomic(); + + /* + * If a pending vIPI was unmasked, raise a HW IPI to ourselves. + * No barriers needed here since this is a self-IPI. + */ + if (atomic_read(this_cpu_ptr(&vipi_flag)) & irq_bit) { + struct cpumask self_mask = { 0, }; + cpumask_set_cpu(smp_processor_id(), &self_mask); + ipi_send_mask(ic->hwirq->irq, &self_mask); + } +} + + +static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); + u32 irq_bit = BIT(irqd_to_hwirq(d)); + int cpu; + bool send; + unsigned long pending; + struct cpumask sendmask = *mask; + + for_each_cpu(cpu, mask) { + /* + * This sequence is the mirror of the one in vipi_unmask(); + * see the comment there. Additionally, release semantics + * ensure that the vIPI flag set is ordered after any shared + * memory accesses that precede it. This therefore also pairs + * with the atomic_fetch_andnot in handle_ipi(). + */ + pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&vipi_flag, cpu)); + + /* + * The atomic_fetch_or_release() above must complete before the + * atomic_read() below to avoid racing vipi_unmask(). + */ + smp_mb__after_atomic(); + + if (!(pending & irq_bit) && + (atomic_read(per_cpu_ptr(&vipi_enable, cpu)) & irq_bit)) { + cpumask_set_cpu(cpu, &sendmask); + send = true; + } + } + + /* + * The flag writes must complete before the physical IPI is issued + * to another CPU. This is implied by the control dependency on + * the result of atomic_read_acquire() above, which is itself + * already ordered after the vIPI flag write. + */ + if (send) + ipi_send_mask(ic->hwirq->irq, &sendmask); +} + +static struct irq_chip vipi_chip = { + .name = "IPI", + .irq_mask = vipi_mask, + .irq_unmask = vipi_unmask, + .ipi_send_mask = vipi_send_mask, +}; + +/* + * IPI IRQ domain + */ + +static void handle_ipi(struct irq_desc *d) +{ + int i; + unsigned long enabled, firing; + + /* + * The mask read does not need to be ordered. Only we can change + * our own mask anyway, so no races are possible here, as long as + * we are properly in the interrupt handler (XXX is this satisfied?). + */ + enabled = atomic_read(this_cpu_ptr(&vipi_enable)); + + /* + * Clear the IPIs we are about to handle. This pairs with the + * atomic_fetch_or_release() in vipi_send_mask(), and needs to be + * ordered after the ic_write() above (to avoid dropping vIPIs) and + * before IPI handling code (to avoid races handling vIPIs before they + * are signaled). The former is taken care of by the release semantics + * of the write portion, while the latter is taken care of by the + * acquire semantics of the read portion. + */ + firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; + + for_each_set_bit(i, &firing, NR_SWIPI) { + struct irq_desc *nd = + irq_resolve_mapping(vipi_irqc->domain, i); + + handle_irq_desc(nd); + } +} + +static int vipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &vipi_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + } + + return 0; +} + +static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + /* Not freeing IPIs */ + WARN_ON(1); +} + +static const struct irq_domain_ops vipi_domain_ops = { + .alloc = vipi_alloc, + .free = vipi_free, +}; + +static int vipi_init_smp(struct vipi_irq_chip *irqc) +{ + struct irq_domain *vipi_domain; + int base_ipi; + + vipi_domain = irq_domain_create_linear(NULL, NR_SWIPI, + &vipi_domain_ops, irqc); + if (WARN_ON(!vipi_domain)) + return -ENOMEM; + + vipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(vipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(vipi_domain, -1, NR_SWIPI, + NUMA_NO_NODE, NULL, false, NULL); + + if (WARN_ON(!base_ipi)) { + irq_domain_remove(vipi_domain); + return -ENOMEM; + } + + set_smp_ipi_range(base_ipi, NR_SWIPI); + + irqc->domain = vipi_domain; + + return 0; +} + +int __init vipi_init(struct irq_data *hwirq) +{ + struct vipi_irq_chip *irqc; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + irqc->hwirq = hwirq; + + if (vipi_init_smp(irqc)) + return -ENOMEM; + + vipi_irqc = irqc; + + irq_set_handler_locked(hwirq, handle_ipi); + + pr_info("Initialized with %d vIPIs\n", NR_SWIPI); + + return 0; +} From 8f33c22a5d566545f88d9a9c73d067bc02448f7f Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:55:47 +0000 Subject: [PATCH 26/31] rename ipi-funnel to vipi --- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/ipi-funnel.c | 72 +++++++++++++++++----------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fe0ca11b354cc5..ed58d15d484c2e 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -73,7 +73,7 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o obj-$(CONFIG_ARM64_MTE) += mte.o obj-y += vdso-wrap.o obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o -obj-$(CONFIG_IPI_FUNNELING) += ipi-funnel.o +obj-$(CONFIG_IPI_FUNNELING) += vipi.o obj-y += probes/ head-y := head.o diff --git a/arch/arm64/kernel/ipi-funnel.c b/arch/arm64/kernel/ipi-funnel.c index efce8aa7f5a39f..00041fb29b23ca 100644 --- a/arch/arm64/kernel/ipi-funnel.c +++ b/arch/arm64/kernel/ipi-funnel.c @@ -21,8 +21,8 @@ #include #include -struct ipi_funnel_irq_chip { - struct irq_domain *ipi_funnel_domain; +struct vipi_irq_chip { + struct irq_domain *vipi_domain; struct irq_data *hwirq; void (*send_hwipi)(struct cpumask *); }; @@ -32,14 +32,14 @@ struct ipi_funnel_irq_chip { static DEFINE_PER_CPU(atomic_t, vipi_flag); static DEFINE_PER_CPU(atomic_t, vipi_enable); -static struct ipi_funnel_irq_chip *ipi_funnel_irqc; +static struct vipi_irq_chip *vipi_irqc; static void handle_ipi(struct irq_desc *d); /* * IPI irqchip */ -static void ipi_funnel_mask(struct irq_data *d) +static void vipi_mask(struct irq_data *d) { u32 irq_bit = BIT(irqd_to_hwirq(d)); @@ -47,9 +47,9 @@ static void ipi_funnel_mask(struct irq_data *d) atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); } -static void ipi_funnel_unmask(struct irq_data *d) +static void vipi_unmask(struct irq_data *d) { - struct ipi_funnel_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); @@ -72,9 +72,9 @@ static void ipi_funnel_unmask(struct irq_data *d) } -static void ipi_funnel_send_mask(struct irq_data *d, const struct cpumask *mask) +static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) { - struct ipi_funnel_irq_chip *ic = irq_data_get_irq_chip_data(d); + struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); u32 irq_bit = BIT(irqd_to_hwirq(d)); int cpu; bool send; @@ -83,7 +83,7 @@ static void ipi_funnel_send_mask(struct irq_data *d, const struct cpumask *mask) for_each_cpu(cpu, mask) { /* - * This sequence is the mirror of the one in ipi_funnel_unmask(); + * This sequence is the mirror of the one in vipi_unmask(); * see the comment there. Additionally, release semantics * ensure that the vIPI flag set is ordered after any shared * memory accesses that precede it. This therefore also pairs @@ -93,7 +93,7 @@ static void ipi_funnel_send_mask(struct irq_data *d, const struct cpumask *mask) /* * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing ipi_funnel_unmask(). + * atomic_read() below to avoid racing vipi_unmask(). */ smp_mb__after_atomic(); @@ -114,11 +114,11 @@ static void ipi_funnel_send_mask(struct irq_data *d, const struct cpumask *mask) ipi_send_mask(ic->hwirq->irq, &sendmask); } -static struct irq_chip ipi_funnel_chip = { +static struct irq_chip vipi_chip = { .name = "IPI", - .irq_mask = ipi_funnel_mask, - .irq_unmask = ipi_funnel_unmask, - .ipi_send_mask = ipi_funnel_send_mask, + .irq_mask = vipi_mask, + .irq_unmask = vipi_unmask, + .ipi_send_mask = vipi_send_mask, }; /* @@ -139,7 +139,7 @@ static void handle_ipi(struct irq_desc *d) /* * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in ipi_funnel_send_mask(), and needs to be + * atomic_fetch_or_release() in vipi_send_mask(), and needs to be * ordered after the ic_write() above (to avoid dropping vIPIs) and * before IPI handling code (to avoid races handling vIPIs before they * are signaled). The former is taken care of by the release semantics @@ -150,68 +150,68 @@ static void handle_ipi(struct irq_desc *d) for_each_set_bit(i, &firing, NR_SWIPI) { struct irq_desc *nd = - irq_resolve_mapping(ipi_funnel_irqc->ipi_funnel_domain, i); + irq_resolve_mapping(vipi_irqc->domain, i); handle_irq_desc(nd); } } -static int ipi_funnel_alloc(struct irq_domain *d, unsigned int virq, +static int vipi_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs, void *args) { int i; for (i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_funnel_chip, d->host_data, + irq_domain_set_info(d, virq + i, i, &vipi_chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void ipi_funnel_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { /* Not freeing IPIs */ WARN_ON(1); } -static const struct irq_domain_ops ipi_funnel_domain_ops = { - .alloc = ipi_funnel_alloc, - .free = ipi_funnel_free, +static const struct irq_domain_ops vipi_domain_ops = { + .alloc = vipi_alloc, + .free = vipi_free, }; -static int ipi_funnel_init_smp(struct ipi_funnel_irq_chip *irqc) +static int vipi_init_smp(struct vipi_irq_chip *irqc) { - struct irq_domain *ipi_funnel_domain; + struct irq_domain *vipi_domain; int base_ipi; - ipi_funnel_domain = irq_domain_create_linear(NULL, NR_SWIPI, - &ipi_funnel_domain_ops, irqc); - if (WARN_ON(!ipi_funnel_domain)) + vipi_domain = irq_domain_create_linear(NULL, NR_SWIPI, + &vipi_domain_ops, irqc); + if (WARN_ON(!vipi_domain)) return -ENOMEM; - ipi_funnel_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(ipi_funnel_domain, DOMAIN_BUS_IPI); + vipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(vipi_domain, DOMAIN_BUS_IPI); - base_ipi = __irq_domain_alloc_irqs(ipi_funnel_domain, -1, NR_SWIPI, + base_ipi = __irq_domain_alloc_irqs(vipi_domain, -1, NR_SWIPI, NUMA_NO_NODE, NULL, false, NULL); if (WARN_ON(!base_ipi)) { - irq_domain_remove(ipi_funnel_domain); + irq_domain_remove(vipi_domain); return -ENOMEM; } set_smp_ipi_range(base_ipi, NR_SWIPI); - irqc->ipi_funnel_domain = ipi_funnel_domain; + irqc->domain = vipi_domain; return 0; } -int __init ipi_funnel_init(struct irq_data *hwirq) +int __init vipi_init(struct irq_data *hwirq) { - struct ipi_funnel_irq_chip *irqc; + struct vipi_irq_chip *irqc; irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); if (!irqc) @@ -219,10 +219,10 @@ int __init ipi_funnel_init(struct irq_data *hwirq) irqc->hwirq = hwirq; - if (ipi_funnel_init_smp(irqc)) + if (vipi_init_smp(irqc)) return -ENOMEM; - ipi_funnel_irqc = irqc; + vipi_irqc = irqc; irq_set_handler_locked(hwirq, handle_ipi); From 6ada2b7da117c838bd1a23d3ae3f7da8763cf00f Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 07:59:07 +0000 Subject: [PATCH 27/31] more renaming --- arch/arm64/kernel/smp.c | 9 ++++++--- arch/arm64/kernel/vipi.c | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 11fb7209551f93..ac5f26281d347c 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -980,9 +980,12 @@ static void ipi_teardown(int cpu) #endif #ifdef CONFIG_IPI_FUNNELING -extern int __init ipi_funnel_init(struct irq_data *hwirq); +extern int __init vipi_init(struct irq_data *hwirq); #else -#define ipi_funnel_init(irqdata) (-EINVAL) +static inline int vipi_init(irqdata) +{ + return -EINVAL; +} #endif void __init set_smp_ipi_range(int ipi_base, int n) @@ -992,7 +995,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) if (n < NR_IPI) { int ret; BUG_ON(n < 1); - ret = ipi_funnel_init(irq_get_irq_data(ipi_base)); + ret = vipi_init(irq_get_irq_data(ipi_base)); if (ret >= 0) return; } diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c index 00041fb29b23ca..b7fbdb766b646a 100644 --- a/arch/arm64/kernel/vipi.c +++ b/arch/arm64/kernel/vipi.c @@ -115,7 +115,7 @@ static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) } static struct irq_chip vipi_chip = { - .name = "IPI", + .name = "VIPI", .irq_mask = vipi_mask, .irq_unmask = vipi_unmask, .ipi_send_mask = vipi_send_mask, From be45284d47d44425a938c37e32f607a5c1bcc072 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 09:48:38 +0000 Subject: [PATCH 28/31] cleanup --- arch/arm64/kernel/vipi.c | 3 +-- drivers/irqchip/irq-apple-aic.c | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kernel/vipi.c b/arch/arm64/kernel/vipi.c index b7fbdb766b646a..01fb505e1e25a2 100644 --- a/arch/arm64/kernel/vipi.c +++ b/arch/arm64/kernel/vipi.c @@ -22,9 +22,8 @@ #include struct vipi_irq_chip { - struct irq_domain *vipi_domain; + struct irq_domain *domain; struct irq_data *hwirq; - void (*send_hwipi)(struct cpumask *); }; #define NR_SWIPI 32 diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 433a46ea8269bb..14b04f24c7bddd 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -105,7 +105,7 @@ struct aic_irq_chip { static struct aic_irq_chip *aic_irqc; -static void aic_handle_ipi(struct pt_regs *regs, int index); +static void aic_handle_ipi(int index, struct pt_regs *regs); static u32 aic_ic_read(struct aic_irq_chip *ic, u32 reg) { @@ -301,14 +301,19 @@ static const struct irq_domain_ops aic_irq_domain_ops = { * IPI irqchip */ +static int aic_ipi_number(struct irq_data *d) +{ + return irqd_to_hwirq(d) ? AIC_IPI_SELF : AIC_IPI_OTHER; +} + static void aic_ipi_mask(struct irq_data *d) { - aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_SET, aic_ipi_number(d)); } static void aic_ipi_unmask(struct irq_data *d) { - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, aic_ipi_number(d)); } static void aic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) @@ -335,21 +340,24 @@ static struct irq_chip ipi_chip = { * IPI IRQ domain */ -static void aic_handle_ipi(struct pt_regs *regs, int index) +static void aic_handle_ipi(int index, struct pt_regs *regs) { + struct irq_domain *domain = aic_irqc->ipi_domain; + struct aic_irq_chip *ic = aic_irqc; /* * Ack the IPI. We need to order this after the AIC event read, but * that is enforced by normal MMIO ordering guarantees. */ - aic_ic_write(aic_irqc, AIC_IPI_ACK, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_ACK, + aic_ipi_number(irq_domain_get_irq_data(domain, index))); - handle_domain_irq(aic_irqc->ipi_domain, index, regs); + handle_domain_irq(domain, index, regs); /* * No ordering needed here; at worst this just changes the timing of * when the next IPI will be delivered. */ - aic_ic_write(aic_irqc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); + aic_ic_write(ic, AIC_IPI_MASK_CLR, AIC_IPI_OTHER); } static int aic_ipi_alloc(struct irq_domain *d, unsigned int virq, From 93edc61c1df6e0626d28eecf00376839eb935c93 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 09:49:09 +0000 Subject: [PATCH 29/31] allow multi-CPU affinities --- drivers/irqchip/irq-apple-aic.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 14b04f24c7bddd..79243e27dda4d9 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -188,14 +188,13 @@ static int aic_irq_set_affinity(struct irq_data *d, irq_hw_number_t hwirq = irqd_to_hwirq(d); struct aic_irq_chip *ic = irq_data_get_irq_chip_data(d); int cpu; + u32 mask = 0; - if (force) - cpu = cpumask_first(mask_val); - else - cpu = cpumask_any_and(mask_val, cpu_online_mask); + for_each_cpu(cpu, mask_val) + mask |= BIT(cpu); - aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, BIT(cpu)); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); + aic_ic_write(ic, AIC_TARGET_CPU + hwirq * 4, mask); + irq_data_update_effective_affinity(d, mask_val); return IRQ_SET_MASK_OK; } From 7a7bfa99b6c566e14dcd6865fb013f9ff5b9d226 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 09:49:35 +0000 Subject: [PATCH 30/31] handle only a single IRQ event at a time --- drivers/irqchip/irq-apple-aic.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 79243e27dda4d9..87ef7dae4a67a0 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -152,28 +152,28 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) struct aic_irq_chip *ic = aic_irqc; u32 event, type, irq; - do { - /* - * We cannot use a relaxed read here, as reads from DMA buffers - * need to be ordered after the IRQ fires. - */ - event = readl(ic->base + AIC_EVENT); - type = FIELD_GET(AIC_EVENT_TYPE, event); - irq = FIELD_GET(AIC_EVENT_NUM, event); - - if (type == AIC_EVENT_TYPE_HW) - handle_domain_irq(ic->hw_domain, irq, regs); - else if (type == AIC_EVENT_TYPE_IPI && irq == 1) - aic_handle_ipi(regs, 0); - else if (event != 0) - pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); - } while (event); + /* + * We cannot use a relaxed read here, as reads from DMA buffers + * need to be ordered after the IRQ fires. + */ + event = readl(ic->base + AIC_EVENT); + type = FIELD_GET(AIC_EVENT_TYPE, event); + irq = FIELD_GET(AIC_EVENT_NUM, event); + + if (type == AIC_EVENT_TYPE_HW) + handle_domain_irq(ic->hw_domain, irq, regs); + else if (type == AIC_EVENT_TYPE_IPI) + aic_handle_ipi(irq, regs); + else if (event != 0) + pr_err_ratelimited("Unknown IRQ event %d, %d\n", type, irq); /* * vGIC maintenance interrupts end up here too, so we need to check * for them separately. This should never trigger if KVM is working * properly, because it will have already taken care of clearing it * on guest exit before this handler runs. + * + * XXX it would be nice to skip this check. */ if (is_kernel_in_hyp_mode() && (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { From 1bda904559c32f1d5847520f022eb92c5ada0cb8 Mon Sep 17 00:00:00 2001 From: Pip Cet Date: Wed, 11 Aug 2021 10:02:07 +0000 Subject: [PATCH 31/31] delete file --- arch/arm64/kernel/ipi-funnel.c | 232 --------------------------------- 1 file changed, 232 deletions(-) delete mode 100644 arch/arm64/kernel/ipi-funnel.c diff --git a/arch/arm64/kernel/ipi-funnel.c b/arch/arm64/kernel/ipi-funnel.c deleted file mode 100644 index 00041fb29b23ca..00000000000000 --- a/arch/arm64/kernel/ipi-funnel.c +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright The Asahi Linux Contributors - * - * Based on irq-lpc32xx: - * Copyright 2015-2016 Vladimir Zapolskiy - * Based on irq-bcm2836: - * Copyright 2015 Broadcom - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct vipi_irq_chip { - struct irq_domain *vipi_domain; - struct irq_data *hwirq; - void (*send_hwipi)(struct cpumask *); -}; - -#define NR_SWIPI 32 - -static DEFINE_PER_CPU(atomic_t, vipi_flag); -static DEFINE_PER_CPU(atomic_t, vipi_enable); - -static struct vipi_irq_chip *vipi_irqc; - -static void handle_ipi(struct irq_desc *d); -/* - * IPI irqchip - */ - -static void vipi_mask(struct irq_data *d) -{ - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - /* No specific ordering requirements needed here. */ - atomic_andnot(irq_bit, this_cpu_ptr(&vipi_enable)); -} - -static void vipi_unmask(struct irq_data *d) -{ - struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - - atomic_or(irq_bit, this_cpu_ptr(&vipi_enable)); - - /* - * The atomic_or() above must complete before the atomic_read() - * below to avoid racing aic_ipi_send_mask(). - */ - smp_mb__after_atomic(); - - /* - * If a pending vIPI was unmasked, raise a HW IPI to ourselves. - * No barriers needed here since this is a self-IPI. - */ - if (atomic_read(this_cpu_ptr(&vipi_flag)) & irq_bit) { - struct cpumask self_mask = { 0, }; - cpumask_set_cpu(smp_processor_id(), &self_mask); - ipi_send_mask(ic->hwirq->irq, &self_mask); - } -} - - -static void vipi_send_mask(struct irq_data *d, const struct cpumask *mask) -{ - struct vipi_irq_chip *ic = irq_data_get_irq_chip_data(d); - u32 irq_bit = BIT(irqd_to_hwirq(d)); - int cpu; - bool send; - unsigned long pending; - struct cpumask sendmask = *mask; - - for_each_cpu(cpu, mask) { - /* - * This sequence is the mirror of the one in vipi_unmask(); - * see the comment there. Additionally, release semantics - * ensure that the vIPI flag set is ordered after any shared - * memory accesses that precede it. This therefore also pairs - * with the atomic_fetch_andnot in handle_ipi(). - */ - pending = atomic_fetch_or_release(irq_bit, per_cpu_ptr(&vipi_flag, cpu)); - - /* - * The atomic_fetch_or_release() above must complete before the - * atomic_read() below to avoid racing vipi_unmask(). - */ - smp_mb__after_atomic(); - - if (!(pending & irq_bit) && - (atomic_read(per_cpu_ptr(&vipi_enable, cpu)) & irq_bit)) { - cpumask_set_cpu(cpu, &sendmask); - send = true; - } - } - - /* - * The flag writes must complete before the physical IPI is issued - * to another CPU. This is implied by the control dependency on - * the result of atomic_read_acquire() above, which is itself - * already ordered after the vIPI flag write. - */ - if (send) - ipi_send_mask(ic->hwirq->irq, &sendmask); -} - -static struct irq_chip vipi_chip = { - .name = "IPI", - .irq_mask = vipi_mask, - .irq_unmask = vipi_unmask, - .ipi_send_mask = vipi_send_mask, -}; - -/* - * IPI IRQ domain - */ - -static void handle_ipi(struct irq_desc *d) -{ - int i; - unsigned long enabled, firing; - - /* - * The mask read does not need to be ordered. Only we can change - * our own mask anyway, so no races are possible here, as long as - * we are properly in the interrupt handler (XXX is this satisfied?). - */ - enabled = atomic_read(this_cpu_ptr(&vipi_enable)); - - /* - * Clear the IPIs we are about to handle. This pairs with the - * atomic_fetch_or_release() in vipi_send_mask(), and needs to be - * ordered after the ic_write() above (to avoid dropping vIPIs) and - * before IPI handling code (to avoid races handling vIPIs before they - * are signaled). The former is taken care of by the release semantics - * of the write portion, while the latter is taken care of by the - * acquire semantics of the read portion. - */ - firing = atomic_fetch_andnot(enabled, this_cpu_ptr(&vipi_flag)) & enabled; - - for_each_set_bit(i, &firing, NR_SWIPI) { - struct irq_desc *nd = - irq_resolve_mapping(vipi_irqc->domain, i); - - handle_irq_desc(nd); - } -} - -static int vipi_alloc(struct irq_domain *d, unsigned int virq, - unsigned int nr_irqs, void *args) -{ - int i; - - for (i = 0; i < nr_irqs; i++) { - irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &vipi_chip, d->host_data, - handle_percpu_devid_irq, NULL, NULL); - } - - return 0; -} - -static void vipi_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) -{ - /* Not freeing IPIs */ - WARN_ON(1); -} - -static const struct irq_domain_ops vipi_domain_ops = { - .alloc = vipi_alloc, - .free = vipi_free, -}; - -static int vipi_init_smp(struct vipi_irq_chip *irqc) -{ - struct irq_domain *vipi_domain; - int base_ipi; - - vipi_domain = irq_domain_create_linear(NULL, NR_SWIPI, - &vipi_domain_ops, irqc); - if (WARN_ON(!vipi_domain)) - return -ENOMEM; - - vipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; - irq_domain_update_bus_token(vipi_domain, DOMAIN_BUS_IPI); - - base_ipi = __irq_domain_alloc_irqs(vipi_domain, -1, NR_SWIPI, - NUMA_NO_NODE, NULL, false, NULL); - - if (WARN_ON(!base_ipi)) { - irq_domain_remove(vipi_domain); - return -ENOMEM; - } - - set_smp_ipi_range(base_ipi, NR_SWIPI); - - irqc->domain = vipi_domain; - - return 0; -} - -int __init vipi_init(struct irq_data *hwirq) -{ - struct vipi_irq_chip *irqc; - - irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); - if (!irqc) - return -ENOMEM; - - irqc->hwirq = hwirq; - - if (vipi_init_smp(irqc)) - return -ENOMEM; - - vipi_irqc = irqc; - - irq_set_handler_locked(hwirq, handle_ipi); - - pr_info("Initialized with %d vIPIs\n", NR_SWIPI); - - return 0; -}