path: root/xen/arch/x86/genapic/x2apic.c

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * x2APIC driver.
 *
 * Copyright (c) 2008, Intel Corporation.
 */

#include <xen/init.h>
#include <xen/cpu.h>
#include <xen/cpumask.h>
#include <xen/param.h>
#include <asm/apicdef.h>
#include <asm/genapic.h>
#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/msr.h>
#include <asm/processor.h>
#include <xen/smp.h>

static DEFINE_PER_CPU_READ_MOSTLY(u32, cpu_2_logical_apicid);
static DEFINE_PER_CPU_READ_MOSTLY(cpumask_t *, cluster_cpus);
static cpumask_t *cluster_cpus_spare;
static DEFINE_PER_CPU(cpumask_var_t, scratch_mask);

static inline u32 x2apic_cluster(unsigned int cpu)
{
    return per_cpu(cpu_2_logical_apicid, cpu) >> 16;
}

static void cf_check init_apic_ldr_x2apic_cluster(void)
{
    unsigned int cpu, this_cpu = smp_processor_id();

    per_cpu(cpu_2_logical_apicid, this_cpu) = apic_read(APIC_LDR);

    if ( per_cpu(cluster_cpus, this_cpu) )
    {
        ASSERT(cpumask_test_cpu(this_cpu, per_cpu(cluster_cpus, this_cpu)));
        return;
    }

    per_cpu(cluster_cpus, this_cpu) = cluster_cpus_spare;
    for_each_online_cpu ( cpu )
    {
        if ( this_cpu == cpu )
            continue;
        /*
         * Guard in particular against the compiler suspecting out-of-bounds
         * array accesses below when NR_CPUS=1 (oddly enough with gcc 10 it
         * is the 1st of these alone which actually helps, not the 2nd, nor
         * are both required together there).
         */
        BUG_ON(this_cpu >= NR_CPUS);
        BUG_ON(cpu >= NR_CPUS);
        if ( x2apic_cluster(this_cpu) != x2apic_cluster(cpu) )
            continue;
        per_cpu(cluster_cpus, this_cpu) = per_cpu(cluster_cpus, cpu);
        break;
    }
    if ( per_cpu(cluster_cpus, this_cpu) == cluster_cpus_spare )
        cluster_cpus_spare = NULL;

    cpumask_set_cpu(this_cpu, per_cpu(cluster_cpus, this_cpu));
}

static const cpumask_t *cf_check vector_allocation_cpumask_x2apic_cluster(
    int cpu)
{
    return per_cpu(cluster_cpus, cpu);
}

static unsigned int cf_check cpu_mask_to_apicid_x2apic_cluster(
    const cpumask_t *cpumask)
{
    unsigned int cpu = cpumask_any(cpumask);
    unsigned int dest = per_cpu(cpu_2_logical_apicid, cpu);
    const cpumask_t *cluster_cpus = per_cpu(cluster_cpus, cpu);

    for_each_cpu ( cpu, cluster_cpus )
        if ( cpumask_test_cpu(cpu, cpumask) )
            dest |= per_cpu(cpu_2_logical_apicid, cpu);

    return dest;
}

static void cf_check send_IPI_self_x2apic(uint8_t vector)
{
    apic_wrmsr(APIC_SELF_IPI, vector);
}

static void cf_check send_IPI_mask_x2apic_phys(
    const cpumask_t *cpumask, int vector)
{
    unsigned int cpu;
    unsigned long flags;
    uint64_t msr_content;

    /*
     * Ensure that any synchronisation data written in program order by this
     * CPU is seen by notified remote CPUs. The WRMSR contained within
     * apic_icr_write() can otherwise be executed early.
     * 
     * The reason smp_mb() is sufficient here is subtle: the register arguments
     * to WRMSR must depend on a memory read executed after the barrier. This
     * is guaranteed by cpu_physical_id(), which reads from a global array (and
     * so cannot be hoisted above the barrier even by a clever compiler).
     */
    smp_mb();

    local_irq_save(flags);

    for_each_cpu ( cpu, cpumask )
    {
        if ( !cpu_online(cpu) || (cpu == smp_processor_id()) )
            continue;
        msr_content = cpu_physical_id(cpu);
        msr_content = (msr_content << 32) | APIC_DM_FIXED |
                      APIC_DEST_PHYSICAL | vector;
        apic_wrmsr(APIC_ICR, msr_content);
    }

    local_irq_restore(flags);
}

static void cf_check send_IPI_mask_x2apic_cluster(
    const cpumask_t *cpumask, int vector)
{
    unsigned int cpu = smp_processor_id();
    cpumask_t *ipimask = per_cpu(scratch_mask, cpu);
    const cpumask_t *cluster_cpus;
    unsigned long flags;

    smp_mb(); /* See above for an explanation. */

    local_irq_save(flags);

    cpumask_andnot(ipimask, &cpu_online_map, cpumask_of(cpu));

    for ( cpumask_and(ipimask, cpumask, ipimask); !cpumask_empty(ipimask);
          cpumask_andnot(ipimask, ipimask, cluster_cpus) )
    {
        uint64_t msr_content = 0;

        cluster_cpus = per_cpu(cluster_cpus, cpumask_first(ipimask));
        for_each_cpu ( cpu, cluster_cpus )
        {
            if ( !cpumask_test_cpu(cpu, ipimask) )
                continue;
            msr_content |= per_cpu(cpu_2_logical_apicid, cpu);
        }

        BUG_ON(!(msr_content & 0xffff));
        msr_content = (msr_content << 32) | APIC_DM_FIXED |
                      APIC_DEST_LOGICAL | vector;
        apic_wrmsr(APIC_ICR, msr_content);
    }

    local_irq_restore(flags);
}

static const struct genapic __initconstrel apic_x2apic_phys = {
    APIC_INIT("x2apic_phys", NULL),
    .int_delivery_mode = dest_Fixed,
    .int_dest_mode = 0 /* physical delivery */,
    .init_apic_ldr = init_apic_ldr_phys,
    .vector_allocation_cpumask = vector_allocation_cpumask_phys,
    .cpu_mask_to_apicid = cpu_mask_to_apicid_phys,
    .send_IPI_mask = send_IPI_mask_x2apic_phys,
    .send_IPI_self = send_IPI_self_x2apic
};

static const struct genapic __initconstrel apic_x2apic_cluster = {
    APIC_INIT("x2apic_cluster", NULL),
    .int_delivery_mode = dest_LowestPrio,
    .int_dest_mode = 1 /* logical delivery */,
    .init_apic_ldr = init_apic_ldr_x2apic_cluster,
    .vector_allocation_cpumask = vector_allocation_cpumask_x2apic_cluster,
    .cpu_mask_to_apicid = cpu_mask_to_apicid_x2apic_cluster,
    .send_IPI_mask = send_IPI_mask_x2apic_cluster,
    .send_IPI_self = send_IPI_self_x2apic
};

static int cf_check update_clusterinfo(
    struct notifier_block *nfb, unsigned long action, void *hcpu)
{
    unsigned int cpu = (unsigned long)hcpu;
    int err = 0;

    switch (action) {
    case CPU_UP_PREPARE:
        per_cpu(cpu_2_logical_apicid, cpu) = BAD_APICID;
        if ( !cluster_cpus_spare )
            cluster_cpus_spare = xzalloc(cpumask_t);
        if ( !cluster_cpus_spare ||
             !cond_alloc_cpumask_var(&per_cpu(scratch_mask, cpu)) )
            err = -ENOMEM;
        break;
    case CPU_UP_CANCELED:
    case CPU_DEAD:
    case CPU_REMOVE:
        if ( park_offline_cpus == (action != CPU_REMOVE) )
            break;
        if ( per_cpu(cluster_cpus, cpu) )
        {
            cpumask_clear_cpu(cpu, per_cpu(cluster_cpus, cpu));
            if ( cpumask_empty(per_cpu(cluster_cpus, cpu)) )
                XFREE(per_cpu(cluster_cpus, cpu));
        }
        FREE_CPUMASK_VAR(per_cpu(scratch_mask, cpu));
        break;
    }

    return notifier_from_errno(err);
}

static struct notifier_block x2apic_cpu_nfb = {
   .notifier_call = update_clusterinfo
};

static int8_t __initdata x2apic_phys = -1;
boolean_param("x2apic_phys", x2apic_phys);

const struct genapic *__init apic_x2apic_probe(void)
{
    if ( x2apic_phys < 0 )
    {
        /*
         * Force physical mode if there's no (full) interrupt remapping support:
         * The ID in clustered mode requires a 32 bit destination field due to
         * the usage of the high 16 bits to hold the cluster ID.
         */
        x2apic_phys = iommu_intremap != iommu_intremap_full ||
                      (acpi_gbl_FADT.flags & ACPI_FADT_APIC_PHYSICAL) ||
                      (IS_ENABLED(CONFIG_X2APIC_PHYSICAL) &&
                       !(acpi_gbl_FADT.flags & ACPI_FADT_APIC_CLUSTER));
    }
    else if ( !x2apic_phys )
        switch ( iommu_intremap )
        {
        case iommu_intremap_off:
        case iommu_intremap_restricted:
            printk("WARNING: x2APIC cluster mode is not supported %s interrupt remapping -"
                   " forcing phys mode\n",
                   iommu_intremap == iommu_intremap_off ? "without"
                                                        : "with restricted");
            x2apic_phys = true;
            break;

        case iommu_intremap_full:
            break;
        }

    if ( x2apic_phys )
        return &apic_x2apic_phys;

    if ( !this_cpu(cluster_cpus) )
    {
        update_clusterinfo(NULL, CPU_UP_PREPARE,
                           (void *)(long)smp_processor_id());
        init_apic_ldr_x2apic_cluster();
        register_cpu_notifier(&x2apic_cpu_nfb);
    }

    return &apic_x2apic_cluster;
}

void __init check_x2apic_preenabled(void)
{
    u32 lo, hi;

    if ( !cpu_has_x2apic )
        return;

    /* Check whether x2apic mode was already enabled by the BIOS. */
    rdmsr(MSR_APIC_BASE, lo, hi);
    if ( lo & APIC_BASE_EXTD )
    {
        printk("x2APIC mode is already enabled by BIOS.\n");
        x2apic_enabled = 1;
        genapic = *apic_x2apic_probe();
    }
}