path: root/core/minute-ia/interrupts.c

/* Copyright 2016 The Chromium OS Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 *
 * Set up the LM2 mIA core & interrupts
 */

#include "common.h"
#include "console.h"
#include "hooks.h"
#include "ia_structs.h"
#include "interrupts.h"
#include "irq_handler.h"
#include "link_defs.h"
#include "mia_panic_internal.h"
#include "registers.h"
#include "task.h"
#include "task_defs.h"
#include "util.h"

/* Console output macros */
#define CPUTS(outstr) cputs(CC_SYSTEM, outstr)
#define CPRINTF(format, args...) cprintf(CC_SYSTEM, format, ## args)
#define CPRINTS(format, args...) cprints(CC_SYSTEM, format, ## args)

/* The IDT  - initialized in init.S */
extern struct idt_entry __idt[NUM_VECTORS];

/* To count the interrupt nesting depth. Usually it is not nested */
volatile uint32_t __in_isr;

static void write_ioapic_reg(const uint32_t reg, const uint32_t val)
{
	IOAPIC_IDX = reg;
	IOAPIC_WDW = val;
}

static uint32_t read_ioapic_reg(const uint32_t reg)
{
	IOAPIC_IDX = reg;
	return IOAPIC_WDW;
}

static void set_ioapic_redtbl_raw(const uint32_t irq, const uint32_t val)
{
	const uint32_t redtbl_lo = IOAPIC_IOREDTBL + 2 * irq;
	const uint32_t redtbl_hi = redtbl_lo + 1;

	write_ioapic_reg(redtbl_lo, val);
	write_ioapic_reg(redtbl_hi, DEST_APIC_ID);
}

/**
 * bitmap for current IRQ's mask status
 * ISH support max 64 IRQs, 64 bit bitmap value is ok
 */
#define ISH_MAX_IOAPIC_IRQS	(64)
uint64_t ioapic_irq_mask_bitmap;

/**
 * disable current all enabled intrrupts
 * return current irq mask bitmap
 * power management typically use 'disable_all_interrupts' to disable current
 * all interrupts and save current interrupts enabling settings before enter
 * low power state, and use 'restore_interrupts' to restore the interrupts
 * settings after exit low power state.
 */
uint64_t disable_all_interrupts(void)
{
	uint64_t saved_map;
	int i;

	saved_map =  ioapic_irq_mask_bitmap;

	for (i = 0; i < ISH_MAX_IOAPIC_IRQS; i++) {
		if (((uint64_t)0x1 << i) & saved_map)
			mask_interrupt(i);
	}

	return saved_map;
}

void restore_interrupts(uint64_t irq_map)
{
	int i;

	/* Disable interrupts until everything is unmasked */
	interrupt_disable();
	for (i = 0; i < ISH_MAX_IOAPIC_IRQS; i++) {
		if (((uint64_t)0x1 << i) & irq_map)
			unmask_interrupt(i);
	}
	interrupt_enable();
}

/*
 * Get lower 32bit of IOAPIC redirection table entry.
 *
 * IOAPIC IRQ redirection table entry has 64 bits:
 *   bit 0-7: interrupt vector to raise on CPU
 *   bit 8-10: delivery mode, how it will send to CPU
 *   bit 11: dest mode
 *   bit 12: delivery status, 0 - idle, 1 - waiting in LAPIC
 *   bit 13: pin polarity
 *   bit 14: remote IRR
 *   bit 15: trigger mode, 0 - edge, 1 - level
 *   bit 16: mask, 0 - irq enable, 1 - irq disable
 *   bit 56-63: destination, LAPIC ID to handle this entry
 *
 * For single core system, driver should ignore higher 32bit of RTE.
 */
uint32_t get_ioapic_redtbl_lo(const unsigned int irq)
{
	return read_ioapic_reg(IOAPIC_IOREDTBL + 2 * irq);
}

void unmask_interrupt(uint32_t irq)
{
	uint32_t val;
	const uint32_t redtbl_lo = IOAPIC_IOREDTBL + 2 * irq;

	val = read_ioapic_reg(redtbl_lo);
	val &= ~IOAPIC_REDTBL_MASK;
	set_ioapic_redtbl_raw(irq, val);
	ioapic_irq_mask_bitmap |= ((uint64_t)0x1) << irq;
}

void mask_interrupt(uint32_t irq)
{
	uint32_t val;
	const uint32_t redtbl_lo = IOAPIC_IOREDTBL + 2 * irq;

	val = read_ioapic_reg(redtbl_lo);
	val |= IOAPIC_REDTBL_MASK;
	set_ioapic_redtbl_raw(irq, val);

	ioapic_irq_mask_bitmap &= ~(((uint64_t)0x1) << irq);
}

/* Maps IRQs to vectors. To be programmed in IOAPIC redirection table */
static const irq_desc_t system_irqs[] = {
	LEVEL_INTR(ISH_I2C0_IRQ, ISH_I2C0_VEC),
	LEVEL_INTR(ISH_I2C1_IRQ, ISH_I2C1_VEC),
	LEVEL_INTR(ISH_I2C2_IRQ, ISH_I2C2_VEC),
	LEVEL_INTR(ISH_WDT_IRQ, ISH_WDT_VEC),
	LEVEL_INTR(ISH_GPIO_IRQ, ISH_GPIO_VEC),
	LEVEL_INTR(ISH_IPC_HOST2ISH_IRQ, ISH_IPC_VEC),
#ifndef CONFIG_ISH_HOST2ISH_COMBINED_ISR
	LEVEL_INTR(ISH_IPC_ISH2HOST_CLR_IRQ, ISH_IPC_ISH2HOST_CLR_VEC),
#endif
	LEVEL_INTR(ISH_HPET_TIMER1_IRQ, ISH_HPET_TIMER1_VEC),
	LEVEL_INTR(ISH_DEBUG_UART_IRQ, ISH_DEBUG_UART_VEC),
	LEVEL_INTR(ISH_FABRIC_IRQ, ISH_FABRIC_VEC),
#ifdef CONFIG_ISH_PM_RESET_PREP
	LEVEL_INTR(ISH_RESET_PREP_IRQ, ISH_RESET_PREP_VEC),
#endif
#ifdef CONFIG_ISH_PM_D0I1
	LEVEL_INTR(ISH_PMU_WAKEUP_IRQ, ISH_PMU_WAKEUP_VEC),
#endif
#ifdef CONFIG_ISH_PM_D3
	LEVEL_INTR(ISH_D3_RISE_IRQ, ISH_D3_RISE_VEC),
#ifndef CONFIG_ISH_NEW_PM
	LEVEL_INTR(ISH_D3_FALL_IRQ, ISH_D3_FALL_VEC),
	LEVEL_INTR(ISH_BME_RISE_IRQ, ISH_BME_RISE_VEC),
	LEVEL_INTR(ISH_BME_FALL_IRQ, ISH_BME_FALL_VEC)
#endif
#endif
};

/**
 * The macro below is used to define 20 exeption handler routines, each
 * of which will push their corresponding interrupt vector number to
 * the stack, and then call exception_panic. The remaining arguments to
 * exception_panic were pushed by the hardware when the exception was
 * called.  The errorcode is pushed to the stack by hardware for vectors
 * 8, 10-14 and 17.  Make sure to push 0 for the other vectors
 *
 * This is done since interrupt vectors 0-31 bypass the APIC ISR register
 * and go directly to the CPU core, so get_current_interrupt_vector
 * cannot be used.
 */
#define DEFINE_EXN_HANDLER(vector)			\
	_DEFINE_EXN_HANDLER(vector, exception_panic_##vector)
#define _DEFINE_EXN_HANDLER(vector, name)		\
	void __keep name(void);				\
	noreturn void name(void)	\
	{						\
		__asm__ ("push $0\n"			\
			 "push $" #vector "\n"		\
			 "call exception_panic\n");	\
		__builtin_unreachable();		\
	}

#define DEFINE_EXN_HANDLER_W_ERRORCODE(vector)		\
	_DEFINE_EXN_HANDLER_W_ERRORCODE(vector, exception_panic_##vector)
#define _DEFINE_EXN_HANDLER_W_ERRORCODE(vector, name)	\
	void __keep name(void);				\
	noreturn void name(void)	\
	{						\
		__asm__ ("push $" #vector "\n"		\
			 "call exception_panic\n");	\
		__builtin_unreachable();		\
	}

DEFINE_EXN_HANDLER(0);
DEFINE_EXN_HANDLER(1);
DEFINE_EXN_HANDLER(2);
DEFINE_EXN_HANDLER(3);
DEFINE_EXN_HANDLER(4);
DEFINE_EXN_HANDLER(5);
DEFINE_EXN_HANDLER(6);
DEFINE_EXN_HANDLER(7);
DEFINE_EXN_HANDLER_W_ERRORCODE(8);
DEFINE_EXN_HANDLER(9);
DEFINE_EXN_HANDLER_W_ERRORCODE(10);
DEFINE_EXN_HANDLER_W_ERRORCODE(11);
DEFINE_EXN_HANDLER_W_ERRORCODE(12);
DEFINE_EXN_HANDLER_W_ERRORCODE(13);
DEFINE_EXN_HANDLER_W_ERRORCODE(14);
DEFINE_EXN_HANDLER(16);
DEFINE_EXN_HANDLER_W_ERRORCODE(17);
DEFINE_EXN_HANDLER(18);
DEFINE_EXN_HANDLER(19);
DEFINE_EXN_HANDLER(20);

/**
 * Use a similar approach for defining an optional handler for
 * watchdog timer expiration. However, this time, hardware does not
 * push errorcode, and we must account for that by pushing zero.
 */
noreturn __keep
void exception_panic_wdt(uint32_t cs)
{
	exception_panic(
		CONFIG_MIA_WDT_VEC,
		0,
		(uint32_t)__builtin_return_address(0),
		cs,
		0);
}

void set_interrupt_gate(uint8_t num, isr_handler_t func, uint8_t flags)
{
	uint16_t code_segment;

	/* When the flat model is used the CS will never change. */
	__asm volatile ("mov %%cs, %0":"=r" (code_segment));

	__idt[num].dword_lo = GEN_IDT_DESC_LO(func, code_segment, flags);
	__idt[num].dword_up = GEN_IDT_DESC_UP(func, code_segment, flags);
}

/**
 * This procedure gets the current interrupt vector number using the
 * APIC ISR register, and should only be called from an interrupt
 * vector context. Note that vectors 0-31, as well as software
 * triggered interrupts (using "int n") bypass the APIC, and this
 * routine will not work for that.
 *
 * Returns an integer in range 0-255 upon success, or 256 (0x100)
 * upon failure.
 */
uint32_t get_current_interrupt_vector(void)
{
	int i;
	uint32_t vec;

	/* In service register */
	volatile uint32_t *ioapic_isr_last = &LAPIC_ISR_LAST_REG;

	/* Scan ISRs from highest priority */
	for (i = 7; i >= 0; i--, ioapic_isr_last -= 4) {
		vec = *ioapic_isr_last;
		if (vec) {
			return (32 * i) + __fls(vec);
		}
	}

	return 0x100;
}

static uint32_t lapic_lvt_error_count;
static uint32_t ioapic_pending_count;
static uint32_t last_esr;

static void print_lpaic_lvt_error(void)
{
	CPRINTS("LAPIC error ESR 0x%02x: %u; IOAPIC pending: %u", last_esr,
		lapic_lvt_error_count, ioapic_pending_count);
}
DECLARE_DEFERRED(print_lpaic_lvt_error);

/*
 * Get LAPIC ISR, TMR, or IRR vector bit.
 *
 * LAPIC ISR, TMR, and IRR bit vector registers are laid out in a way that
 * skips 3 32bit word after one 32 bit entry:
 *
 *  ADDR         |  32 vectors   |    +0x4    |   +0x8    |   +0xC
 * --------------+---------------+------------+-----------+------------
 *  BASE         |  0 ~ 31       |    skip 96 bits
 * --------------+---------------+------------+-----------+------------
 *  BASE + 0x10  |  32 ~ 64      |    skip 96 bits
 * --------------+---------------+------------+-----------+------------
 *  BASE + 0x20  |  64 ~ 96      |    skip 96 bits
 * --------------+---------------+------------+-----------+------------
 *  ...
 *
 * From Kernel LAPIC driver:
 * #define VEC_POS(v) ((v) & (32 - 1))
 * #define REG_POS(v) (((v) >> 5) << 4)
 */
static inline unsigned int lapic_get_vector(volatile uint32_t *reg_base,
					    uint32_t vector)
{
	/*
	 * Since we are using array indexing, we need to divide the vec_pos by
	 * sizeof(uint32_t), i.e. shift to the right 2.
	 */
	uint32_t reg_pos = (vector >> 5) << 2;
	uint32_t vec_pos = vector & (32 - 1);

	return reg_base[reg_pos] & BIT(vec_pos);
}

/*
 * Normally, LAPIC_LVT_ERROR_VECTOR doesn't need a handler. But ISH IOAPIC
 * has an unknown bug on high frequency interrupts. A similar issue has been
 * found in PII/PIII era according to x86 APIC Kernel driver. When IOAPIC
 * routing entry is masked/unmasked at a high rate, IOAPIC line gets stuck and
 * no more interrupts are received from it.
 *
 * The solution in Kernel driver changes interrupt distribution model. But it
 * doesn't solve the problem completely. Just make it hang less frequent.
 *
 * ISH IOAPIC-LAPIC was configured in a way so we can manually send EOI (end of
 * interrupt) to IOAPIC. So in the workaround below, we ack all IOAPIC vectors
 * not in LAPIC IRR (interrupt request register). The side effect is we kicked
 * out some of the interrupts without handling them. It depends on the
 * peripheral hardware design if it re-send this irq.
 */
void handle_lapic_lvt_error(void)
{
	uint32_t esr = LAPIC_ESR_REG;
	uint32_t ioapic_redtbl, vec;
	int irq, max_irq_entries;

	/* Ack LVT ERROR exception */
	LAPIC_ESR_REG = 0;

	/*
	 * When IOAPIC has more than 1 interrupts in remote IRR state,
	 * LAPIC raises internal error.
	 */
	if (esr & LAPIC_ERR_RECV_ILLEGAL) {
		lapic_lvt_error_count++;

		/* Scan redirect table entries */
		max_irq_entries = (read_ioapic_reg(IOAPIC_VERSION) >> 16) &
				  0xff;
		for (irq = 0; irq < max_irq_entries; irq++) {
			ioapic_redtbl = get_ioapic_redtbl_lo(irq);
			/* Skip masked IRQs */
			if (ioapic_redtbl & IOAPIC_REDTBL_MASK)
				continue;
			/* If pending interrupt is not in LAPIC, clear it. */
			if (ioapic_redtbl & IOAPIC_REDTBL_IRR) {
				vec = IRQ_TO_VEC(irq);
				if (!lapic_get_vector(&LAPIC_IRR_REG, vec)) {
					/* End of interrupt */
					IOAPIC_EOI_REG = vec;
					ioapic_pending_count++;
				}
			}
		}
	}

	if (esr) {
		/* Don't print in interrupt context because it is too slow */
		last_esr = esr;
		hook_call_deferred(&print_lpaic_lvt_error_data, 0);
	}
}

/* LAPIC LVT error is not an IRQ and can not use DECLARE_IRQ() to call. */
void _lapic_error_handler(void);
__asm__ (
	".section .text._lapic_error_handler\n"
	"_lapic_error_handler:\n"
		"pusha\n"
		ASM_LOCK_PREFIX "addl $1, __in_isr\n"
		"movl %esp, %eax\n"
		"movl $stack_end, %esp\n"
		"push %eax\n"
#ifdef CONFIG_TASK_PROFILING
		"push $" STRINGIFY(CONFIG_IRQ_COUNT) "\n"
		"call task_start_irq_handler\n"
		"addl $0x04, %esp\n"
#endif
		"call handle_lapic_lvt_error\n"
		"pop %esp\n"
		"movl $0x00, (0xFEE000B0)\n"	/* Set EOI for LAPIC */
		ASM_LOCK_PREFIX "subl $1, __in_isr\n"
		"popa\n"
		"iret\n"
	);

/* Should only be called in interrupt context */
void unhandled_vector(void)
{
	uint32_t vec = get_current_interrupt_vector();
	CPRINTF("Ignoring vector 0x%0x!\n", vec);
	/* Put the vector number in eax so default_int_handler can use it */
	asm("" : : "a" (vec));
}

/**
 * Called from SOFTIRQ_VECTOR when software is trigger an IRQ manually
 *
 * If IRQ is out of range, then no routine should be called
 */
void call_irq_service_routine(uint32_t irq)
{
	const struct irq_def *p = __irq_data;

	/* If just rescheduling a task, we won't have a routine to call */
	if (irq >= CONFIG_IRQ_COUNT)
		return;

	for (; p < __irq_data_end; p++) {
		if (p->irq == irq) {
			p->routine();
			break;
		}
	}

	if (p == __irq_data_end)
		CPRINTS("IRQ %d routine not found!", irq);
}

void lapic_restore(void)
{
	LAPIC_ESR_REG = 0;
	APIC_SPURIOUS_INT = LAPIC_SPURIOUS_INT_VECTOR | APIC_ENABLE_BIT;
	APIC_LVT_ERROR = LAPIC_LVT_ERROR_VECTOR;
}

void init_interrupts(void)
{
	unsigned entry;
	const struct irq_def *p;
	unsigned num_system_irqs = ARRAY_SIZE(system_irqs);
	unsigned max_entries = (read_ioapic_reg(IOAPIC_VERSION) >> 16) & 0xff;

	/* Setup gates for IRQs declared by drivers using DECLARE_IRQ */
	for (p = __irq_data; p < __irq_data_end; p++)
		set_interrupt_gate(IRQ_TO_VEC(p->irq),
				   p->handler,
				   IDT_DESC_FLAGS);

	/* Software generated IRQ */
	set_interrupt_gate(SOFTIRQ_VECTOR, sw_irq_handler, IDT_DESC_FLAGS);

	/* Setup gate for LAPIC_LVT_ERROR vector; clear any remnant error. */
	LAPIC_ESR_REG = 0;
	set_interrupt_gate(LAPIC_LVT_ERROR_VECTOR, _lapic_error_handler,
			   IDT_DESC_FLAGS);

	/* Mask all interrupts by default in IOAPIC */
	for (entry = 0; entry < max_entries; entry++)
		set_ioapic_redtbl_raw(entry, IOAPIC_REDTBL_MASK);

	/* Enable pre-defined interrupts */
	for (entry = 0; entry < num_system_irqs; entry++)
		set_ioapic_redtbl_raw(system_irqs[entry].irq,
				      system_irqs[entry].vector |
				      IOAPIC_REDTBL_DELMOD_FIXED |
				      IOAPIC_REDTBL_DESTMOD_PHYS |
				      IOAPIC_REDTBL_MASK |
				      system_irqs[entry].polarity |
				      system_irqs[entry].trigger);

	set_interrupt_gate(ISH_TS_VECTOR, __switchto, IDT_DESC_FLAGS);

	/* Bind exception handlers to print panic message */
	set_interrupt_gate(0, exception_panic_0, IDT_DESC_FLAGS);
	set_interrupt_gate(1, exception_panic_1, IDT_DESC_FLAGS);
	set_interrupt_gate(2, exception_panic_2, IDT_DESC_FLAGS);
	set_interrupt_gate(3, exception_panic_3, IDT_DESC_FLAGS);
	set_interrupt_gate(4, exception_panic_4, IDT_DESC_FLAGS);
	set_interrupt_gate(5, exception_panic_5, IDT_DESC_FLAGS);
	set_interrupt_gate(6, exception_panic_6, IDT_DESC_FLAGS);
	set_interrupt_gate(7, exception_panic_7, IDT_DESC_FLAGS);
	set_interrupt_gate(8, exception_panic_8, IDT_DESC_FLAGS);
	set_interrupt_gate(9, exception_panic_9, IDT_DESC_FLAGS);
	set_interrupt_gate(10, exception_panic_10, IDT_DESC_FLAGS);
	set_interrupt_gate(11, exception_panic_11, IDT_DESC_FLAGS);
	set_interrupt_gate(12, exception_panic_12, IDT_DESC_FLAGS);
	set_interrupt_gate(13, exception_panic_13, IDT_DESC_FLAGS);
	set_interrupt_gate(14, exception_panic_14, IDT_DESC_FLAGS);
	set_interrupt_gate(16, exception_panic_16, IDT_DESC_FLAGS);
	set_interrupt_gate(17, exception_panic_17, IDT_DESC_FLAGS);
	set_interrupt_gate(18, exception_panic_18, IDT_DESC_FLAGS);
	set_interrupt_gate(19, exception_panic_19, IDT_DESC_FLAGS);
	set_interrupt_gate(20, exception_panic_20, IDT_DESC_FLAGS);

	/*
	 * Set up watchdog expiration like a panic, that way we can
	 * use the common panic handling code, and also properly
	 * retrieve EIP.
	 */
	if (IS_ENABLED(CONFIG_WATCHDOG))
		set_interrupt_gate(CONFIG_MIA_WDT_VEC,
				   (isr_handler_t)exception_panic_wdt,
				   IDT_DESC_FLAGS);

	/* Note: At reset, ID field is already set to 0 in APIC ID register */

	/* Enable the APIC, mapping the spurious interrupt at the same time. */
	APIC_SPURIOUS_INT = LAPIC_SPURIOUS_INT_VECTOR | APIC_ENABLE_BIT;

	/* Set timer error vector. */
	APIC_LVT_ERROR = LAPIC_LVT_ERROR_VECTOR;
}