/* $Id: x86.c,v 1.127.2.70 2010/11/07 19:46:06 mikpe Exp $

 * x86/x86_64 performance-monitoring counters driver.

 *

 * Copyright (C) 1999-2010  Mikael Pettersson

 */

#include <linux/version.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)

#include <linux/config.h>

#endif

#define __NO_VERSION__

#include <linux/module.h>

#include <linux/init.h>

#include <linux/sched.h>

#include <linux/fs.h>

#include <linux/perfctr.h>

#include <asm/msr.h>

#undef MSR_P6_PERFCTR0

#undef MSR_P6_EVNTSEL0

#undef MSR_K7_PERFCTR0

#undef MSR_K7_EVNTSEL0

#undef MSR_CORE_PERF_FIXED_CTR0

#undef MSR_CORE_PERF_FIXED_CTR_CTRL

#undef MSR_CORE_PERF_GLOBAL_CTRL

#undef MSR_IA32_MISC_ENABLE

#undef MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL

#undef MSR_IA32_DEBUGCTLMSR

#include <asm/fixmap.h>

#include <asm/apic.h>

struct hw_interrupt_type;

#include <asm/hw_irq.h>

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) && defined(CONFIG_X86_LOCAL_APIC)

#include <asm/nmi.h>

#endif

#include "compat.h"

#include "x86_compat.h"

#include "x86_tests.h"

/* Support for lazy evntsel and perfctr MSR updates. */

struct per_cpu_cache {	/* roughly a subset of perfctr_cpu_state */

	union {

		unsigned int p5_cesr;

		unsigned int id;	/* cache owner id */

	} k1;

	struct {

		/* NOTE: these caches have physical indices, not virtual */

		unsigned int evntsel[18];

		union {

			unsigned int escr[0x3E2-0x3A0];

			unsigned int evntsel_high[18];

		};

		unsigned int pebs_enable;

		unsigned int pebs_matrix_vert;

	} control;

	unsigned int core2_fixed_ctr_ctrl;

	unsigned int nhlm_offcore_rsp[2];

} ____cacheline_aligned;

static struct per_cpu_cache per_cpu_cache[NR_CPUS] __cacheline_aligned;

#define __get_cpu_cache(cpu) (&per_cpu_cache[cpu])

#define get_cpu_cache() __get_cpu_cache(smp_processor_id())

/* Structure for counter snapshots, as 32-bit values. */

struct perfctr_low_ctrs {

	unsigned int tsc;

	unsigned int pmc[18];

};

/* Structures for describing the set of PMU MSRs. */

struct perfctr_msr_range {

	unsigned int first_msr;

	unsigned int nr_msrs;

};

struct perfctr_pmu_msrs {

	const struct perfctr_msr_range *perfctrs; /* for {reserve,release}_perfctr_nmi() */

	const struct perfctr_msr_range *evntsels; /* for {reserve,release}_evntsel_nmi() */

	const struct perfctr_msr_range *extras;

	void (*clear_counters)(int init);

};

/* Intel P5, Cyrix 6x86MX/MII/III, Centaur WinChip C6/2/3 */

#define MSR_P5_CESR		0x11

#define MSR_P5_CTR0		0x12		/* .. 0x13 */

#define P5_CESR_CPL		0x00C0

#define P5_CESR_RESERVED	(~0x01FF)

#define MII_CESR_RESERVED	(~0x05FF)

#define C6_CESR_RESERVED	(~0x00FF)

/* Intel P6, VIA C3 */

#define MSR_P6_PERFCTR0		0xC1		/* .. 0xC4 */

#define MSR_P6_EVNTSEL0		0x186		/* .. 0x189 */

#define P6_EVNTSEL_ENABLE	0x00400000

#define P6_EVNTSEL_INT		0x00100000

#define P6_EVNTSEL_CPL		0x00030000

#define P6_EVNTSEL_RESERVED	0x00280000

#define VC3_EVNTSEL1_RESERVED	(~0x1FF)

/* Intel Core */

#define MSR_IA32_DEBUGCTLMSR		0x000001D9

#define MSR_IA32_DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI	(1<<12)

#define MSR_CORE_PERF_FIXED_CTR0	0x309	/* .. 0x30B */

#define MSR_CORE_PERF_FIXED_CTR_CTRL	0x38D

#define MSR_CORE_PERF_FIXED_CTR_CTRL_PMIANY	0x00000888

#define MSR_CORE_PERF_GLOBAL_CTRL	0x38F

#define CORE2_PMC_FIXED_FLAG		(1<<30)

#define CORE2_PMC_FIXED_MASK		0x3

/* Intel Nehalem */

#define MSR_OFFCORE_RSP0	0x1A6		/* Westmere has another at 0x1A7 */

#define OFFCORE_RSP_RESERVED	(~0xF7FF)

/* AMD K7 */

#define MSR_K7_EVNTSEL0		0xC0010000	/* .. 0xC0010003 */

#define MSR_K7_PERFCTR0		0xC0010004	/* .. 0xC0010007 */

/* AMD K8 */

#define IS_K8_NB_EVENT(EVNTSEL)	((((EVNTSEL) >> 5) & 0x7) == 0x7)

/* AMD Family 10h */

#define FAM10H_EVNTSEL_HIGH_RESERVED	(~0x30F)

/* Intel P4, Intel Pentium M, Intel Core */

#define MSR_IA32_MISC_ENABLE	0x1A0

#define MSR_IA32_MISC_ENABLE_PERF_AVAIL (1<<7)	/* read-only status bit */

#define MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL (1<<12) /* read-only status bit */

/* Intel P4 */

#define MSR_P4_PERFCTR0		0x300		/* .. 0x311 */

#define MSR_P4_CCCR0		0x360		/* .. 0x371 */

#define MSR_P4_ESCR0		0x3A0		/* .. 0x3E1, with some gaps */

#define MSR_P4_PEBS_ENABLE	0x3F1

#define P4_PE_REPLAY_TAG_BITS	0x00000607

#define P4_PE_UOP_TAG		0x01000000

#define P4_PE_RESERVED		0xFEFFF9F8	/* only allow ReplayTagging */

#define MSR_P4_PEBS_MATRIX_VERT	0x3F2

#define P4_PMV_REPLAY_TAG_BITS	0x00000003

#define P4_PMV_RESERVED		0xFFFFFFFC

#define P4_CCCR_OVF		0x80000000

#define P4_CCCR_CASCADE		0x40000000

#define P4_CCCR_OVF_PMI_T1	0x08000000

#define P4_CCCR_OVF_PMI_T0	0x04000000

#define P4_CCCR_FORCE_OVF	0x02000000

#define P4_CCCR_ACTIVE_THREAD	0x00030000

#define P4_CCCR_ENABLE		0x00001000

#define P4_CCCR_ESCR_SELECT(X)	(((X) >> 13) & 0x7)

#define P4_CCCR_EXTENDED_CASCADE	0x00000800

#define P4_CCCR_RESERVED	(0x300007FF|P4_CCCR_OVF|P4_CCCR_OVF_PMI_T1)

#define P4_ESCR_CPL_T1		0x00000003

#define P4_ESCR_CPL_T0		0x0000000C

#define P4_ESCR_TAG_ENABLE	0x00000010

#define P4_ESCR_RESERVED	(0x80000000)

#define P4_FAST_RDPMC		0x80000000

#define P4_MASK_FAST_RDPMC	0x0000001F	/* we only need low 5 bits */

#define rdmsr_low(msr,low) \

	__asm__ __volatile__("rdmsr" : "=a"(low) : "c"(msr) : "edx")

#define rdpmc_low(ctr,low) \

	__asm__ __volatile__("rdpmc" : "=a"(low) : "c"(ctr) : "edx")

static void clear_msr_range(unsigned int base, unsigned int n)

{

	unsigned int i;

	for(i = 0; i < n; ++i)

		wrmsr(base+i, 0, 0);

}

static inline void set_in_cr4_local(unsigned int mask)

{

	write_cr4(read_cr4() | mask);

}

static inline void clear_in_cr4_local(unsigned int mask)

{

	write_cr4(read_cr4() & ~mask);

}

static unsigned int new_id(void)

{

	static DEFINE_SPINLOCK(lock);

	static unsigned int counter;

	int id;

	spin_lock(&lock);

	id = ++counter;

	spin_unlock(&lock);

	return id;

}

#if defined(CONFIG_X86_LOCAL_APIC)

static inline void perfctr_cpu_mask_interrupts(const struct per_cpu_cache *cache)

{

	__perfctr_cpu_mask_interrupts();

}

static inline void perfctr_cpu_unmask_interrupts(const struct per_cpu_cache *cache)

{

	__perfctr_cpu_unmask_interrupts();

}

#else	/* CONFIG_X86_LOCAL_APIC */

#define perfctr_cstatus_has_ictrs(cstatus)	0

#undef cpu_has_apic

#define cpu_has_apic				0

#undef apic_write

#define apic_write(reg,vector)			do{}while(0)

#endif	/* CONFIG_X86_LOCAL_APIC */

#if defined(CONFIG_SMP)

static inline void

set_isuspend_cpu(struct perfctr_cpu_state *state, int cpu)

{

	state->k1.isuspend_cpu = cpu;

}

static inline int

is_isuspend_cpu(const struct perfctr_cpu_state *state, int cpu)

{

	return state->k1.isuspend_cpu == cpu;

}

static inline void clear_isuspend_cpu(struct perfctr_cpu_state *state)

{

	state->k1.isuspend_cpu = NR_CPUS;

}

#else	/* CONFIG_SMP */

static inline void set_isuspend_cpu(struct perfctr_cpu_state *state, int cpu) { }

static inline int is_isuspend_cpu(const struct perfctr_cpu_state *state, int cpu) { return 1; }

static inline void clear_isuspend_cpu(struct perfctr_cpu_state *state) { }

#endif	/* CONFIG_SMP */

/****************************************************************

 *								*

 * Driver procedures.						*

 *								*

 ****************************************************************/

/*

 * Intel P5 family (Pentium, family code 5).

 * - One TSC and two 40-bit PMCs.

 * - A single 32-bit CESR (MSR 0x11) controls both PMCs.

 *   CESR has two halves, each controlling one PMC.

 *   To keep the API reasonably clean, the user puts 16 bits of

 *   control data in each counter's evntsel; the driver combines

 *   these to a single 32-bit CESR value.

 * - Overflow interrupts are not available.

 * - Pentium MMX added the RDPMC instruction. RDPMC has lower

 *   overhead than RDMSR and it can be used in user-mode code.

 * - The MMX events are not symmetric: some events are only available

 *   for some PMC, and some event codes denote different events

 *   depending on which PMCs they control.

 */

/* shared with MII and C6 */

static int p5_like_check_control(struct perfctr_cpu_state *state,

				 unsigned int reserved_bits, int is_c6)

{

	unsigned short cesr_half[2];

	unsigned int pmc, evntsel, i;

	if (state->control.nrictrs != 0 || state->control.nractrs > 2)

		return -EINVAL;

	cesr_half[0] = 0;

	cesr_half[1] = 0;

	for(i = 0; i < state->control.nractrs; ++i) {

		pmc = state->control.pmc_map[i];

		state->pmc[i].map = pmc;

		if (pmc > 1 || cesr_half[pmc] != 0)

			return -EINVAL;

		evntsel = state->control.evntsel[i];

		/* protect reserved bits */

		if ((evntsel & reserved_bits) != 0)

			return -EPERM;

		/* the CPL field (if defined) must be non-zero */

		if (!is_c6 && !(evntsel & P5_CESR_CPL))

			return -EINVAL;

		cesr_half[pmc] = evntsel;

	}

	state->k1.id = (cesr_half[1] << 16) | cesr_half[0];

	return 0;

}

static int p5_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	return p5_like_check_control(state, P5_CESR_RESERVED, 0);

}

/* shared with MII but not C6 */

static void p5_write_control(const struct perfctr_cpu_state *state)

{

	struct per_cpu_cache *cache;

	unsigned int cesr;

	cesr = state->k1.id;

	if (!cesr)	/* no PMC is on (this test doesn't work on C6) */

		return;

	cache = get_cpu_cache();

	if (cache->k1.p5_cesr != cesr) {

		cache->k1.p5_cesr = cesr;

		wrmsr(MSR_P5_CESR, cesr, 0);

	}

}

static void p5_read_counters(const struct perfctr_cpu_state *state,

			     struct perfctr_low_ctrs *ctrs)

{

	unsigned int cstatus, nrctrs, i;

	/* The P5 doesn't allocate a cache line on a write miss, so do

	   a dummy read to avoid a write miss here _and_ a read miss

	   later in our caller. */

	asm("" : : "r"(ctrs->tsc));

	cstatus = state->cstatus;

	if (perfctr_cstatus_has_tsc(cstatus))

		rdtscl(ctrs->tsc);

	nrctrs = perfctr_cstatus_nractrs(cstatus);

	for(i = 0; i < nrctrs; ++i) {

		unsigned int pmc = state->pmc[i].map;

		rdmsr_low(MSR_P5_CTR0+pmc, ctrs->pmc[i]);

	}

}

/* used by all except pre-MMX P5 */

static void rdpmc_read_counters(const struct perfctr_cpu_state *state,

				struct perfctr_low_ctrs *ctrs)

{

	unsigned int cstatus, nrctrs, i;

	cstatus = state->cstatus;

	if (perfctr_cstatus_has_tsc(cstatus))

		rdtscl(ctrs->tsc);

	nrctrs = perfctr_cstatus_nractrs(cstatus);

	for(i = 0; i < nrctrs; ++i) {

		unsigned int pmc = state->pmc[i].map;

		rdpmc_low(pmc, ctrs->pmc[i]);

	}

}

/* shared with MII and C6 */

static const struct perfctr_msr_range p5_extras[] = {

	{ MSR_P5_CESR, 1+2 },

	{ 0, 0 },

};

static const struct perfctr_pmu_msrs p5_pmu_msrs = {

	.extras = p5_extras,

};

/*

 * Cyrix 6x86/MII/III.

 * - Same MSR assignments as P5 MMX. Has RDPMC and two 48-bit PMCs.

 * - Event codes and CESR formatting as in the plain P5 subset.

 * - Many but not all P5 MMX event codes are implemented.

 * - Cyrix adds a few more event codes. The event code is widened

 *   to 7 bits, and Cyrix puts the high bit in CESR bit 10

 *   (and CESR bit 26 for PMC1).

 */

static int mii_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	return p5_like_check_control(state, MII_CESR_RESERVED, 0);

}

/*

 * Centaur WinChip C6/2/3.

 * - Same MSR assignments as P5 MMX. Has RDPMC and two 40-bit PMCs.

 * - CESR is formatted with two halves, like P5. However, there

 *   are no defined control fields for e.g. CPL selection, and

 *   there is no defined method for stopping the counters.

 * - Only a few event codes are defined.

 * - The 64-bit TSC is synthesised from the low 32 bits of the

 *   two PMCs, and CESR has to be set up appropriately.

 *   Reprogramming CESR causes RDTSC to yield invalid results.

 *   (The C6 may also hang in this case, due to C6 erratum I-13.)

 *   Therefore, using the PMCs on any of these processors requires

 *   that the TSC is not accessed at all:

 *   1. The kernel must be configured or a TSC-less processor, i.e.

 *      generic 586 or less.

 *   2. The "notsc" boot parameter must be passed to the kernel.

 *   3. User-space libraries and code must also be configured and

 *      compiled for a generic 586 or less.

 */

#if !defined(CONFIG_X86_TSC)

static int c6_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	if (state->control.tsc_on)

		return -EINVAL;

	return p5_like_check_control(state, C6_CESR_RESERVED, 1);

}

static void c6_write_control(const struct perfctr_cpu_state *state)

{

	struct per_cpu_cache *cache;

	unsigned int cesr;

	if (perfctr_cstatus_nractrs(state->cstatus) == 0) /* no PMC is on */

		return;

	cache = get_cpu_cache();

	cesr = state->k1.id;

	if (cache->k1.p5_cesr != cesr) {

		cache->k1.p5_cesr = cesr;

		wrmsr(MSR_P5_CESR, cesr, 0);

	}

}

#endif	/* !CONFIG_X86_TSC */

/*

 * Intel P6 family (Pentium Pro, Pentium II, Pentium III, Pentium M,

 * Intel Core, Intel Core 2, Atom, and Core i7, including Xeon and Celeron versions.

 * - One TSC and two 40-bit PMCs.

 *   Core i7 has four 48-bit PMCs.

 * - One 32-bit EVNTSEL MSR for each PMC.

 * - EVNTSEL0 contains a global enable/disable bit.

 *   That bit is reserved in EVNTSEL1.

 *   On Core 2, Atom, and Core i7 each EVNTSEL has its own enable/disable bit.

 * - Each EVNTSEL contains a CPL field.

 * - Overflow interrupts are possible, but requires that the

 *   local APIC is available. Some Mobile P6s have no local APIC.

 * - The PMCs cannot be initialised with arbitrary values, since

 *   wrmsr fills the high bits by sign-extending from bit 31.

 * - Most events are symmetric, but a few are not.

 * - Core 2 adds three fixed-function counters. A single shared control

 *   register has the control bits (CPL:2 + PMI:1) for these counters.

 * - Initial Atoms appear to have one fixed-function counter.

 */

static int is_fam10h;

static int amd_is_multicore;		/* northbridge events need special care */

static int amd_is_k8_mc_RevE;

static cpumask_t amd_mc_core0_mask;	/* only these may use NB events */

static int p6_has_separate_enables;	/* affects EVNTSEL.ENable rules */

static unsigned int p6_nr_pmcs;	/* number of general-purpose counters */

static unsigned int p6_nr_ffcs;	/* number of fixed-function counters */

static unsigned int nhlm_nr_offcore_rsps; /* number of OFFCORE_RSP MSRs */

/* shared with K7 */

static int p6_like_check_control(struct perfctr_cpu_state *state, int is_k7, cpumask_t *cpumask)

{

	unsigned int evntsel, i, nractrs, nrctrs, pmc_mask, pmc;

	unsigned int core2_fixed_ctr_ctrl;

	unsigned int max_nrctrs;

	unsigned int amd_mc_nb_event_seen;

	max_nrctrs = is_k7 ? 4 : p6_nr_pmcs + p6_nr_ffcs;

	nractrs = state->control.nractrs;

	nrctrs = nractrs + state->control.nrictrs;

	if (nrctrs < nractrs || nrctrs > max_nrctrs)

		return -EINVAL;

	pmc_mask = 0;

	core2_fixed_ctr_ctrl = 0;	/* must be zero on CPUs != Core 2 */

	amd_mc_nb_event_seen = 0;

	for(i = 0; i < nrctrs; ++i) {

		pmc = state->control.pmc_map[i];

		state->pmc[i].map = pmc;

		/* pmc_map[i] is what we pass to RDPMC

		 * to check that pmc_map[] is well-defined on Core 2,

		 * we map FIXED_CTR 0x40000000+N to PMC p6_nr_pmcs+N

		 */

		if (!is_k7 && p6_nr_ffcs != 0) {

			if (pmc & CORE2_PMC_FIXED_FLAG)

				pmc = p6_nr_pmcs + (pmc & ~CORE2_PMC_FIXED_FLAG);

			else if (pmc >= p6_nr_pmcs)

				return -EINVAL;

		}

		if (pmc >= max_nrctrs || (pmc_mask & (1<

			return -EINVAL;

		pmc_mask |= (1<

		/*

		 * check evntsel_high on AMD Fam10h

		 * on others we force it to zero (should return -EINVAL but

		 * having zeroes there has not been a requirement before)

		 */

		if (is_fam10h) {

			unsigned int evntsel_high = state->control.evntsel_high[i];

			if (evntsel_high & FAM10H_EVNTSEL_HIGH_RESERVED)

				return -EINVAL;

		} else

			state->control.evntsel_high[i] = 0;

		/* check evntsel */

		evntsel = state->control.evntsel[i];

		/* handle per-thread counting of AMD multicore northbridge events */

		if (cpumask != NULL && amd_is_multicore && IS_K8_NB_EVENT(evntsel)) {

			/* K8 RevE NB event erratum is incompatible with per-thread counters */

			if (amd_is_k8_mc_RevE)

				return -EPERM;

			/* remember to restrict this session to amd_mc_core0_mask */

			amd_mc_nb_event_seen = 1;

		}

		/* protect reserved bits */

		if (evntsel & P6_EVNTSEL_RESERVED)

			return -EPERM;

		/* check ENable bit */

		if (is_k7 || p6_has_separate_enables) {

			/* ENable bit must be set in each evntsel */

			if (!(evntsel & P6_EVNTSEL_ENABLE))

				return -EINVAL;

		} else {

			/* only evntsel[0] has the ENable bit */

			if (evntsel & P6_EVNTSEL_ENABLE) {

				if (pmc > 0)

					return -EPERM;

			} else {

				if (pmc == 0)

					return -EINVAL;

			}

		}

		/* the CPL field must be non-zero */

		if (!(evntsel & P6_EVNTSEL_CPL))

			return -EINVAL;

		/* INT bit must be off for a-mode and on for i-mode counters */

		if (evntsel & P6_EVNTSEL_INT) {

			if (i < nractrs)

				return -EINVAL;

		} else {

			if (i >= nractrs)

				return -EINVAL;

		}

		if (!is_k7 && p6_nr_ffcs != 0) {

			pmc = state->control.pmc_map[i];

			if (pmc & CORE2_PMC_FIXED_FLAG) {

				unsigned int ctl = 0;

				ctl |= ((evntsel >> 17) & 1) << 0;	/* CPL.OS */

				ctl |= ((evntsel >> 16) & 1) << 1;	/* CPL.USR */

				ctl |= ((evntsel >> 20) & 1) << 3;	/* INT/PMI */

				core2_fixed_ctr_ctrl |= ctl << (pmc & CORE2_PMC_FIXED_MASK) * 4;

			}

		}

	}

	/*

	 * check offcore_rsp[] on Intel Nehalem

	 * on others we force it to zero (should return -EINVAL but

	 * having zeroes there has not been a requirement before)

	 */

	for (i = 0; i < 2; ++i) {

		if (i < nhlm_nr_offcore_rsps) {

			unsigned int offcore_rsp = state->control.nhlm.offcore_rsp[i];

			if (offcore_rsp & OFFCORE_RSP_RESERVED)

				return -EINVAL;

		} else

			state->control.nhlm.offcore_rsp[i] = 0;

	}

	state->core2_fixed_ctr_ctrl = core2_fixed_ctr_ctrl;

	state->k1.id = new_id();

	if (amd_mc_nb_event_seen)

		*cpumask = amd_mc_core0_mask;

	return 0;

}

static int p6_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	return p6_like_check_control(state, 0, cpumask);

}

#ifdef CONFIG_X86_LOCAL_APIC

/* PRE: perfctr_cstatus_has_ictrs(state->cstatus) != 0 */

/* shared with K7 and P4 */

static void p6_like_isuspend(struct perfctr_cpu_state *state,

			     unsigned int msr_evntsel0)

{

	struct per_cpu_cache *cache;

	unsigned int cstatus, nrctrs, i;

	int cpu;

	unsigned int pending = 0;

	cpu = smp_processor_id();

	set_isuspend_cpu(state, cpu); /* early to limit cpu's live range */

	cache = __get_cpu_cache(cpu);

	perfctr_cpu_mask_interrupts(cache);

	cstatus = state->cstatus;

	nrctrs = perfctr_cstatus_nrctrs(cstatus);

	if (state->core2_fixed_ctr_ctrl & MSR_CORE_PERF_FIXED_CTR_CTRL_PMIANY) {

		cache->core2_fixed_ctr_ctrl = 0;

		wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, 0, 0);

	}

	for(i = perfctr_cstatus_nractrs(cstatus); i < nrctrs; ++i) {

		unsigned int pmc_raw, pmc_idx, now;

		pmc_raw = state->pmc[i].map;

		if (!(pmc_raw & CORE2_PMC_FIXED_FLAG)) {

			/* Note: P4_MASK_FAST_RDPMC is a no-op for P6 and K7.

			   We don't need to make it into a parameter. */

			pmc_idx = pmc_raw & P4_MASK_FAST_RDPMC;

			cache->control.evntsel[pmc_idx] = 0;

			cache->control.evntsel_high[pmc_idx] = 0;

			/* On P4 this intensionally also clears the CCCR.OVF flag. */

			wrmsr(msr_evntsel0+pmc_idx, 0, 0);

		}

		/* P4 erratum N17 does not apply since we read only low 32 bits. */

		rdpmc_low(pmc_raw, now);

		state->pmc[i].sum += now - state->pmc[i].start;

		state->pmc[i].start = now;

		if ((int)now >= 0)

			++pending;

	}

	state->pending_interrupt = pending;

	/* cache->k1.id is still == state->k1.id */

}

/* PRE: perfctr_cstatus_has_ictrs(state->cstatus) != 0 */

/* shared with K7 and P4 */

static void p6_like_iresume(const struct perfctr_cpu_state *state,

			    unsigned int msr_evntsel0,

			    unsigned int msr_perfctr0)

{

	struct per_cpu_cache *cache;

	unsigned int cstatus, nrctrs, i;

	int cpu;

	cpu = smp_processor_id();

	cache = __get_cpu_cache(cpu);

	perfctr_cpu_unmask_interrupts(cache);

	if (cache->k1.id == state->k1.id) {

		cache->k1.id = 0; /* force reload of cleared EVNTSELs */

		if (is_isuspend_cpu(state, cpu))

			return; /* skip reload of PERFCTRs */

	}

	cstatus = state->cstatus;

	nrctrs = perfctr_cstatus_nrctrs(cstatus);

	/* If the control wasn't ours we must disable the

	   counters before reinitialising them. */

	if ((state->core2_fixed_ctr_ctrl & MSR_CORE_PERF_FIXED_CTR_CTRL_PMIANY) &&

	    cache->core2_fixed_ctr_ctrl != 0) {

		cache->core2_fixed_ctr_ctrl = 0;

		wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, 0, 0);

	}

	for(i = perfctr_cstatus_nractrs(cstatus); i < nrctrs; ++i) {

		unsigned int pmc_raw = state->pmc[i].map;

		unsigned int msr_perfctr;

		unsigned int pmc_value_hi;

		if (pmc_raw & CORE2_PMC_FIXED_FLAG) {

			msr_perfctr = MSR_CORE_PERF_FIXED_CTR0 + (pmc_raw & CORE2_PMC_FIXED_MASK);

			/* Limit the value written to a fixed-function counter's MSR

			 * to 40 bits. Extraneous high bits cause GP faults on Model 23

			 * Core2s, while earlier processors would just ignore them.

			 */

			pmc_value_hi = 0xff;

		} else {

			/* Note: P4_MASK_FAST_RDPMC is a no-op for P6 and K7.

			   We don't need to make it into a parameter. */

			unsigned int pmc_idx = pmc_raw & P4_MASK_FAST_RDPMC;

			/* If the control wasn't ours we must disable the evntsels

			   before reinitialising the counters, to prevent unexpected

			   counter increments and missed overflow interrupts. */

			if (cache->control.evntsel[pmc_idx]) {

				cache->control.evntsel[pmc_idx] = 0;

				cache->control.evntsel_high[pmc_idx] = 0;

				wrmsr(msr_evntsel0+pmc_idx, 0, 0);

			}

			msr_perfctr = msr_perfctr0 + pmc_idx;

			pmc_value_hi = -1;

		}

		/* P4 erratum N15 does not apply since the CCCR is disabled. */

		wrmsr(msr_perfctr, state->pmc[i].start, pmc_value_hi);

	}

	/* cache->k1.id remains != state->k1.id */

}

static void p6_isuspend(struct perfctr_cpu_state *state)

{

	p6_like_isuspend(state, MSR_P6_EVNTSEL0);

}

static void p6_iresume(const struct perfctr_cpu_state *state)

{

	p6_like_iresume(state, MSR_P6_EVNTSEL0, MSR_P6_PERFCTR0);

}

#endif	/* CONFIG_X86_LOCAL_APIC */

/* shared with K7 and VC3 */

static void p6_like_write_control(const struct perfctr_cpu_state *state,

				  unsigned int msr_evntsel0)

{

	struct per_cpu_cache *cache;

	unsigned int nrctrs, i;

	cache = get_cpu_cache();

	if (cache->k1.id == state->k1.id)

		return;

	nrctrs = perfctr_cstatus_nrctrs(state->cstatus);

	for(i = 0; i < nrctrs; ++i) {

		unsigned int pmc, evntsel, evntsel_high;

		pmc = state->pmc[i].map;

		if (pmc & CORE2_PMC_FIXED_FLAG)

			continue;

		evntsel = state->control.evntsel[i];

		evntsel_high = state->control.evntsel_high[i];

		if (evntsel != cache->control.evntsel[pmc] ||

		    evntsel_high != cache->control.evntsel_high[pmc]) {

			cache->control.evntsel[pmc] = evntsel;

			cache->control.evntsel_high[pmc] = evntsel_high;

			wrmsr(msr_evntsel0+pmc, evntsel, evntsel_high);

		}

	}

	if (state->core2_fixed_ctr_ctrl != 0 &&

	    state->core2_fixed_ctr_ctrl != cache->core2_fixed_ctr_ctrl) {

		cache->core2_fixed_ctr_ctrl = state->core2_fixed_ctr_ctrl;

		wrmsr(MSR_CORE_PERF_FIXED_CTR_CTRL, state->core2_fixed_ctr_ctrl, 0);

	}

	for (i = 0; i < 2; ++i) {

		unsigned int offcore_rsp;

		offcore_rsp = state->control.nhlm.offcore_rsp[i];

		if (offcore_rsp != cache->nhlm_offcore_rsp[i]) {

			cache->nhlm_offcore_rsp[i] = offcore_rsp;

			wrmsr(MSR_OFFCORE_RSP0+i, offcore_rsp, 0);

		}

	}

	cache->k1.id = state->k1.id;

}

/* shared with VC3, Generic*/

static void p6_write_control(const struct perfctr_cpu_state *state)

{

	p6_like_write_control(state, MSR_P6_EVNTSEL0);

}

static struct perfctr_msr_range p6_perfctrs[] = {

	{ MSR_P6_PERFCTR0, 2 }, /* on Core i7 we'll update this count */

	{ 0, 0 },

};

static struct perfctr_msr_range p6_evntsels[] = {

	{ MSR_P6_EVNTSEL0, 2 }, /* on Core i7 we'll update this count */

	{ 0, 0 },

};

static const struct perfctr_pmu_msrs p6_pmu_msrs = {

	.perfctrs = p6_perfctrs,

	.evntsels = p6_evntsels,

};

static struct perfctr_msr_range core2_extras[] = {

	{ MSR_CORE_PERF_FIXED_CTR0, 3 }, /* on Atom we'll update this count */

	{ MSR_CORE_PERF_FIXED_CTR_CTRL, 1 },

	{ MSR_OFFCORE_RSP0, 0 },	/* on Nehalem we'll update this count */

	{ 0, 0 },

};

static void core2_clear_counters(int init)

{

	if (init) {

		unsigned int low, high;

		rdmsr(MSR_IA32_DEBUGCTLMSR, low, high);

		low &= ~MSR_IA32_DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI;

		wrmsr(MSR_IA32_DEBUGCTLMSR, low, high);

		wrmsr(MSR_CORE_PERF_GLOBAL_CTRL, (1 << p6_nr_pmcs) - 1, (1 << p6_nr_ffcs) - 1);

	}

}

static const struct perfctr_pmu_msrs core2_pmu_msrs = {

	.perfctrs = p6_perfctrs,

	.evntsels = p6_evntsels,

	.extras = core2_extras,

	.clear_counters = core2_clear_counters,

};

/*

 * AMD K7 family (Athlon, Duron).

 * - Somewhat similar to the Intel P6 family.

 * - Four 48-bit PMCs.

 * - Four 32-bit EVNTSEL MSRs with similar layout as in P6.

 * - Completely different MSR assignments :-(

 * - Fewer countable events defined :-(

 * - The events appear to be completely symmetric.

 * - The EVNTSEL MSRs are symmetric since each has its own enable bit.

 * - Publicly available documentation is incomplete.

 * - K7 model 1 does not have a local APIC. AMD Document #22007

 *   Revision J hints that it may use debug interrupts instead.

 *

 * The K8 has the same hardware layout as the K7. It also has

 * better documentation and a different set of available events.

 *

 * AMD Family 10h is similar to the K7, but the EVNTSEL MSRs

 * have been widened to 64 bits.

 */

static int k7_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	return p6_like_check_control(state, 1, cpumask);

}

#ifdef CONFIG_X86_LOCAL_APIC

static void k7_isuspend(struct perfctr_cpu_state *state)

{

	p6_like_isuspend(state, MSR_K7_EVNTSEL0);

}

static void k7_iresume(const struct perfctr_cpu_state *state)

{

	p6_like_iresume(state, MSR_K7_EVNTSEL0, MSR_K7_PERFCTR0);

}

#endif	/* CONFIG_X86_LOCAL_APIC */

static void k7_write_control(const struct perfctr_cpu_state *state)

{

	p6_like_write_control(state, MSR_K7_EVNTSEL0);

}

static const struct perfctr_msr_range k7_perfctrs[] = {

	{ MSR_K7_PERFCTR0, 4 },

	{ 0, 0 },

};

static const struct perfctr_msr_range k7_evntsels[] = {

	{ MSR_K7_EVNTSEL0, 4 },

	{ 0, 0 },

};

static const struct perfctr_pmu_msrs k7_pmu_msrs = {

	.perfctrs = k7_perfctrs,

	.evntsels = k7_evntsels,

};

/*

 * VIA C3 family.

 * - A Centaur design somewhat similar to the P6/Celeron.

 * - PERFCTR0 is an alias for the TSC, and EVNTSEL0 is read-only.

 * - PERFCTR1 is 32 bits wide.

 * - EVNTSEL1 has no defined control fields, and there is no

 *   defined method for stopping the counter.

 * - According to testing, the reserved fields in EVNTSEL1 have

 *   no function. We always fill them with zeroes.

 * - Only a few event codes are defined.

 * - No local APIC or interrupt-mode support.

 * - pmc_map[0] must be 1, if nractrs == 1.

 */

static int vc3_check_control(struct perfctr_cpu_state *state, cpumask_t *cpumask)

{

	if (state->control.nrictrs || state->control.nractrs > 1)

		return -EINVAL;

	if (state->control.nractrs == 1) {

		if (state->control.pmc_map[0] != 1)

			return -EINVAL;

		state->pmc[0].map = 1;

		if (state->control.evntsel[0] & VC3_EVNTSEL1_RESERVED)

			return -EPERM;

		state->k1.id = state->control.evntsel[0];

	} else

		state->k1.id = 0;

	return 0;

}

static void vc3_clear_counters(int init)

{

	/* Not documented, but seems to be default after boot. */

	wrmsr(MSR_P6_EVNTSEL0+1, 0x00070079, 0);

}

static const struct perfctr_pmu_msrs vc3_pmu_msrs = {

	.clear_counters = vc3_clear_counters,

};

/*

 * Intel Pentium 4.

 * Current implementation restrictions:

 * - No DS/PEBS support.

 *

 * Known quirks:

 * - OVF_PMI+FORCE_OVF counters must have an ireset value of -1.

 *   This allows the regular overflow check to also handle FORCE_OVF

 *   counters. Not having this restriction would lead to MAJOR

 *   complications in the driver's "detect overflow counters" code.

 *   There is no loss of functionality since the ireset value doesn't

 *   affect the counter's PMI rate for FORCE_OVF counters.

 * - In experiments with FORCE_OVF counters, and regular OVF_PMI