/arch/x86_64/kernel/smp.c

  1/*
  2 *	Intel SMP support routines.
  3 *
  4 *	(c) 1995 Alan Cox, Building #3 <alan@redhat.com>
  5 *	(c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
  6 *      (c) 2002,2003 Andi Kleen, SuSE Labs.
  7 *
  8 *	This code is released under the GNU General Public License version 2 or
  9 *	later.
 10 */
 11
 12#include <linux/init.h>
 13
 14#include <linux/mm.h>
 15#include <linux/irq.h>
 16#include <linux/delay.h>
 17#include <linux/spinlock.h>
 18#include <linux/smp_lock.h>
 19#include <linux/smp.h>
 20#include <linux/kernel_stat.h>
 21#include <linux/mc146818rtc.h>
 22#include <linux/interrupt.h>
 23
 24#include <asm/mtrr.h>
 25#include <asm/pgalloc.h>
 26#include <asm/tlbflush.h>
 27#include <asm/mach_apic.h>
 28#include <asm/mmu_context.h>
 29#include <asm/proto.h>
 30
 31/*
 32 *	Smarter SMP flushing macros. 
 33 *		c/o Linus Torvalds.
 34 *
 35 *	These mean you can really definitely utterly forget about
 36 *	writing to user space from interrupts. (Its not allowed anyway).
 37 *
 38 *	Optimizations Manfred Spraul <manfred@colorfullife.com>
 39 */
 40
 41static cpumask_t flush_cpumask;
 42static struct mm_struct * flush_mm;
 43static unsigned long flush_va;
 44static DEFINE_SPINLOCK(tlbstate_lock);
 45#define FLUSH_ALL	-1ULL
 46
 47/*
 48 * We cannot call mmdrop() because we are in interrupt context, 
 49 * instead update mm->cpu_vm_mask.
 50 */
 51static inline void leave_mm (unsigned long cpu)
 52{
 53	if (read_pda(mmu_state) == TLBSTATE_OK)
 54		BUG();
 55	clear_bit(cpu, &read_pda(active_mm)->cpu_vm_mask);
 56	load_cr3(swapper_pg_dir);
 57}
 58
 59/*
 60 *
 61 * The flush IPI assumes that a thread switch happens in this order:
 62 * [cpu0: the cpu that switches]
 63 * 1) switch_mm() either 1a) or 1b)
 64 * 1a) thread switch to a different mm
 65 * 1a1) clear_bit(cpu, &old_mm->cpu_vm_mask);
 66 * 	Stop ipi delivery for the old mm. This is not synchronized with
 67 * 	the other cpus, but smp_invalidate_interrupt ignore flush ipis
 68 * 	for the wrong mm, and in the worst case we perform a superfluous
 69 * 	tlb flush.
 70 * 1a2) set cpu mmu_state to TLBSTATE_OK
 71 * 	Now the smp_invalidate_interrupt won't call leave_mm if cpu0
 72 *	was in lazy tlb mode.
 73 * 1a3) update cpu active_mm
 74 * 	Now cpu0 accepts tlb flushes for the new mm.
 75 * 1a4) set_bit(cpu, &new_mm->cpu_vm_mask);
 76 * 	Now the other cpus will send tlb flush ipis.
 77 * 1a4) change cr3.
 78 * 1b) thread switch without mm change
 79 *	cpu active_mm is correct, cpu0 already handles
 80 *	flush ipis.
 81 * 1b1) set cpu mmu_state to TLBSTATE_OK
 82 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
 83 * 	Atomically set the bit [other cpus will start sending flush ipis],
 84 * 	and test the bit.
 85 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
 86 * 2) switch %%esp, ie current
 87 *
 88 * The interrupt must handle 2 special cases:
 89 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
 90 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
 91 *   runs in kernel space, the cpu could load tlb entries for user space
 92 *   pages.
 93 *
 94 * The good news is that cpu mmu_state is local to each cpu, no
 95 * write/read ordering problems.
 96 */
 97
 98/*
 99 * TLB flush IPI:
100 *
101 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
102 * 2) Leave the mm if we are in the lazy tlb mode.
103 */
104
105asmlinkage void smp_invalidate_interrupt (void)
106{
107	unsigned long cpu;
108
109	cpu = get_cpu();
110
111	if (!cpu_isset(cpu, flush_cpumask))
112		goto out;
113		/* 
114		 * This was a BUG() but until someone can quote me the
115		 * line from the intel manual that guarantees an IPI to
116		 * multiple CPUs is retried _only_ on the erroring CPUs
117		 * its staying as a return
118		 *
119		 * BUG();
120		 */
121		 
122	if (flush_mm == read_pda(active_mm)) {
123		if (read_pda(mmu_state) == TLBSTATE_OK) {
124			if (flush_va == FLUSH_ALL)
125				local_flush_tlb();
126			else
127				__flush_tlb_one(flush_va);
128		} else
129			leave_mm(cpu);
130	}
131	ack_APIC_irq();
132	cpu_clear(cpu, flush_cpumask);
133
134out:
135	put_cpu_no_resched();
136}
137
138static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
139						unsigned long va)
140{
141	cpumask_t tmp;
142	/*
143	 * A couple of (to be removed) sanity checks:
144	 *
145	 * - we do not send IPIs to not-yet booted CPUs.
146	 * - current CPU must not be in mask
147	 * - mask must exist :)
148	 */
149	BUG_ON(cpus_empty(cpumask));
150	cpus_and(tmp, cpumask, cpu_online_map);
151	BUG_ON(!cpus_equal(tmp, cpumask));
152	BUG_ON(cpu_isset(smp_processor_id(), cpumask));
153	if (!mm)
154		BUG();
155
156	/*
157	 * I'm not happy about this global shared spinlock in the
158	 * MM hot path, but we'll see how contended it is.
159	 * Temporarily this turns IRQs off, so that lockups are
160	 * detected by the NMI watchdog.
161	 */
162	spin_lock(&tlbstate_lock);
163	
164	flush_mm = mm;
165	flush_va = va;
166	cpus_or(flush_cpumask, cpumask, flush_cpumask);
167
168	/*
169	 * We have to send the IPI only to
170	 * CPUs affected.
171	 */
172	send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
173
174	while (!cpus_empty(flush_cpumask))
175		mb();	/* nothing. lockup detection does not belong here */;
176
177	flush_mm = NULL;
178	flush_va = 0;
179	spin_unlock(&tlbstate_lock);
180}
181	
182void flush_tlb_current_task(void)
183{
184	struct mm_struct *mm = current->mm;
185	cpumask_t cpu_mask;
186
187	preempt_disable();
188	cpu_mask = mm->cpu_vm_mask;
189	cpu_clear(smp_processor_id(), cpu_mask);
190
191	local_flush_tlb();
192	if (!cpus_empty(cpu_mask))
193		flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
194	preempt_enable();
195}
196
197void flush_tlb_mm (struct mm_struct * mm)
198{
199	cpumask_t cpu_mask;
200
201	preempt_disable();
202	cpu_mask = mm->cpu_vm_mask;
203	cpu_clear(smp_processor_id(), cpu_mask);
204
205	if (current->active_mm == mm) {
206		if (current->mm)
207			local_flush_tlb();
208		else
209			leave_mm(smp_processor_id());
210	}
211	if (!cpus_empty(cpu_mask))
212		flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
213
214	preempt_enable();
215}
216
217void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
218{
219	struct mm_struct *mm = vma->vm_mm;
220	cpumask_t cpu_mask;
221
222	preempt_disable();
223	cpu_mask = mm->cpu_vm_mask;
224	cpu_clear(smp_processor_id(), cpu_mask);
225
226	if (current->active_mm == mm) {
227		if(current->mm)
228			__flush_tlb_one(va);
229		 else
230		 	leave_mm(smp_processor_id());
231	}
232
233	if (!cpus_empty(cpu_mask))
234		flush_tlb_others(cpu_mask, mm, va);
235
236	preempt_enable();
237}
238
239static void do_flush_tlb_all(void* info)
240{
241	unsigned long cpu = smp_processor_id();
242
243	__flush_tlb_all();
244	if (read_pda(mmu_state) == TLBSTATE_LAZY)
245		leave_mm(cpu);
246}
247
248void flush_tlb_all(void)
249{
250	on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
251}
252
253void smp_kdb_stop(void)
254{
255	send_IPI_allbutself(KDB_VECTOR);
256}
257
258/*
259 * this function sends a 'reschedule' IPI to another CPU.
260 * it goes straight through and wastes no time serializing
261 * anything. Worst case is that we lose a reschedule ...
262 */
263
264void smp_send_reschedule(int cpu)
265{
266	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
267}
268
269/*
270 * Structure and data for smp_call_function(). This is designed to minimise
271 * static memory requirements. It also looks cleaner.
272 */
273static DEFINE_SPINLOCK(call_lock);
274
275struct call_data_struct {
276	void (*func) (void *info);
277	void *info;
278	atomic_t started;
279	atomic_t finished;
280	int wait;
281};
282
283static struct call_data_struct * call_data;
284
285/*
286 * this function sends a 'generic call function' IPI to all other CPUs
287 * in the system.
288 */
289static void __smp_call_function (void (*func) (void *info), void *info,
290				int nonatomic, int wait)
291{
292	struct call_data_struct data;
293	int cpus = num_online_cpus()-1;
294
295	if (!cpus)
296		return;
297
298	data.func = func;
299	data.info = info;
300	atomic_set(&data.started, 0);
301	data.wait = wait;
302	if (wait)
303		atomic_set(&data.finished, 0);
304
305	call_data = &data;
306	wmb();
307	/* Send a message to all other CPUs and wait for them to respond */
308	send_IPI_allbutself(CALL_FUNCTION_VECTOR);
309
310	/* Wait for response */
311	while (atomic_read(&data.started) != cpus)
312		cpu_relax();
313
314	if (!wait)
315		return;
316
317	while (atomic_read(&data.finished) != cpus)
318		cpu_relax();
319}
320
321/*
322 * smp_call_function - run a function on all other CPUs.
323 * @func: The function to run. This must be fast and non-blocking.
324 * @info: An arbitrary pointer to pass to the function.
325 * @nonatomic: currently unused.
326 * @wait: If true, wait (atomically) until function has completed on other
327 *        CPUs.
328 *
329 * Returns 0 on success, else a negative status code. Does not return until
330 * remote CPUs are nearly ready to execute func or are or have executed.
331 *
332 * You must not call this function with disabled interrupts or from a
333 * hardware interrupt handler or from a bottom half handler.
334 * Actually there are a few legal cases, like panic.
335 */
336int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
337			int wait)
338{
339	spin_lock(&call_lock);
340	__smp_call_function(func,info,nonatomic,wait);
341	spin_unlock(&call_lock);
342	return 0;
343}
344
345void smp_stop_cpu(void)
346{
347	/*
348	 * Remove this CPU:
349	 */
350	cpu_clear(smp_processor_id(), cpu_online_map);
351	local_irq_disable();
352	disable_local_APIC();
353	local_irq_enable(); 
354}
355
356static void smp_really_stop_cpu(void *dummy)
357{
358	smp_stop_cpu(); 
359	for (;;) 
360		asm("hlt"); 
361} 
362
363void smp_send_stop(void)
364{
365	int nolock = 0;
366	if (reboot_force)
367		return;
368	/* Don't deadlock on the call lock in panic */
369	if (!spin_trylock(&call_lock)) {
370		/* ignore locking because we have paniced anyways */
371		nolock = 1;
372	}
373	__smp_call_function(smp_really_stop_cpu, NULL, 0, 0);
374	if (!nolock)
375		spin_unlock(&call_lock);
376
377	local_irq_disable();
378	disable_local_APIC();
379	local_irq_enable();
380}
381
382/*
383 * Reschedule call back. Nothing to do,
384 * all the work is done automatically when
385 * we return from the interrupt.
386 */
387asmlinkage void smp_reschedule_interrupt(void)
388{
389	ack_APIC_irq();
390}
391
392asmlinkage void smp_call_function_interrupt(void)
393{
394	void (*func) (void *info) = call_data->func;
395	void *info = call_data->info;
396	int wait = call_data->wait;
397
398	ack_APIC_irq();
399	/*
400	 * Notify initiating CPU that I've grabbed the data and am
401	 * about to execute the function
402	 */
403	mb();
404	atomic_inc(&call_data->started);
405	/*
406	 * At this point the info structure may be out of scope unless wait==1
407	 */
408	irq_enter();
409	(*func)(info);
410	irq_exit();
411	if (wait) {
412		mb();
413		atomic_inc(&call_data->finished);
414	}
415}