![[first]](../icons/first.png){kind=icon}
![[previous]](../icons/n_left.png){kind=icon}
![[next]](../icons/n_right.png){kind=icon}
![[last]](../icons/last.png){kind=icon}
![[top]](../icons/n_top.png){kind=icon}
![[bottom]](../icons/bottom.png){kind=icon}
![[index]](../icons/index.png){kind=icon}
![[help]](../icons/help.png){kind=icon}
*/
1 /*
2 * Intel MP v1.1/v1.4 specification support routines for multi-pentium
3 * hosts.
4 *
5 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
6 * Supported by Caldera http://www.caldera.com.
7 * Much of the core SMP work is based on previous work by Thomas Radke, to
8 * whom a great many thanks are extended.
9 *
10 * Thanks to Intel for making available several different Pentium and
11 * Pentium Pro MP machines.
12 *
13 * This code is released under the GNU public license version 2 or
14 * later.
15 *
16 * Fixes
17 * Felix Koop : NR_CPUS used properly
18 * Jose Renau : Handle single CPU case.
19 * Alan Cox : By repeated request 8) - Total BogoMIP report.
20 * Greg Wright : Fix for kernel stacks panic.
21 * Erich Boleyn : MP v1.4 and additional changes.
22 */
23
24 #include <linux/kernel.h>
25 #include <linux/string.h>
26 #include <linux/timer.h>
27 #include <linux/sched.h>
28 #include <linux/mm.h>
29 #include <linux/kernel_stat.h>
30 #include <linux/delay.h>
31 #include <linux/mc146818rtc.h>
32 #include <asm/i82489.h>
33 #include <linux/smp.h>
34 #include <asm/pgtable.h>
35 #include <asm/bitops.h>
36 #include <asm/pgtable.h>
37 #include <asm/smp.h>
38
39 /*
40 * Why isnt this somewhere standard ??
41 */
42
43 extern __inline int max(int a,int b)
/* ![[next]](../icons/right.png)
![[first]](../icons/n_first.png)
![[top]](../icons/top.png)
*/
44 {
45 if(a>b)
46 return a;
47 return b;
48 }
49
50
51 int smp_found_config=0; /* Have we found an SMP box */
52
53 unsigned long cpu_present_map = 0; /* Bitmask of existing CPU's */
54 int smp_num_cpus; /* Total count of live CPU's */
55 int smp_threads_ready=0; /* Set when the idlers are all forked */
56 volatile int cpu_number_map[NR_CPUS]; /* which CPU maps to which logical number */
57 volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; /* We always use 0 the rest is ready for parallel delivery */
58 volatile unsigned long smp_invalidate_needed; /* Used for the invalidate map thats also checked in the spinlock */
59 struct cpuinfo_x86 cpu_data[NR_CPUS]; /* Per cpu bogomips and other parameters */
60 static unsigned int num_processors = 1; /* Internal processor count */
61 int smp_top_cpu = 0; /* Highest used APIC id */
62 static unsigned long io_apic_addr = 0xFEC00000; /* Address of the I/O apic (not yet used) */
63 unsigned char boot_cpu_id = 0; /* Processor that is doing the boot up */
64 static unsigned char *kstack_base,*kstack_end; /* Kernel stack list pointers */
65 static int smp_activated = 0; /* Tripped once we need to start cross invalidating */
66 int apic_version[NR_CPUS]; /* APIC version number */
67 static volatile int smp_commenced=0; /* Tripped when we start scheduling */
68 unsigned long apic_addr=0xFEE00000; /* Address of APIC (defaults to 0xFEE00000) */
69 unsigned long nlong = 0; /* dummy used for apic_reg address + 0x20 */
70 unsigned char *apic_reg=((unsigned char *)(&nlong))-0x20;/* Later set to the vremap() of the APIC */
71 unsigned long apic_retval; /* Just debugging the assembler.. */
72 unsigned char *kernel_stacks[NR_CPUS]; /* Kernel stack pointers for CPU's (debugging) */
73
74 static volatile unsigned char smp_cpu_in_msg[NR_CPUS]; /* True if this processor is sending an IPI */
75 static volatile unsigned long smp_msg_data; /* IPI data pointer */
76 static volatile int smp_src_cpu; /* IPI sender processor */
77 static volatile int smp_msg_id; /* Message being sent */
78
79 volatile unsigned long kernel_flag=0; /* Kernel spinlock */
80 volatile unsigned char active_kernel_processor = NO_PROC_ID; /* Processor holding kernel spinlock */
81 volatile unsigned long kernel_counter=0; /* Number of times the processor holds the lock */
82 volatile unsigned long syscall_count=0; /* Number of times the processor holds the syscall lock */
83
84 volatile unsigned long ipi_count; /* Number of IPI's delivered */
85 #ifdef __SMP_PROF__
86 volatile unsigned long smp_spins[NR_CPUS]={0}; /* Count interrupt spins */
87 volatile unsigned long smp_spins_syscall[NR_CPUS]={0}; /* Count syscall spins */
88 volatile unsigned long smp_spins_syscall_cur[NR_CPUS]={0};/* Count spins for the actual syscall */
89 volatile unsigned long smp_spins_sys_idle[NR_CPUS]={0}; /* Count spins for sys_idle */
90 volatile unsigned long smp_idle_count[1+NR_CPUS]={0,}; /* Count idle ticks */
91 #endif
92 #if defined (__SMP_PROF__)
93 volatile unsigned long smp_idle_map=0; /* Map for idle processors */
94 #endif
95
96 volatile unsigned long smp_proc_in_lock[NR_CPUS] = {0,};/* for computing process time */
97 volatile unsigned long smp_process_available=0;
98
99 /*#define SMP_DEBUG*/
100
101 #ifdef SMP_DEBUG
102 #define SMP_PRINTK(x) printk x
103 #else
104 #define SMP_PRINTK(x)
105 #endif
106
107
108 /*
109 * Checksum an MP configuration block.
110 */
111
112 static int mpf_checksum(unsigned char *mp, int len)
/* ![[previous]](../icons/left.png)
*/
113 {
114 int sum=0;
115 while(len--)
116 sum+=*mp++;
117 return sum&0xFF;
118 }
119
120 /*
121 * Processor encoding in an MP configuration block
122 */
123
124 static char *mpc_family(int family,int model)
/* */
125 {
126 static char n[32];
127 static char *model_defs[]=
128 {
129 "80486DX","80486DX",
130 "80486SX","80486DX/2 or 80487",
131 "80486SL","Intel5X2(tm)",
132 "Unknown","Unknown",
133 "80486DX/4"
134 };
135 if(family==0x6)
136 return("Pentium(tm) Pro");
137 if(family==0x5)
138 return("Pentium(tm)");
139 if(family==0x0F && model==0x0F)
140 return("Special controller");
141 if(family==0x04 && model<9)
142 return model_defs[model];
143 sprintf(n,"Unknown CPU [%d:%d]",family, model);
144 return n;
145 }
146
147 /*
148 * Read the MPC
149 */
150
151 static int smp_read_mpc(struct mp_config_table *mpc)
/* */
152 {
153 char str[16];
154 int count=sizeof(*mpc);
155 int apics=0;
156 unsigned char *mpt=((unsigned char *)mpc)+count;
157
158 if(memcmp(mpc->mpc_signature,MPC_SIGNATURE,4))
159 {
160 printk("Bad signature [%c%c%c%c].\n",
161 mpc->mpc_signature[0],
162 mpc->mpc_signature[1],
163 mpc->mpc_signature[2],
164 mpc->mpc_signature[3]);
165 return 1;
166 }
167 if(mpf_checksum((unsigned char *)mpc,mpc->mpc_length))
168 {
169 printk("Checksum error.\n");
170 return 1;
171 }
172 if(mpc->mpc_spec!=0x01 && mpc->mpc_spec!=0x04)
173 {
174 printk("Bad Config Table version (%d)!!\n",mpc->mpc_spec);
175 return 1;
176 }
177 memcpy(str,mpc->mpc_oem,8);
178 str[8]=0;
179 printk("OEM ID: %s ",str);
180 memcpy(str,mpc->mpc_productid,12);
181 str[12]=0;
182 printk("Product ID: %s ",str);
183 printk("APIC at: 0x%lX\n",mpc->mpc_lapic);
184
185 /* set the local APIC address */
186 apic_addr = mpc->mpc_lapic;
187
188 /*
189 * Now process the configuration blocks.
190 */
191
192 while(count<mpc->mpc_length)
193 {
194 switch(*mpt)
195 {
196 case MP_PROCESSOR:
197 {
198 struct mpc_config_processor *m=
199 (struct mpc_config_processor *)mpt;
200 if(m->mpc_cpuflag&CPU_ENABLED)
201 {
202 printk("Processor #%d %s APIC version %d\n",
203 m->mpc_apicid,
204 mpc_family((m->mpc_cpufeature&
205 CPU_FAMILY_MASK)>>8,
206 (m->mpc_cpufeature&
207 CPU_MODEL_MASK)>>4),
208 m->mpc_apicver);
209 #ifdef SMP_DEBUG
210 if(m->mpc_featureflag&(1<<0))
211 printk(" Floating point unit present.\n");
212 if(m->mpc_featureflag&(1<<7))
213 printk(" Machine Exception supported.\n");
214 if(m->mpc_featureflag&(1<<8))
215 printk(" 64 bit compare & exchange supported.\n");
216 if(m->mpc_featureflag&(1<<9))
217 printk(" Internal APIC present.\n");
218 #endif
219 if(m->mpc_cpuflag&CPU_BOOTPROCESSOR)
220 {
221 SMP_PRINTK((" Bootup CPU\n"));
222 boot_cpu_id=m->mpc_apicid;
223 nlong = boot_cpu_id<<24; /* Dummy 'self' for bootup */
224 }
225 else /* Boot CPU already counted */
226 num_processors++;
227
228 if(m->mpc_apicid>NR_CPUS)
229 printk("Processor #%d unused. (Max %d processors).\n",m->mpc_apicid, NR_CPUS);
230 else
231 {
232 cpu_present_map|=(1<<m->mpc_apicid);
233 apic_version[m->mpc_apicid]=m->mpc_apicver;
234 }
235 }
236 mpt+=sizeof(*m);
237 count+=sizeof(*m);
238 break;
239 }
240 case MP_BUS:
241 {
242 struct mpc_config_bus *m=
243 (struct mpc_config_bus *)mpt;
244 memcpy(str,m->mpc_bustype,6);
245 str[6]=0;
246 SMP_PRINTK(("Bus #%d is %s\n",
247 m->mpc_busid,
248 str));
249 mpt+=sizeof(*m);
250 count+=sizeof(*m);
251 break;
252 }
253 case MP_IOAPIC:
254 {
255 struct mpc_config_ioapic *m=
256 (struct mpc_config_ioapic *)mpt;
257 if(m->mpc_flags&MPC_APIC_USABLE)
258 {
259 apics++;
260 printk("I/O APIC #%d Version %d at 0x%lX.\n",
261 m->mpc_apicid,m->mpc_apicver,
262 m->mpc_apicaddr);
263 io_apic_addr = m->mpc_apicaddr;
264 }
265 mpt+=sizeof(*m);
266 count+=sizeof(*m);
267 break;
268 }
269 case MP_INTSRC:
270 {
271 struct mpc_config_intsrc *m=
272 (struct mpc_config_intsrc *)mpt;
273
274 mpt+=sizeof(*m);
275 count+=sizeof(*m);
276 break;
277 }
278 case MP_LINTSRC:
279 {
280 struct mpc_config_intlocal *m=
281 (struct mpc_config_intlocal *)mpt;
282 mpt+=sizeof(*m);
283 count+=sizeof(*m);
284 break;
285 }
286 }
287 }
288 if(apics>1)
289 printk("Warning: Multiple APIC's not supported.\n");
290 return num_processors;
291 }
292
293 /*
294 * Scan the memory blocks for an SMP configuration block.
295 */
296
297 void smp_scan_config(unsigned long base, unsigned long length)
/* */
298 {
299 unsigned long *bp=(unsigned long *)base;
300 struct intel_mp_floating *mpf;
301
302 SMP_PRINTK(("Scan SMP from %p for %ld bytes.\n",
303 bp,length));
304 if(sizeof(*mpf)!=16)
305 printk("Error: MPF size\n");
306
307 while(length>0)
308 {
309 if(*bp==SMP_MAGIC_IDENT)
310 {
311 mpf=(struct intel_mp_floating *)bp;
312 if(mpf->mpf_length==1 &&
313 !mpf_checksum((unsigned char *)bp,16) &&
314 (mpf->mpf_specification == 1
315 || mpf->mpf_specification == 4) )
316 {
317 printk("Intel MultiProcessor Specification v1.%d\n", mpf->mpf_specification);
318 if(mpf->mpf_feature2&(1<<7))
319 printk(" IMCR and PIC compatibility mode.\n");
320 else
321 printk(" Virtual Wire compatibility mode.\n");
322 smp_found_config=1;
323 /*
324 * Now see if we need to read further.
325 */
326 if(mpf->mpf_feature1!=0)
327 {
328 num_processors=2;
329 printk("I/O APIC at 0xFEC00000.\n");
330 printk("Bus#0 is ");
331 }
332 switch(mpf->mpf_feature1)
333 {
334 case 1:
335 case 5:
336 printk("ISA\n");
337 break;
338 case 2:
339 printk("EISA with no IRQ8 chaining\n");
340 break;
341 case 6:
342 case 3:
343 printk("EISA\n");
344 break;
345 case 4:
346 case 7:
347 printk("MCA\n");
348 break;
349 case 0:
350 break;
351 default:
352 printk("???\nUnknown standard configuration %d\n",
353 mpf->mpf_feature1);
354 return;
355 }
356 if(mpf->mpf_feature1>4)
357 printk("Bus #1 is PCI\n");
358 /*
359 * Read the physical hardware table.
360 */
361 if(mpf->mpf_physptr)
362 smp_read_mpc((void *)mpf->mpf_physptr);
363 else
364 {
365 unsigned long cfg;
366
367 /*
368 * If no table present, determine
369 * what the CPU mapping is.
370 */
371
372 /*
373 *
374 * HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
375 *
376 */
377 /*
378 * Standard page mapping
379 * functions don't work yet.
380 * We know that page 0 is not
381 * used. Steal it for now!
382 */
383
384 cfg=pg0[0];
385 pg0[0] = (apic_addr | 7);
386 local_invalidate();
387
388 boot_cpu_id = GET_APIC_ID(*((volatile unsigned long *) APIC_ID));
389 nlong = boot_cpu_id<<24; /* Dummy 'self' for bootup */
390
391 /*
392 * Give it back
393 */
394
395 pg0[0]= cfg;
396 local_invalidate();
397
398 /*
399 *
400 * END OF HACK END OF HACK END OF HACK END OF HACK END OF HACK
401 *
402 */
403
404 /*
405 * If boot CPU != 0, other CPU
406 * is 0, else other CPU is 1.
407 */
408 if (boot_cpu_id)
409 cpu_present_map=1 | (1 << boot_cpu_id);
410 else
411 cpu_present_map=3;
412 }
413 printk("Processors: %d\n", num_processors);
414 /*
415 * Only use the first one found.
416 */
417 return;
418 }
419 }
420 bp+=4;
421 length-=16;
422 }
423 }
424
425 /*
426 * Trampoline 80x86 program as an array.
427 */
428
429 static unsigned char trampoline_data[]={
430 #include "trampoline.hex"
431 };
432
433 /*
434 * Currently trivial. Write the real->protected mode
435 * bootstrap into the page concerned. The caller
436 * has made sure its suitably aligned.
437 */
438
439 static void install_trampoline(unsigned char *mp)
/* */
440 {
441 memcpy(mp,trampoline_data,sizeof(trampoline_data));
442 }
443
444 /*
445 * We are called very early to get the low memory for the trampoline/kernel stacks
446 * This has to be done by mm/init.c to parcel us out nice low memory. We allocate
447 * the kernel stacks at 4K, 8K, 12K... currently (0-03FF is preserved for SMM and
448 * other things).
449 */
450
451 unsigned long smp_alloc_memory(unsigned long mem_base)
/* */
452 {
453 int size=(num_processors-1)*PAGE_SIZE; /* Number of stacks needed */
454 /*
455 * Our stacks have to be below the 1Mb line, and mem_base on entry
456 * is 4K aligned.
457 */
458
459 if(mem_base+size>=0x9F000)
460 panic("smp_alloc_memory: Insufficient low memory for kernel stacks.\n");
461 kstack_base=(void *)mem_base;
462 mem_base+=size;
463 kstack_end=(void *)mem_base;
464 return mem_base;
465 }
466
467 /*
468 * Hand out stacks one at a time.
469 */
470
471 static void *get_kernel_stack(void)
/* */
472 {
473 void *stack=kstack_base;
474 if(kstack_base>=kstack_end)
475 return NULL;
476 kstack_base+=PAGE_SIZE;
477 return stack;
478 }
479
480
481 /*
482 * The bootstrap kernel entry code has set these up. Save them for
483 * a given CPU
484 */
485
486 void smp_store_cpu_info(int id)
/* */
487 {
488 struct cpuinfo_x86 *c=&cpu_data[id];
489 c->hard_math=hard_math; /* Always assumed same currently */
490 c->x86=x86;
491 c->x86_model=x86_model;
492 c->x86_mask=x86_mask;
493 c->x86_capability=x86_capability;
494 c->fdiv_bug=fdiv_bug;
495 c->wp_works_ok=wp_works_ok; /* Always assumed the same currently */
496 c->hlt_works_ok=hlt_works_ok;
497 c->udelay_val=loops_per_sec;
498 strcpy(c->x86_vendor_id, x86_vendor_id);
499 }
500
501 /*
502 * Architecture specific routine called by the kernel just before init is
503 * fired off. This allows the BP to have everything in order [we hope].
504 * At the end of this all the AP's will hit the system scheduling and off
505 * we go. Each AP will load the system gdt's and jump through the kernel
506 * init into idle(). At this point the scheduler will one day take over
507 * and give them jobs to do. smp_callin is a standard routine
508 * we use to track CPU's as they power up.
509 */
510
511 void smp_commence(void)
/* */
512 {
513 /*
514 * Lets the callin's below out of their loop.
515 */
516 smp_commenced=1;
517 }
518
519 void smp_callin(void)
/* */
520 {
521 int cpuid=GET_APIC_ID(apic_read(APIC_ID));
522 unsigned long l;
523 /*
524 * Activate our APIC
525 */
526
527 SMP_PRINTK(("CALLIN %d\n",smp_processor_id()));
528 l=apic_read(APIC_SPIV);
529 l|=(1<<8); /* Enable */
530 apic_write(APIC_SPIV,l);
531 sti();
532 /*
533 * Get our bogomips.
534 */
535 calibrate_delay();
536 /*
537 * Save our processor parameters
538 */
539 smp_store_cpu_info(cpuid);
540 /*
541 * Allow the master to continue.
542 */
543 set_bit(cpuid, (unsigned long *)&cpu_callin_map[0]);
544 /*
545 * Until we are ready for SMP scheduling
546 */
547 load_ldt(0);
548 /* printk("Testing faulting...\n");
549 *(long *)0=1; OOPS... */
550 local_invalidate();
551 while(!smp_commenced);
552 if (cpu_number_map[cpuid] == -1)
553 while(1);
554 local_invalidate();
555 SMP_PRINTK(("Commenced..\n"));
556
557 load_TR(cpu_number_map[cpuid]);
558 /* while(1);*/
559 }
560
561 /*
562 * Cycle through the processors sending pentium IPI's to boot each.
563 */
564
565 void smp_boot_cpus(void)
/* */
566 {
567 int i,j;
568 int cpucount=0;
569 unsigned long cfg;
570 void *stack;
571 extern unsigned long init_user_stack[];
572
573 /*
574 * Initialize the logical to physical cpu number mapping
575 */
576
577 for (i = 0; i < NR_CPUS; i++)
578 cpu_number_map[i] = -1;
579
580 /*
581 * Setup boot CPU information
582 */
583
584 kernel_stacks[boot_cpu_id]=(void *)init_user_stack; /* Set up for boot processor first */
585
586 smp_store_cpu_info(boot_cpu_id); /* Final full version of the data */
587
588 cpu_present_map |= (1 << smp_processor_id());
589 cpu_number_map[boot_cpu_id] = 0;
590 active_kernel_processor=boot_cpu_id;
591
592 /*
593 * If we don't conform to the Intel MPS standard, get out
594 * of here now!
595 */
596
597 if (!smp_found_config)
598 return;
599
600 /*
601 * Map the local APIC into kernel space
602 */
603
604 apic_reg = vremap(apic_addr,4096);
605
606 if(apic_reg == NULL)
607 panic("Unable to map local apic.\n");
608
609 #ifdef SMP_DEBUG
610 {
611 int reg;
612
613 /*
614 * This is to verify that we're looking at
615 * a real local APIC. Check these against
616 * your board if the CPUs aren't getting
617 * started for no apparent reason.
618 */
619
620 reg = apic_read(APIC_VERSION);
621 SMP_PRINTK(("Getting VERSION: %x\n", reg));
622
623 apic_write(APIC_VERSION, 0);
624 reg = apic_read(APIC_VERSION);
625 SMP_PRINTK(("Getting VERSION: %x\n", reg));
626
627 /*
628 * The two version reads above should print the same
629 * NON-ZERO!!! numbers. If the second one is zero,
630 * there is a problem with the APIC write/read
631 * definitions.
632 *
633 * The next two are just to see if we have sane values.
634 * They're only really relevant if we're in Virtual Wire
635 * compatibility mode, but most boxes are anymore.
636 */
637
638
639 reg = apic_read(APIC_LVT0);
640 SMP_PRINTK(("Getting LVT0: %x\n", reg));
641
642 reg = apic_read(APIC_LVT1);
643 SMP_PRINTK(("Getting LVT1: %x\n", reg));
644 }
645 #endif
646
647 /*
648 * Enable the local APIC
649 */
650
651 cfg=apic_read(APIC_SPIV);
652 cfg|=(1<<8); /* Enable APIC */
653 apic_write(APIC_SPIV,cfg);
654
655 udelay(10);
656
657 /*
658 * Now scan the cpu present map and fire up the other CPUs.
659 */
660
661 SMP_PRINTK(("CPU map: %lx\n", cpu_present_map));
662
663 for(i=0;i<NR_CPUS;i++)
664 {
665 /*
666 * Don't even attempt to start the boot CPU!
667 */
668 if (i == boot_cpu_id)
669 {
670 smp_top_cpu=max(smp_top_cpu,i);
671 continue;
672 }
673
674 if (cpu_present_map & (1 << i))
675 {
676 unsigned long send_status, accept_status;
677 int timeout, num_starts;
678
679 /*
680 * We need a kernel stack for each processor.
681 */
682
683 stack=get_kernel_stack(); /* We allocated these earlier */
684 if(stack==NULL)
685 panic("No memory for processor stacks.\n");
686 kernel_stacks[i]=stack;
687 install_trampoline(stack);
688
689 printk("Booting processor %d stack %p: ",i,stack); /* So we set whats up */
690
691 /*
692 * This gunge runs the startup process for
693 * the targeted processor.
694 */
695
696 SMP_PRINTK(("Setting warm reset code and vector.\n"));
697
698 /*
699 * Install a writable page 0 entry.
700 */
701
702 cfg=pg0[0];
703
704 CMOS_WRITE(0xa, 0xf);
705 pg0[0]=7;
706 local_invalidate();
707 *((volatile unsigned short *) 0x469) = ((unsigned long)stack)>>4;
708 *((volatile unsigned short *) 0x467) = 0;
709
710 /*
711 * Protect it again
712 */
713
714 pg0[0]= cfg;
715 local_invalidate();
716
717 /*
718 * Be paranoid about clearing APIC errors.
719 */
720
721 apic_write(APIC_ESR, 0);
722 accept_status = (apic_read(APIC_ESR) & 0xEF);
723
724 /*
725 * Status is now clean
726 */
727
728 send_status = 0;
729 accept_status = 0;
730
731 /*
732 * Starting actual IPI sequence...
733 */
734
735 SMP_PRINTK(("Asserting INIT.\n"));
736
737 /*
738 * Turn INIT on
739 */
740
741 cfg=apic_read(APIC_ICR2);
742 cfg&=0x00FFFFFF;
743 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i)); /* Target chip */
744 cfg=apic_read(APIC_ICR);
745 cfg&=~0xCDFFF; /* Clear bits */
746 cfg |= (APIC_DEST_FIELD | APIC_DEST_LEVELTRIG
747 | APIC_DEST_ASSERT | APIC_DEST_DM_INIT);
748 apic_write(APIC_ICR, cfg); /* Send IPI */
749
750 udelay(200);
751 SMP_PRINTK(("Deasserting INIT.\n"));
752
753 cfg=apic_read(APIC_ICR2);
754 cfg&=0x00FFFFFF;
755 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i)); /* Target chip */
756 cfg=apic_read(APIC_ICR);
757 cfg&=~0xCDFFF; /* Clear bits */
758 cfg |= (APIC_DEST_FIELD | APIC_DEST_LEVELTRIG
759 | APIC_DEST_DM_INIT);
760 apic_write(APIC_ICR, cfg); /* Send IPI */
761
762 /*
763 * Should we send STARTUP IPIs ?
764 *
765 * Determine this based on the APIC version.
766 * If we don't have an integrated APIC, don't
767 * send the STARTUP IPIs.
768 */
769
770 if ( apic_version[i] & 0xF0 )
771 num_starts = 2;
772 else
773 num_starts = 0;
774
775 /*
776 * Run STARTUP IPI loop.
777 */
778
779 for (j = 0; !(send_status || accept_status)
780 && (j < num_starts) ; j++)
781 {
782 SMP_PRINTK(("Sending STARTUP #%d.\n",j));
783
784 apic_write(APIC_ESR, 0);
785
786 /*
787 * STARTUP IPI
788 */
789
790 cfg=apic_read(APIC_ICR2);
791 cfg&=0x00FFFFFF;
792 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i)); /* Target chip */
793 cfg=apic_read(APIC_ICR);
794 cfg&=~0xCDFFF; /* Clear bits */
795 cfg |= (APIC_DEST_FIELD
796 | APIC_DEST_DM_STARTUP
797 | (((int) stack) >> 12) ); /* Boot on the stack */
798 apic_write(APIC_ICR, cfg); /* Kick the second */
799
800 timeout = 0;
801 do {
802 udelay(10);
803 } while ( (send_status = (apic_read(APIC_ICR) & 0x1000))
804 && (timeout++ < 1000));
805 udelay(200);
806
807 accept_status = (apic_read(APIC_ESR) & 0xEF);
808 }
809
810 if (send_status) /* APIC never delivered?? */
811 printk("APIC never delivered???\n");
812 if (accept_status) /* Send accept error */
813 printk("APIC delivery error (%lx).\n", accept_status);
814
815 if( !(send_status || accept_status) )
816 {
817 for(timeout=0;timeout<50000;timeout++)
818 {
819 if(cpu_callin_map[0]&(1<<i))
820 break; /* It has booted */
821 udelay(100); /* Wait 5s total for a response */
822 }
823 if(cpu_callin_map[0]&(1<<i))
824 {
825 cpucount++;
826 /* number CPUs logically, starting from 1 (BSP is 0) */
827 cpu_number_map[i] = cpucount;
828 smp_top_cpu=max(smp_top_cpu,i);
829
830 }
831 else
832 {
833 if(*((volatile unsigned char *)8192)==0xA5)
834 printk("Stuck ??\n");
835 else
836 printk("Not responding.\n");
837 }
838 }
839
840 /* mark "stuck" area as not stuck */
841 *((volatile unsigned long *)8192) = 0;
842 }
843
844 /*
845 * Make sure we unmap all failed CPUs
846 */
847
848 if (cpu_number_map[i] == -1)
849 cpu_present_map &= ~(1 << i);
850 }
851
852 /*
853 * Cleanup possible dangling ends...
854 */
855
856 /*
857 * Install writable page 0 entry.
858 */
859
860 cfg = pg0[0];
861 pg0[0] = 3; /* writeable, present, addr 0 */
862 local_invalidate();
863
864 /*
865 * Paranoid: Set warm reset code and vector here back
866 * to default values.
867 */
868
869 CMOS_WRITE(0, 0xf);
870
871 *((volatile long *) 0x467) = 0;
872
873 /*
874 * Restore old page 0 entry.
875 */
876
877 pg0[0] = cfg;
878 local_invalidate();
879
880 /*
881 * Allow the user to impress friends.
882 */
883
884 if(cpucount==0)
885 {
886 printk("Error: only one processor found.\n");
887 cpu_present_map=(1<<smp_processor_id());
888 }
889 else
890 {
891 unsigned long bogosum=0;
892 for(i=0;i<32;i++)
893 {
894 if(cpu_present_map&(1<<i))
895 bogosum+=cpu_data[i].udelay_val;
896 }
897 printk("Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
898 cpucount+1,
899 (bogosum+2500)/500000,
900 ((bogosum+2500)/5000)%100);
901 smp_activated=1;
902 smp_num_cpus=cpucount+1;
903 }
904 }
905
906
907 /*
908 * A non wait message cannot pass data or cpu source info. This current setup
909 * is only safe because the kernel lock owner is the only person who can send a message.
910 *
911 * Wrapping this whole block in a spinlock is not the safe answer either. A processor may
912 * get stuck with irq's off waiting to send a message and thus not replying to the person
913 * spinning for a reply....
914 *
915 * In the end invalidate ought to be the NMI and a very very short function (to avoid the old
916 * IDE disk problems), and other messages sent with IRQ's enabled in a civilised fashion. That
917 * will also boost performance.
918 */
919
920 void smp_message_pass(int target, int msg, unsigned long data, int wait)
/* */
921 {
922 unsigned long cfg;
923 unsigned long target_map;
924 int p=smp_processor_id();
925 int irq=0x2d; /* IRQ 13 */
926 int ct=0;
927 static volatile int message_cpu = NO_PROC_ID;
928
929 /*
930 * During boot up send no messages
931 */
932
933 if(!smp_activated || !smp_commenced)
934 return;
935
936
937 /*
938 * Skip the reschedule if we are waiting to clear a
939 * message at this time. The reschedule cannot wait
940 * but is not critical.
941 */
942
943 if(msg==MSG_RESCHEDULE) /* Reschedules we do via trap 0x30 */
944 {
945 irq=0x30;
946 if(smp_cpu_in_msg[p])
947 return;
948 }
949
950 /*
951 * Sanity check we don't re-enter this across CPU's. Only the kernel
952 * lock holder may send messages. For a STOP_CPU we are bringing the
953 * entire box to the fastest halt we can..
954 */
955
956 if(message_cpu!=NO_PROC_ID && msg!=MSG_STOP_CPU)
957 {
958 panic("CPU #%d: Message pass %d but pass in progress by %d of %d\n",
959 smp_processor_id(),msg,message_cpu, smp_msg_id);
960 }
961 message_cpu=smp_processor_id();
962
963
964 /*
965 * We are busy
966 */
967
968 smp_cpu_in_msg[p]++;
969
970 /*
971 * Reschedule is currently special
972 */
973
974 if(msg!=MSG_RESCHEDULE)
975 {
976 smp_src_cpu=p;
977 smp_msg_id=msg;
978 smp_msg_data=data;
979 }
980
981 /* printk("SMP message pass #%d to %d of %d\n",
982 p, msg, target);*/
983
984 /*
985 * Wait for the APIC to become ready - this should never occur. Its
986 * a debugging check really.
987 */
988
989 while(ct<1000)
990 {
991 cfg=apic_read(APIC_ICR);
992 if(!(cfg&(1<<12)))
993 break;
994 ct++;
995 udelay(10);
996 }
997
998 /*
999 * Just pray... there is nothing more we can do
1000 */
1001
1002 if(ct==1000)
1003 printk("CPU #%d: previous IPI still not cleared after 10mS", smp_processor_id());
1004
1005 /*
1006 * Program the APIC to deliver the IPI
1007 */
1008
1009 cfg=apic_read(APIC_ICR2);
1010 cfg&=0x00FFFFFF;
1011 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(target)); /* Target chip */
1012 cfg=apic_read(APIC_ICR);
1013 cfg&=~0xFDFFF; /* Clear bits */
1014 cfg|=APIC_DEST_FIELD|APIC_DEST_DM_FIXED|irq; /* Send an IRQ 13 */
1015
1016 /*
1017 * Set the target requirement
1018 */
1019
1020 if(target==MSG_ALL_BUT_SELF)
1021 {
1022 cfg|=APIC_DEST_ALLBUT;
1023 target_map=cpu_present_map;
1024 cpu_callin_map[0]=(1<<smp_src_cpu);
1025 }
1026 else if(target==MSG_ALL)
1027 {
1028 cfg|=APIC_DEST_ALLINC;
1029 target_map=cpu_present_map;
1030 cpu_callin_map[0]=0;
1031 }
1032 else
1033 {
1034 target_map=(1<<target);
1035 cpu_callin_map[0]=0;
1036 }
1037
1038 /*
1039 * Send the IPI. The write to APIC_ICR fires this off.
1040 */
1041
1042 apic_write(APIC_ICR, cfg);
1043
1044 /*
1045 * Spin waiting for completion
1046 */
1047
1048 switch(wait)
1049 {
1050 case 1:
1051 while(cpu_callin_map[0]!=target_map); /* Spin on the pass */
1052 break;
1053 case 2:
1054 while(smp_invalidate_needed); /* Wait for invalidate map to clear */
1055 break;
1056 }
1057
1058 /*
1059 * Record our completion
1060 */
1061
1062 smp_cpu_in_msg[p]--;
1063 message_cpu=NO_PROC_ID;
1064 }
1065
1066 /*
1067 * This is fraught with deadlocks. Linus does an invalidate at a whim
1068 * even with IRQ's off. We have to avoid a pair of crossing invalidates
1069 * or we are doomed. See the notes about smp_message_pass.
1070 */
1071
1072 void smp_invalidate(void)
/* */
1073 {
1074 unsigned long flags;
1075 if(smp_activated && smp_processor_id()!=active_kernel_processor)
1076 panic("CPU #%d:Attempted invalidate IPI when not AKP(=%d)\n",smp_processor_id(),active_kernel_processor);
1077 /* printk("SMI-");*/
1078
1079 /*
1080 * The assignment is safe because its volatile so the compiler cannot reorder it,
1081 * because the i586 has strict memory ordering and because only the kernel lock holder
1082 * may issue an invalidate. If you break any one of those three change this to an atomic
1083 * bus locked or.
1084 */
1085
1086 smp_invalidate_needed=cpu_present_map&~(1<<smp_processor_id());
1087
1088 /*
1089 * Processors spinning on the lock will see this IRQ late. The smp_invalidate_needed map will
1090 * ensure they dont do a spurious invalidate or miss one.
1091 */
1092
1093 save_flags(flags);
1094 cli();
1095 smp_message_pass(MSG_ALL_BUT_SELF, MSG_INVALIDATE_TLB, 0L, 2);
1096
1097 /*
1098 * Flush the local TLB
1099 */
1100
1101 local_invalidate();
1102
1103 restore_flags(flags);
1104
1105 /*
1106 * Completed.
1107 */
1108
1109 /* printk("SMID\n");*/
1110 }
1111
1112 /*
1113 * Reschedule call back
1114 */
1115
1116 void smp_reschedule_irq(int cpl, struct pt_regs *regs)
/* */
1117 {
1118 #ifdef DEBUGGING_SMP_RESCHED
1119 static int ct=0;
1120 if(ct==0)
1121 {
1122 printk("Beginning scheduling on CPU#%d\n",smp_processor_id());
1123 ct=1;
1124 }
1125 #endif
1126 if(smp_processor_id()!=active_kernel_processor)
1127 panic("SMP Reschedule on CPU #%d, but #%d is active.\n",
1128 smp_processor_id(), active_kernel_processor);
1129
1130 need_resched=1;
1131
1132 /*
1133 * Clear the IPI
1134 */
1135 apic_read(APIC_SPIV); /* Dummy read */
1136 apic_write(APIC_EOI, 0); /* Docs say use 0 for future compatibility */
1137 }
1138
1139 /*
1140 * Message call back.
1141 */
1142
1143 void smp_message_irq(int cpl, void *dev_id, struct pt_regs *regs)
/* ![[last]](../icons/n_last.png)
*/
1144 {
1145 int i=smp_processor_id();
1146 /* static int n=0;
1147 if(n++<NR_CPUS)
1148 printk("IPI %d->%d(%d,%ld)\n",smp_src_cpu,i,smp_msg_id,smp_msg_data);*/
1149 switch(smp_msg_id)
1150 {
1151 case 0: /* IRQ 13 testing - boring */
1152 return;
1153
1154 /*
1155 * A TLB flush is needed.
1156 */
1157
1158 case MSG_INVALIDATE_TLB:
1159 if(clear_bit(i,(unsigned long *)&smp_invalidate_needed))
1160 local_invalidate();
1161 set_bit(i, (unsigned long *)&cpu_callin_map[0]);
1162 /* cpu_callin_map[0]|=1<<smp_processor_id();*/
1163 break;
1164
1165 /*
1166 * Halt other CPU's for a panic or reboot
1167 */
1168 case MSG_STOP_CPU:
1169 while(1)
1170 {
1171 if(cpu_data[smp_processor_id()].hlt_works_ok)
1172 __asm__("hlt");
1173 }
1174 default:
1175 printk("CPU #%d sent invalid cross CPU message to CPU #%d: %X(%lX).\n",
1176 smp_src_cpu,smp_processor_id(),smp_msg_id,smp_msg_data);
1177 break;
1178 }
1179 /*
1180 * Clear the IPI, so we can receive future IPI's
1181 */
1182
1183 apic_read(APIC_SPIV); /* Dummy read */
1184 apic_write(APIC_EOI, 0); /* Docs say use 0 for future compatibility */
1185 }
![[bottom]](../icons/n_bottom.png){kind=icon}
*/