This source file includes following definitions.
- mpf_checksum
- mpc_family
- smp_read_mpc
- smp_scan_config
- install_trampoline
- smp_alloc_memory
- get_kernel_stack
- smp_store_cpu_info
- smp_commence
- smp_callin
- smp_boot_cpus
- smp_message_pass
- smp_invalidate
- smp_reschedule_irq
- smp_message_irq
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 #include <linux/kernel.h>
19 #include <linux/string.h>
20 #include <linux/config.h>
21 #include <linux/timer.h>
22 #include <linux/sched.h>
23 #include <linux/mm.h>
24 #include <linux/kernel_stat.h>
25 #include <linux/delay.h>
26 #include <asm/i82489.h>
27 #include <linux/smp.h>
28 #include <asm/bitops.h>
29 #include <asm/smp.h>
30
31 extern void *vremap(unsigned long offset, unsigned long size);
32
33 static int smp_found_config=0;
34
35 unsigned long cpu_present_map = 0;
36 int smp_num_cpus;
37 int smp_threads_ready=0;
38 volatile unsigned long cpu_callin_map[NR_CPUS] = {0,};
39 volatile unsigned long smp_invalidate_needed;
40 struct cpuinfo_x86 cpu_data[NR_CPUS];
41 static unsigned int num_processors = 0;
42 static unsigned long io_apic_addr = 0;
43 unsigned char boot_cpu_id = 0;
44 static unsigned char *kstack_base,*kstack_end;
45 static int smp_activated = 0;
46 static volatile int smp_commenced=0;
47 static unsigned char nlong=0;
48 unsigned char *apic_reg=((unsigned char *)&nlong)-0x20;
49 unsigned long apic_retval;
50 unsigned char *kernel_stacks[NR_CPUS];
51
52 static volatile unsigned char smp_cpu_in_msg[NR_CPUS];
53 static volatile unsigned long smp_msg_data;
54 static volatile int smp_src_cpu;
55 static volatile int smp_msg_id;
56
57 volatile unsigned long kernel_flag=0;
58 volatile unsigned char active_kernel_processor = NO_PROC_ID;
59 volatile unsigned long kernel_counter=0;
60 volatile unsigned long syscall_count=0;
61 volatile unsigned long smp_spins=0;
62
63 volatile unsigned long ipi_count;
64
65
66
67
68
69 static int mpf_checksum(unsigned char *mp, int len)
70 {
71 int sum=0;
72 while(len--)
73 sum+=*mp++;
74 return sum&0xFF;
75 }
76
77
78
79
80
81 static char *mpc_family(int family,int model)
82 {
83 static char n[32];
84 static char *model_defs[]=
85 {
86 "80486DX","80486DX",
87 "80486SX","80486DX/2 or 80487",
88 "80486SL","Intel5X2(tm)",
89 "Unknown","Unknown",
90 "80486DX/4"
91 };
92 if(family==0x5)
93 return("Pentium(tm)");
94 if(family==0x0F && model==0x0F)
95 return("Special controller");
96 if(family==0x04 && model<9)
97 return model_defs[model];
98 sprintf(n,"Unknown CPU [%d:%d]",family, model);
99 return n;
100 }
101
102
103
104
105
106 static int smp_read_mpc(struct mp_config_table *mpc)
107 {
108 char str[16];
109 int count=sizeof(*mpc);
110 int apics=0;
111 unsigned char *mpt=((unsigned char *)mpc)+count;
112
113 if(memcmp(mpc->mpc_signature,MPC_SIGNATURE,4))
114 {
115 printk("Bad signature [%c%c%c%c].\n",
116 mpc->mpc_signature[0],
117 mpc->mpc_signature[1],
118 mpc->mpc_signature[2],
119 mpc->mpc_signature[3]);
120 return 1;
121 }
122 if(mpf_checksum((unsigned char *)mpc,mpc->mpc_length))
123 {
124 printk("Checksum error.\n");
125 return 1;
126 }
127 if(mpc->mpc_spec!=0x01)
128 {
129 printk("Unsupported version (%d)\n",mpc->mpc_spec);
130 return 1;
131 }
132 memcpy(str,mpc->mpc_oem,8);
133 str[8]=0;
134 printk("OEM ID: %s ",str);
135 memcpy(str,mpc->mpc_productid,12);
136 str[12]=0;
137 printk("Product ID: %s ",str);
138 printk("APIC at: 0x%lX\n",mpc->mpc_lapic);
139
140
141
142
143
144 while(count<mpc->mpc_length)
145 {
146 switch(*mpt)
147 {
148 case MP_PROCESSOR:
149 {
150 struct mpc_config_processor *m=
151 (struct mpc_config_processor *)mpt;
152 if(m->mpc_cpuflag&CPU_ENABLED)
153 {
154 printk("Processor #%d %s APIC version %d\n",
155 m->mpc_apicid,
156 mpc_family((m->mpc_cpufeature&
157 CPU_FAMILY_MASK)>>8,
158 (m->mpc_cpufeature&
159 CPU_MODEL_MASK)>>4),
160 m->mpc_apicver);
161 if(m->mpc_featureflag&(1<<0))
162 printk(" Floating point unit present.\n");
163 if(m->mpc_featureflag&(1<<7))
164 printk(" Machine Exception supported.\n");
165 if(m->mpc_featureflag&(1<<8))
166 printk(" 64 bit compare & exchange supported.\n");
167 if(m->mpc_featureflag&(1<<9))
168 printk(" Internal APIC present.\n");
169 if(m->mpc_cpuflag&CPU_BOOTPROCESSOR)
170 {
171 printk(" Bootup CPU\n");
172 boot_cpu_id=m->mpc_apicid;
173 nlong = boot_cpu_id<<24;
174 }
175 else
176 num_processors++;
177
178 if(m->mpc_apicid>NR_CPUS)
179 printk("Processor #%d unused. (Max %d processors).\n",m->mpc_apicid, NR_CPUS);
180 else
181 cpu_present_map|=(1<<m->mpc_apicid);
182 }
183 mpt+=sizeof(*m);
184 count+=sizeof(*m);
185 break;
186 }
187 case MP_BUS:
188 {
189 struct mpc_config_bus *m=
190 (struct mpc_config_bus *)mpt;
191 memcpy(str,m->mpc_bustype,6);
192 str[6]=0;
193 printk("Bus #%d is %s\n",
194 m->mpc_busid,
195 str);
196 mpt+=sizeof(*m);
197 count+=sizeof(*m);
198 break;
199 }
200 case MP_IOAPIC:
201 {
202 struct mpc_config_ioapic *m=
203 (struct mpc_config_ioapic *)mpt;
204 if(m->mpc_flags&MPC_APIC_USABLE)
205 {
206 apics++;
207 printk("I/O APIC #%d Version %d at 0x%lX.\n",
208 m->mpc_apicid,m->mpc_apicver,
209 m->mpc_apicaddr);
210 io_apic_addr = m->mpc_apicaddr;
211 }
212 mpt+=sizeof(*m);
213 count+=sizeof(*m);
214 break;
215 }
216 case MP_INTSRC:
217 {
218 struct mpc_config_intsrc *m=
219 (struct mpc_config_intsrc *)mpt;
220
221 mpt+=sizeof(*m);
222 count+=sizeof(*m);
223 break;
224 }
225 case MP_LINTSRC:
226 {
227 struct mpc_config_intlocal *m=
228 (struct mpc_config_intlocal *)mpt;
229 mpt+=sizeof(*m);
230 count+=sizeof(*m);
231 break;
232 }
233 }
234 }
235 if(apics>1)
236 printk("Warning: Multiple APIC's not supported.\n");
237 return num_processors;
238 }
239
240
241
242
243
244 void smp_scan_config(unsigned long base, unsigned long length)
245 {
246 unsigned long *bp=(unsigned long *)base;
247 struct intel_mp_floating *mpf;
248 num_processors = 1;
249
250
251
252 if(sizeof(*mpf)!=16)
253 printk("Error: MPF size\n");
254
255 while(length>0)
256 {
257 if(*bp==SMP_MAGIC_IDENT)
258 {
259 mpf=(struct intel_mp_floating *)bp;
260 if(mpf->mpf_length==1 &&
261 !mpf_checksum((unsigned char *)bp,16) &&
262 mpf->mpf_specification==1)
263 {
264 printk("Intel multiprocessing (MPv1.1) available.\n");
265 if(mpf->mpf_feature2&(1<<7))
266 printk(" IMCR and PIC mode supported.\n");
267 smp_found_config=1;
268
269
270
271 if(mpf->mpf_feature1!=0)
272 {
273 num_processors=2;
274 printk("I/O APIC at 0xFEC00000.\n");
275 printk("Bus#0 is ");
276 }
277 switch(mpf->mpf_feature1)
278 {
279 case 1:
280 printk("ISA");
281 break;
282 case 2:
283 printk("EISA with no IRQ8 chaining");
284 break;
285 case 3:
286 printk("EISA");
287 break;
288 case 4:
289 printk("MCA");
290 break;
291 case 5:
292 printk("ISA\nBus#1 is PCI");
293 break;
294 case 6:
295 printk("EISA\nBus #1 is PCI");
296 break;
297 case 7:
298 printk("MCA\nBus #1 is PCI");
299 break;
300 case 0:
301 break;
302 default:
303 printk("???\nUnknown standard configuration %d\n",
304 mpf->mpf_feature1);
305 return;
306 }
307 if(mpf->mpf_physptr)
308 smp_read_mpc((void *)mpf->mpf_physptr);
309 printk("Processors: %d\n", num_processors);
310 }
311 }
312 bp+=4;
313 length-=16;
314 }
315 }
316
317
318
319
320
321 static unsigned char trampoline_data[]={
322 #include "trampoline.hex"
323 };
324
325
326
327
328
329
330
331 static void install_trampoline(unsigned char *mp)
332 {
333 memcpy(mp,trampoline_data,sizeof(trampoline_data));
334 }
335
336
337
338
339
340
341
342
343 unsigned long smp_alloc_memory(unsigned long mem_base)
344 {
345 int size=(num_processors-1)*PAGE_SIZE;
346
347
348
349
350
351 if(mem_base+size>=0x9F000)
352 panic("smp_alloc_memory: Insufficient low memory for kernel stacks.\n");
353 kstack_base=(void *)mem_base;
354 mem_base+=size;
355 kstack_end=(void *)mem_base;
356 return mem_base;
357 }
358
359
360
361
362
363 static void *get_kernel_stack(void)
364 {
365 void *stack=kstack_base;
366 if(kstack_base>=kstack_end)
367 return NULL;
368 kstack_base+=PAGE_SIZE;
369 return stack;
370 }
371
372
373
374
375
376
377
378 void smp_store_cpu_info(int id)
379 {
380 struct cpuinfo_x86 *c=&cpu_data[id];
381 c->hard_math=hard_math;
382 c->x86=x86;
383 c->x86_model=x86_model;
384 c->x86_mask=x86_mask;
385 c->x86_capability=x86_capability;
386 c->fdiv_bug=fdiv_bug;
387 c->wp_works_ok=wp_works_ok;
388 c->hlt_works_ok=hlt_works_ok;
389 c->udelay_val=loops_per_sec;
390 strcpy(c->x86_vendor_id, x86_vendor_id);
391 }
392
393
394
395
396
397
398
399
400
401
402
403 void smp_commence(void)
404 {
405
406
407
408 smp_commenced=1;
409 }
410
411 void smp_callin(void)
412 {
413 int cpuid=GET_APIC_ID(apic_read(APIC_ID));
414 unsigned long l;
415
416
417
418
419
420 l=apic_read(APIC_SPIV);
421 l|=(1<<8);
422 apic_write(APIC_SPIV,l);
423 sti();
424
425
426
427 calibrate_delay();
428
429
430
431 smp_store_cpu_info(cpuid);
432
433
434
435 set_bit(cpuid, &cpu_callin_map[0]);
436
437
438
439 load_ldt(0);
440
441
442 local_invalidate();
443 while(!smp_commenced);
444 local_invalidate();
445
446
447
448 load_TR(cpuid);
449
450 }
451
452
453
454
455
456 void smp_boot_cpus(void)
457 {
458 int i=0;
459 int cpucount=0;
460 void *stack;
461 extern unsigned long init_user_stack[];
462
463
464
465
466
467 apic_reg = vremap(0xFEE00000,4096);
468
469
470 if(apic_reg == NULL)
471 panic("Unable to map local apic.\n");
472
473
474
475
476
477
478 kernel_stacks[boot_cpu_id]=(void *)init_user_stack;
479
480 smp_store_cpu_info(boot_cpu_id);
481
482 active_kernel_processor=boot_cpu_id;
483
484 for(i=0;i<NR_CPUS;i++)
485 {
486 if((cpu_present_map&(1<<i)) && i!=boot_cpu_id)
487 {
488 unsigned long cfg;
489 int timeout;
490
491
492
493
494
495 stack=get_kernel_stack();
496 if(stack==NULL)
497 panic("No memory for processor stacks.\n");
498 kernel_stacks[i]=stack;
499 install_trampoline(stack);
500
501 printk("Booting processor %d stack %p: ",i,stack);
502
503
504
505
506
507 cfg=apic_read(APIC_SPIV);
508 cfg|=(1<<8);
509 apic_write(APIC_SPIV,cfg);
510
511
512
513
514
515
516
517 cfg=apic_read(APIC_ICR2);
518 cfg&=0x00FFFFFF;
519 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(i));
520 cfg=apic_read(APIC_ICR);
521 cfg&=~0xFDFFF ;
522 cfg|=APIC_DEST_FIELD|APIC_DEST_DM_STARTUP|(((unsigned long)stack)>>12);
523 apic_write(APIC_ICR, cfg);
524 udelay(10);
525 cfg=apic_read(APIC_ESR);
526 if(cfg&4)
527 printk("Processor refused startup request.\n");
528 else
529 {
530 for(timeout=0;timeout<50000;timeout++)
531 {
532 if(cpu_callin_map[0]&(1<<i))
533 break;
534 udelay(100);
535 }
536 if(cpu_callin_map[0]&(1<<i))
537 cpucount++;
538 else
539 {
540
541
542
543
544 if(*((unsigned char *)8192)==0xA5)
545 printk("Stuck ??\n");
546 else
547 printk("Not responding.\n");
548 cpu_present_map&=~(1<<i);
549 }
550 }
551 }
552 }
553
554
555
556 if(cpucount==0)
557 printk("Error: only one processor found.\n");
558 else
559 {
560 printk("Total of %d processors activated.\n", cpucount+1);
561 smp_activated=1;
562 smp_num_cpus=cpucount+1;
563 }
564 }
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580 void smp_message_pass(int target, int msg, unsigned long data, int wait)
581 {
582 unsigned long cfg;
583 unsigned long target_map;
584 int p=smp_processor_id();
585 int irq=0x2d;
586 int ct=0;
587 static volatile int message_cpu = NO_PROC_ID;
588
589
590
591
592
593 if(!smp_activated)
594 return;
595
596
597
598
599
600
601
602
603 if(msg==MSG_RESCHEDULE)
604 {
605 irq=0x30;
606 if(smp_cpu_in_msg[p])
607 return;
608 }
609
610
611
612
613
614
615
616 if(message_cpu!=NO_PROC_ID && msg!=MSG_STOP_CPU)
617 {
618 panic("CPU #%d: Message pass %d but pass in progress by %d of %d\n",
619 smp_processor_id(),msg,message_cpu, smp_msg_id);
620 }
621 message_cpu=smp_processor_id();
622
623
624
625
626
627
628 smp_cpu_in_msg[p]++;
629
630
631
632
633
634 if(msg!=MSG_RESCHEDULE)
635 {
636 smp_src_cpu=p;
637 smp_msg_id=msg;
638 smp_msg_data=data;
639 }
640
641
642
643
644
645
646
647
648
649 while(ct<1000)
650 {
651 cfg=apic_read(APIC_ICR);
652 if(!(cfg&(1<<12)))
653 break;
654 ct++;
655 udelay(10);
656 }
657
658
659
660
661
662 if(ct==1000)
663 printk("CPU #%d: previous IPI still not cleared after 10mS", smp_processor_id());
664
665
666
667
668
669 cfg=apic_read(APIC_ICR2);
670 cfg&=0x00FFFFFF;
671 apic_write(APIC_ICR2, cfg|SET_APIC_DEST_FIELD(target));
672 cfg=apic_read(APIC_ICR);
673 cfg&=~0xFDFFF;
674 cfg|=APIC_DEST_FIELD|APIC_DEST_DM_FIXED|irq;
675
676
677
678
679
680 if(target==MSG_ALL_BUT_SELF)
681 {
682 cfg|=APIC_DEST_ALLBUT;
683 target_map=cpu_present_map;
684 cpu_callin_map[0]=(1<<smp_src_cpu);
685 }
686 else if(target==MSG_ALL)
687 {
688 cfg|=APIC_DEST_ALLINC;
689 target_map=cpu_present_map;
690 cpu_callin_map[0]=0;
691 }
692 else
693 {
694 target_map=(1<<target);
695 cpu_callin_map[0]=0;
696 }
697
698
699
700
701
702 apic_write(APIC_ICR, cfg);
703
704
705
706
707
708 switch(wait)
709 {
710 case 1:
711 while(cpu_callin_map[0]!=target_map);
712 break;
713 case 2:
714 while(smp_invalidate_needed);
715 break;
716 }
717
718
719
720
721
722 smp_cpu_in_msg[p]--;
723 message_cpu=NO_PROC_ID;
724 }
725
726
727
728
729
730
731
732 void smp_invalidate(void)
733 {
734 unsigned long flags;
735 if(smp_activated && smp_processor_id()!=active_kernel_processor)
736 panic("CPU #%d:Attempted invalidate IPI when not AKP(=%d)\n",smp_processor_id(),active_kernel_processor);
737
738
739
740
741
742
743
744
745
746 smp_invalidate_needed=cpu_present_map&~(1<<smp_processor_id());
747
748
749
750
751
752
753 save_flags(flags);
754 cli();
755 smp_message_pass(MSG_ALL_BUT_SELF, MSG_INVALIDATE_TLB, 0L, 2);
756
757
758
759
760
761 local_invalidate();
762
763 restore_flags(flags);
764
765
766
767
768
769
770 }
771
772
773
774
775
776 void smp_reschedule_irq(int cpl, struct pt_regs *regs)
777 {
778 static int ct=0;
779 if(ct==0)
780 {
781 printk("Beginning scheduling on CPU#%d\n",smp_processor_id());
782 ct=1;
783 }
784
785 if(smp_processor_id()!=active_kernel_processor)
786 panic("SMP Reschedule on CPU #%d, but #%d is active.\n",
787 smp_processor_id(), active_kernel_processor);
788
789
790
791
792 if (user_mode(regs))
793 {
794 current->utime++;
795 if (current->pid)
796 {
797 if (current->priority < 15)
798 kstat.cpu_nice++;
799 else
800 kstat.cpu_user++;
801 }
802
803 if (current->it_virt_value && !(--current->it_virt_value)) {
804 current->it_virt_value = current->it_virt_incr;
805 send_sig(SIGVTALRM,current,1);
806 }
807 } else {
808 current->stime++;
809 if(current->pid)
810 kstat.cpu_system++;
811 #ifdef CONFIG_PROFILE
812 if (prof_buffer && current->pid) {
813 extern int _stext;
814 unsigned long eip = regs->eip - (unsigned long) &_stext;
815 eip >>= CONFIG_PROFILE_SHIFT;
816 if (eip < prof_len)
817 prof_buffer[eip]++;
818 }
819 #endif
820 }
821
822
823
824 if ((current->rlim[RLIMIT_CPU].rlim_max != RLIM_INFINITY) &&
825 (((current->stime + current->utime) / HZ) >= current->rlim[RLIMIT_CPU].rlim_max))
826 send_sig(SIGKILL, current, 1);
827 if ((current->rlim[RLIMIT_CPU].rlim_cur != RLIM_INFINITY) &&
828 (((current->stime + current->utime) % HZ) == 0)) {
829 unsigned long psecs = (current->stime + current->utime) / HZ;
830
831 if (psecs == current->rlim[RLIMIT_CPU].rlim_cur)
832 send_sig(SIGXCPU, current, 1);
833
834 else if ((psecs > current->rlim[RLIMIT_CPU].rlim_cur) &&
835 ((psecs - current->rlim[RLIMIT_CPU].rlim_cur) % 5) == 0)
836 send_sig(SIGXCPU, current, 1);
837 }
838
839
840 if (current->it_prof_value && !(--current->it_prof_value)) {
841 current->it_prof_value = current->it_prof_incr;
842 send_sig(SIGPROF,current,1);
843 }
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858 if ( 0 > --current->counter || current->pid == 0)
859 {
860 current->counter = 0;
861 need_resched=1;
862 }
863
864
865
866
867
868 apic_read(APIC_SPIV);
869 apic_write(APIC_EOI, 0);
870 }
871
872
873
874
875
876 void smp_message_irq(int cpl, struct pt_regs *regs)
877 {
878 static int n=0;
879 int i=smp_processor_id();
880
881
882 switch(smp_msg_id)
883 {
884 case 0:
885 return;
886
887
888
889
890
891 case MSG_INVALIDATE_TLB:
892 if(clear_bit(i,&smp_invalidate_needed))
893 local_invalidate();
894 set_bit(i, &cpu_callin_map[0]);
895 cpu_callin_map[0]|=1<<smp_processor_id();
896 break;
897
898
899
900
901 case MSG_STOP_CPU:
902 while(1)
903 {
904 if(cpu_data[smp_processor_id()].hlt_works_ok)
905 __asm__("hlt");
906 }
907 default:
908 printk("CPU #%d sent invalid cross CPU message to CPU #%d: %X(%lX).\n",
909 smp_src_cpu,smp_processor_id(),smp_msg_id,smp_msg_data);
910 break;
911 }
912
913
914
915
916 apic_read(APIC_SPIV);
917 apic_write(APIC_EOI, 0);
918 }