This source file includes following definitions.
- rw_swap_page
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- __get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/kernel_stat.h>
17 #include <linux/errno.h>
18 #include <linux/string.h>
19 #include <linux/stat.h>
20
21 #include <asm/system.h>
22 #include <asm/bitops.h>
23
24 #define MAX_SWAPFILES 8
25
26 #define SWP_USED 1
27 #define SWP_WRITEOK 3
28
29 #define SWP_TYPE(entry) (((entry) & 0xfe) >> 1)
30 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
31 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
32
33 static int nr_swapfiles = 0;
34 static struct wait_queue * lock_queue = NULL;
35
36 static struct swap_info_struct {
37 unsigned long flags;
38 struct inode * swap_file;
39 unsigned int swap_device;
40 unsigned char * swap_map;
41 unsigned char * swap_lockmap;
42 int pages;
43 int lowest_bit;
44 int highest_bit;
45 unsigned long max;
46 } swap_info[MAX_SWAPFILES];
47
48 extern unsigned long free_page_list;
49 extern int shm_swap (int);
50
51
52
53
54
55 #define NR_LAST_FREE_PAGES 32
56 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
57
58 void rw_swap_page(int rw, unsigned long entry, char * buf)
59 {
60 unsigned long type, offset;
61 struct swap_info_struct * p;
62
63 type = SWP_TYPE(entry);
64 if (type >= nr_swapfiles) {
65 printk("Internal error: bad swap-device\n");
66 return;
67 }
68 p = &swap_info[type];
69 offset = SWP_OFFSET(entry);
70 if (offset >= p->max) {
71 printk("rw_swap_page: weirdness\n");
72 return;
73 }
74 if (!(p->flags & SWP_USED)) {
75 printk("Trying to swap to unused swap-device\n");
76 return;
77 }
78 while (set_bit(offset,p->swap_lockmap))
79 sleep_on(&lock_queue);
80 if (rw == READ)
81 kstat.pswpin++;
82 else
83 kstat.pswpout++;
84 if (p->swap_device) {
85 ll_rw_page(rw,p->swap_device,offset,buf);
86 } else if (p->swap_file) {
87 unsigned int zones[8];
88 unsigned int block;
89 int i, j;
90
91 block = offset << (12 - p->swap_file->i_sb->s_blocksize_bits);
92
93 for (i=0, j=0; j< PAGE_SIZE ; i++, j +=p->swap_file->i_sb->s_blocksize)
94 if (!(zones[i] = bmap(p->swap_file,block++))) {
95 printk("rw_swap_page: bad swap file\n");
96 return;
97 }
98 ll_rw_swap_file(rw,p->swap_file->i_dev, zones, i,buf);
99 } else
100 printk("re_swap_page: no swap file or device\n");
101 if (offset && !clear_bit(offset,p->swap_lockmap))
102 printk("rw_swap_page: lock already cleared\n");
103 wake_up(&lock_queue);
104 }
105
106 unsigned int get_swap_page(void)
107 {
108 struct swap_info_struct * p;
109 unsigned int offset, type;
110
111 p = swap_info;
112 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
113 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
114 continue;
115 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
116 if (p->swap_map[offset])
117 continue;
118 p->swap_map[offset] = 1;
119 nr_swap_pages--;
120 if (offset == p->highest_bit)
121 p->highest_bit--;
122 p->lowest_bit = offset;
123 return SWP_ENTRY(type,offset);
124 }
125 }
126 return 0;
127 }
128
129 unsigned long swap_duplicate(unsigned long entry)
130 {
131 struct swap_info_struct * p;
132 unsigned long offset, type;
133
134 if (!entry)
135 return 0;
136 offset = SWP_OFFSET(entry);
137 type = SWP_TYPE(entry);
138 if (type == SHM_SWP_TYPE)
139 return entry;
140 if (type >= nr_swapfiles) {
141 printk("Trying to duplicate nonexistent swap-page\n");
142 return 0;
143 }
144 p = type + swap_info;
145 if (offset >= p->max) {
146 printk("swap_free: weirdness\n");
147 return 0;
148 }
149 if (!p->swap_map[offset]) {
150 printk("swap_duplicate: trying to duplicate unused page\n");
151 return 0;
152 }
153 p->swap_map[offset]++;
154 return entry;
155 }
156
157 void swap_free(unsigned long entry)
158 {
159 struct swap_info_struct * p;
160 unsigned long offset, type;
161
162 if (!entry)
163 return;
164 type = SWP_TYPE(entry);
165 if (type == SHM_SWP_TYPE)
166 return;
167 if (type >= nr_swapfiles) {
168 printk("Trying to free nonexistent swap-page\n");
169 return;
170 }
171 p = & swap_info[type];
172 offset = SWP_OFFSET(entry);
173 if (offset >= p->max) {
174 printk("swap_free: weirdness\n");
175 return;
176 }
177 if (!(p->flags & SWP_USED)) {
178 printk("Trying to free swap from unused swap-device\n");
179 return;
180 }
181 while (set_bit(offset,p->swap_lockmap))
182 sleep_on(&lock_queue);
183 if (offset < p->lowest_bit)
184 p->lowest_bit = offset;
185 if (offset > p->highest_bit)
186 p->highest_bit = offset;
187 if (!p->swap_map[offset])
188 printk("swap_free: swap-space map bad (entry %08lx)\n",entry);
189 else
190 if (!--p->swap_map[offset])
191 nr_swap_pages++;
192 if (!clear_bit(offset,p->swap_lockmap))
193 printk("swap_free: lock already cleared\n");
194 wake_up(&lock_queue);
195 }
196
197 void swap_in(unsigned long *table_ptr)
198 {
199 unsigned long entry;
200 unsigned long page;
201
202 entry = *table_ptr;
203 if (PAGE_PRESENT & entry) {
204 printk("trying to swap in present page\n");
205 return;
206 }
207 if (!entry) {
208 printk("No swap page in swap_in\n");
209 return;
210 }
211 if (SWP_TYPE(entry) == SHM_SWP_TYPE) {
212 shm_no_page ((unsigned long *) table_ptr);
213 return;
214 }
215 if (!(page = get_free_page(GFP_KERNEL))) {
216 oom(current);
217 page = BAD_PAGE;
218 } else
219 read_swap_page(entry, (char *) page);
220 if (*table_ptr != entry) {
221 free_page(page);
222 return;
223 }
224 *table_ptr = page | (PAGE_DIRTY | PAGE_PRIVATE);
225 swap_free(entry);
226 }
227
228 static inline int try_to_swap_out(unsigned long * table_ptr)
229 {
230 int i;
231 unsigned long page;
232 unsigned long entry;
233
234 page = *table_ptr;
235 if (!(PAGE_PRESENT & page))
236 return 0;
237 if (page >= high_memory)
238 return 0;
239 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
240 return 0;
241 if (PAGE_ACCESSED & page) {
242 *table_ptr &= ~PAGE_ACCESSED;
243 return 0;
244 }
245 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
246 if (last_free_pages[i] == (page & PAGE_MASK))
247 return 0;
248 if (PAGE_DIRTY & page) {
249 page &= PAGE_MASK;
250 if (mem_map[MAP_NR(page)] != 1)
251 return 0;
252 if (!(entry = get_swap_page()))
253 return 0;
254 *table_ptr = entry;
255 invalidate();
256 write_swap_page(entry, (char *) page);
257 free_page(page);
258 return 1;
259 }
260 page &= PAGE_MASK;
261 *table_ptr = 0;
262 invalidate();
263 free_page(page);
264 return 1 + mem_map[MAP_NR(page)];
265 }
266
267
268
269
270
271
272 asmlinkage int sys_idle(void)
273 {
274 need_resched = 1;
275 return 0;
276 }
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291 #ifdef NEW_SWAP
292
293
294
295
296 #define SWAP_MIN 4
297 #define SWAP_MAX 32
298
299
300
301
302
303 #define SWAP_RATIO 128
304
305 static int swap_out(unsigned int priority)
306 {
307 static int swap_task;
308 int table;
309 int page;
310 long pg_table;
311 int loop;
312 int counter = NR_TASKS * 2 >> priority;
313 struct task_struct *p;
314
315 counter = NR_TASKS * 2 >> priority;
316 for(; counter >= 0; counter--, swap_task++) {
317
318
319
320
321 loop = 0;
322 while(1) {
323 if(swap_task >= NR_TASKS) {
324 swap_task = 1;
325 if(loop)
326
327 return 0;
328 loop = 1;
329 }
330
331 p = task[swap_task];
332 if(p && p->swappable && p->rss)
333 break;
334
335 swap_task++;
336 }
337
338
339
340
341 if(! p -> swap_cnt) {
342 p->dec_flt = (p->dec_flt * 3) / 4 + p->maj_flt - p->old_maj_flt;
343 p->old_maj_flt = p->maj_flt;
344
345 if(p->dec_flt >= SWAP_RATIO / SWAP_MIN) {
346 p->dec_flt = SWAP_RATIO / SWAP_MIN;
347 p->swap_cnt = SWAP_MIN;
348 } else if(p->dec_flt <= SWAP_RATIO / SWAP_MAX)
349 p->swap_cnt = SWAP_MAX;
350 else
351 p->swap_cnt = SWAP_RATIO / p->dec_flt;
352 }
353
354
355
356
357 for(table = p->swap_table; table < 1024; table++) {
358 pg_table = ((unsigned long *) p->tss.cr3)[table];
359 if(pg_table >= high_memory)
360 continue;
361 if(mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)
362 continue;
363 if(!(PAGE_PRESENT & pg_table)) {
364 printk("swap_out: bad page-table at pg_dir[%d]: %08lx\n",
365 table, pg_table);
366 ((unsigned long *) p->tss.cr3)[table] = 0;
367 continue;
368 }
369 pg_table &= 0xfffff000;
370
371
372
373
374 for(page = p->swap_page; page < 1024; page++) {
375 switch(try_to_swap_out(page + (unsigned long *) pg_table)) {
376 case 0:
377 break;
378
379 case 1:
380 p->rss--;
381
382 p->swap_table = table;
383 p->swap_page = page + 1;
384 if((--p->swap_cnt) == 0)
385 swap_task++;
386 return 1;
387
388 default:
389 p->rss--;
390 break;
391 }
392 }
393
394 p->swap_page = 0;
395 }
396
397
398
399
400
401 p->swap_table = 0;
402 }
403 return 0;
404 }
405
406 #else
407
408
409
410
411
412
413
414
415
416 static int swap_out(unsigned int priority)
417 {
418 static int swap_task = 1;
419 static int swap_table = 0;
420 static int swap_page = 0;
421 int counter = NR_TASKS*8;
422 int pg_table;
423 struct task_struct * p;
424
425 counter >>= priority;
426 check_task:
427 if (counter-- < 0)
428 return 0;
429 if (swap_task >= NR_TASKS) {
430 swap_task = 1;
431 goto check_task;
432 }
433 p = task[swap_task];
434 if (!p || !p->swappable) {
435 swap_task++;
436 goto check_task;
437 }
438 check_dir:
439 if (swap_table >= PTRS_PER_PAGE) {
440 swap_table = 0;
441 swap_task++;
442 goto check_task;
443 }
444 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
445 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
446 swap_table++;
447 goto check_dir;
448 }
449 if (!(PAGE_PRESENT & pg_table)) {
450 printk("bad page-table at pg_dir[%d]: %08x\n",
451 swap_table,pg_table);
452 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
453 swap_table++;
454 goto check_dir;
455 }
456 pg_table &= PAGE_MASK;
457 check_table:
458 if (swap_page >= PTRS_PER_PAGE) {
459 swap_page = 0;
460 swap_table++;
461 goto check_dir;
462 }
463 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
464 case 0: break;
465 case 1: p->rss--; return 1;
466 default: p->rss--;
467 }
468 swap_page++;
469 goto check_table;
470 }
471
472 #endif
473
474 static int try_to_free_page(void)
475 {
476 int i=6;
477
478 while (i--) {
479 if (shrink_buffers(i))
480 return 1;
481 if (shm_swap(i))
482 return 1;
483 if (swap_out(i))
484 return 1;
485 }
486 return 0;
487 }
488
489
490
491
492
493
494 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
495 {
496 addr &= PAGE_MASK;
497 *(unsigned long *) addr = *queue;
498 *queue = addr;
499 }
500
501
502
503
504
505
506
507
508
509
510
511
512 void free_page(unsigned long addr)
513 {
514 if (addr < high_memory) {
515 unsigned short * map = mem_map + MAP_NR(addr);
516
517 if (*map) {
518 if (!(*map & MAP_PAGE_RESERVED)) {
519 unsigned long flag;
520
521 save_flags(flag);
522 cli();
523 if (!--*map) {
524 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
525 add_mem_queue(addr,&secondary_page_list);
526 nr_secondary_pages++;
527 restore_flags(flag);
528 return;
529 }
530 add_mem_queue(addr,&free_page_list);
531 nr_free_pages++;
532 }
533 restore_flags(flag);
534 }
535 return;
536 }
537 printk("Trying to free free memory (%08lx): memory probabably corrupted\n",addr);
538 printk("PC = %08lx\n",*(((unsigned long *)&addr)-1));
539 return;
540 }
541 }
542
543
544
545
546
547
548
549
550
551
552
553 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
554 cli(); \
555 if ((result = queue) != 0) { \
556 if (!(result & ~PAGE_MASK) && result < high_memory) { \
557 queue = *(unsigned long *) result; \
558 if (!mem_map[MAP_NR(result)]) { \
559 mem_map[MAP_NR(result)] = 1; \
560 nr--; \
561 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
562 restore_flags(flag); \
563 return result; \
564 } \
565 printk("Free page %08lx has mem_map = %d\n", \
566 result,mem_map[MAP_NR(result)]); \
567 } else \
568 printk("Result = 0x%08lx - memory map destroyed\n", result); \
569 queue = 0; \
570 nr = 0; \
571 } else if (nr) { \
572 printk(#nr " is %d, but " #queue " is empty\n",nr); \
573 nr = 0; \
574 } \
575 restore_flags(flag)
576
577
578
579
580
581
582
583
584
585
586 unsigned long __get_free_page(int priority)
587 {
588 unsigned long result, flag;
589 static unsigned long index = 0;
590
591
592
593
594
595
596 if (intr_count && priority != GFP_ATOMIC) {
597 static int count = 0;
598 if (++count < 5) {
599 printk("gfp called nonatomically from interrupt %08lx\n",
600 ((unsigned long *)&priority)[-1]);
601 priority = GFP_ATOMIC;
602 }
603 }
604 save_flags(flag);
605 repeat:
606 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
607 if (priority == GFP_BUFFER)
608 return 0;
609 if (priority != GFP_ATOMIC)
610 if (try_to_free_page())
611 goto repeat;
612 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
613 return 0;
614 }
615
616
617
618
619
620
621 static int try_to_unuse(unsigned int type)
622 {
623 int nr, pgt, pg;
624 unsigned long page, *ppage;
625 unsigned long tmp = 0;
626 struct task_struct *p;
627
628 nr = 0;
629
630
631
632
633 repeat:
634 for (; nr < NR_TASKS ; nr++) {
635 p = task[nr];
636 if (!p)
637 continue;
638 for (pgt = 0 ; pgt < PTRS_PER_PAGE ; pgt++) {
639 ppage = pgt + ((unsigned long *) p->tss.cr3);
640 page = *ppage;
641 if (!page)
642 continue;
643 if (!(page & PAGE_PRESENT) || (page >= high_memory))
644 continue;
645 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
646 continue;
647 ppage = (unsigned long *) (page & PAGE_MASK);
648 for (pg = 0 ; pg < PTRS_PER_PAGE ; pg++,ppage++) {
649 page = *ppage;
650 if (!page)
651 continue;
652 if (page & PAGE_PRESENT)
653 continue;
654 if (SWP_TYPE(page) != type)
655 continue;
656 if (!tmp) {
657 if (!(tmp = __get_free_page(GFP_KERNEL)))
658 return -ENOMEM;
659 goto repeat;
660 }
661 read_swap_page(page, (char *) tmp);
662 if (*ppage == page) {
663 *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
664 ++p->rss;
665 swap_free(page);
666 tmp = 0;
667 }
668 goto repeat;
669 }
670 }
671 }
672 free_page(tmp);
673 return 0;
674 }
675
676 asmlinkage int sys_swapoff(const char * specialfile)
677 {
678 struct swap_info_struct * p;
679 struct inode * inode;
680 unsigned int type;
681 int i;
682
683 if (!suser())
684 return -EPERM;
685 i = namei(specialfile,&inode);
686 if (i)
687 return i;
688 p = swap_info;
689 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
690 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
691 continue;
692 if (p->swap_file) {
693 if (p->swap_file == inode)
694 break;
695 } else {
696 if (!S_ISBLK(inode->i_mode))
697 continue;
698 if (p->swap_device == inode->i_rdev)
699 break;
700 }
701 }
702 iput(inode);
703 if (type >= nr_swapfiles)
704 return -EINVAL;
705 p->flags = SWP_USED;
706 i = try_to_unuse(type);
707 if (i) {
708 p->flags = SWP_WRITEOK;
709 return i;
710 }
711 nr_swap_pages -= p->pages;
712 iput(p->swap_file);
713 p->swap_file = NULL;
714 p->swap_device = 0;
715 vfree(p->swap_map);
716 p->swap_map = NULL;
717 free_page((long) p->swap_lockmap);
718 p->swap_lockmap = NULL;
719 p->flags = 0;
720 return 0;
721 }
722
723
724
725
726
727
728 asmlinkage int sys_swapon(const char * specialfile)
729 {
730 struct swap_info_struct * p;
731 struct inode * swap_inode;
732 unsigned int type;
733 int i,j;
734 int error;
735
736 if (!suser())
737 return -EPERM;
738 p = swap_info;
739 for (type = 0 ; type < nr_swapfiles ; type++,p++)
740 if (!(p->flags & SWP_USED))
741 break;
742 if (type >= MAX_SWAPFILES)
743 return -EPERM;
744 if (type >= nr_swapfiles)
745 nr_swapfiles = type+1;
746 p->flags = SWP_USED;
747 p->swap_file = NULL;
748 p->swap_device = 0;
749 p->swap_map = NULL;
750 p->swap_lockmap = NULL;
751 p->lowest_bit = 0;
752 p->highest_bit = 0;
753 p->max = 1;
754 error = namei(specialfile,&swap_inode);
755 if (error)
756 goto bad_swap;
757 error = -EBUSY;
758 if (swap_inode->i_count != 1)
759 goto bad_swap;
760 error = -EINVAL;
761 if (S_ISBLK(swap_inode->i_mode)) {
762 p->swap_device = swap_inode->i_rdev;
763 iput(swap_inode);
764 error = -ENODEV;
765 if (!p->swap_device)
766 goto bad_swap;
767 error = -EBUSY;
768 for (i = 0 ; i < nr_swapfiles ; i++) {
769 if (i == type)
770 continue;
771 if (p->swap_device == swap_info[i].swap_device)
772 goto bad_swap;
773 }
774 } else if (S_ISREG(swap_inode->i_mode))
775 p->swap_file = swap_inode;
776 else
777 goto bad_swap;
778 p->swap_lockmap = (unsigned char *) get_free_page(GFP_USER);
779 if (!p->swap_lockmap) {
780 printk("Unable to start swapping: out of memory :-)\n");
781 error = -ENOMEM;
782 goto bad_swap;
783 }
784 read_swap_page(SWP_ENTRY(type,0), (char *) p->swap_lockmap);
785 if (memcmp("SWAP-SPACE",p->swap_lockmap+4086,10)) {
786 printk("Unable to find swap-space signature\n");
787 error = -EINVAL;
788 goto bad_swap;
789 }
790 memset(p->swap_lockmap+PAGE_SIZE-10,0,10);
791 j = 0;
792 p->lowest_bit = 0;
793 p->highest_bit = 0;
794 for (i = 1 ; i < 8*PAGE_SIZE ; i++) {
795 if (test_bit(i,p->swap_lockmap)) {
796 if (!p->lowest_bit)
797 p->lowest_bit = i;
798 p->highest_bit = i;
799 p->max = i+1;
800 j++;
801 }
802 }
803 if (!j) {
804 printk("Empty swap-file\n");
805 error = -EINVAL;
806 goto bad_swap;
807 }
808 p->swap_map = (unsigned char *) vmalloc(p->max);
809 if (!p->swap_map) {
810 error = -ENOMEM;
811 goto bad_swap;
812 }
813 for (i = 1 ; i < p->max ; i++) {
814 if (test_bit(i,p->swap_lockmap))
815 p->swap_map[i] = 0;
816 else
817 p->swap_map[i] = 0x80;
818 }
819 p->swap_map[0] = 0x80;
820 memset(p->swap_lockmap,0,PAGE_SIZE);
821 p->flags = SWP_WRITEOK;
822 p->pages = j;
823 nr_swap_pages += j;
824 printk("Adding Swap: %dk swap-space\n",j<<2);
825 return 0;
826 bad_swap:
827 free_page((long) p->swap_lockmap);
828 vfree(p->swap_map);
829 iput(p->swap_file);
830 p->swap_device = 0;
831 p->swap_file = NULL;
832 p->swap_map = NULL;
833 p->swap_lockmap = NULL;
834 p->flags = 0;
835 return error;
836 }
837
838 void si_swapinfo(struct sysinfo *val)
839 {
840 unsigned int i, j;
841
842 val->freeswap = val->totalswap = 0;
843 for (i = 0; i < nr_swapfiles; i++) {
844 if (!(swap_info[i].flags & SWP_USED))
845 continue;
846 for (j = 0; j < swap_info[i].max; ++j)
847 switch (swap_info[i].swap_map[j]) {
848 case 128:
849 continue;
850 case 0:
851 ++val->freeswap;
852 default:
853 ++val->totalswap;
854 }
855 }
856 val->freeswap <<= PAGE_SHIFT;
857 val->totalswap <<= PAGE_SHIFT;
858 return;
859 }