This source file includes following definitions.
- rw_swap_page
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- __get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19
20 #include <asm/system.h>
21 #include <asm/bitops.h>
22
23 #define MAX_SWAPFILES 8
24
25 #define SWP_USED 1
26 #define SWP_WRITEOK 3
27
28 #define SWP_TYPE(entry) (((entry) & 0xfe) >> 1)
29 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
30 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
31
32 static int nr_swapfiles = 0;
33 static struct wait_queue * lock_queue = NULL;
34
35 static struct swap_info_struct {
36 unsigned long flags;
37 struct inode * swap_file;
38 unsigned int swap_device;
39 unsigned char * swap_map;
40 unsigned char * swap_lockmap;
41 int pages;
42 int lowest_bit;
43 int highest_bit;
44 unsigned long max;
45 } swap_info[MAX_SWAPFILES];
46
47 extern unsigned long free_page_list;
48 extern int shm_swap (int);
49
50
51
52
53
54 #define NR_LAST_FREE_PAGES 32
55 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
56
57 void rw_swap_page(int rw, unsigned long entry, char * buf)
58 {
59 unsigned long type, offset;
60 struct swap_info_struct * p;
61
62 type = SWP_TYPE(entry);
63 if (type >= nr_swapfiles) {
64 printk("Internal error: bad swap-device\n");
65 return;
66 }
67 p = &swap_info[type];
68 offset = SWP_OFFSET(entry);
69 if (offset >= p->max) {
70 printk("rw_swap_page: weirdness\n");
71 return;
72 }
73 if (!(p->flags & SWP_USED)) {
74 printk("Trying to swap to unused swap-device\n");
75 return;
76 }
77 while (set_bit(offset,p->swap_lockmap))
78 sleep_on(&lock_queue);
79 if (p->swap_device) {
80 ll_rw_page(rw,p->swap_device,offset,buf);
81 } else if (p->swap_file) {
82 unsigned int zones[8];
83 unsigned int block;
84 int i, j;
85
86 block = offset << (12 - p->swap_file->i_sb->s_blocksize_bits);
87
88 for (i=0, j=0; j< PAGE_SIZE ; i++, j +=p->swap_file->i_sb->s_blocksize)
89 if (!(zones[i] = bmap(p->swap_file,block++))) {
90 printk("rw_swap_page: bad swap file\n");
91 return;
92 }
93 ll_rw_swap_file(rw,p->swap_file->i_dev, zones, i,buf);
94 } else
95 printk("re_swap_page: no swap file or device\n");
96 if (offset && !clear_bit(offset,p->swap_lockmap))
97 printk("rw_swap_page: lock already cleared\n");
98 wake_up(&lock_queue);
99 }
100
101 unsigned int get_swap_page(void)
102 {
103 struct swap_info_struct * p;
104 unsigned int offset, type;
105
106 p = swap_info;
107 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
108 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
109 continue;
110 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
111 if (p->swap_map[offset])
112 continue;
113 p->swap_map[offset] = 1;
114 nr_swap_pages--;
115 if (offset == p->highest_bit)
116 p->highest_bit--;
117 p->lowest_bit = offset;
118 return SWP_ENTRY(type,offset);
119 }
120 }
121 return 0;
122 }
123
124 unsigned long swap_duplicate(unsigned long entry)
125 {
126 struct swap_info_struct * p;
127 unsigned long offset, type;
128
129 if (!entry)
130 return 0;
131 offset = SWP_OFFSET(entry);
132 type = SWP_TYPE(entry);
133 if (type == SHM_SWP_TYPE)
134 return entry;
135 if (type >= nr_swapfiles) {
136 printk("Trying to duplicate nonexistent swap-page\n");
137 return 0;
138 }
139 p = type + swap_info;
140 if (offset >= p->max) {
141 printk("swap_free: weirdness\n");
142 return 0;
143 }
144 if (!p->swap_map[offset]) {
145 printk("swap_duplicate: trying to duplicate unused page\n");
146 return 0;
147 }
148 p->swap_map[offset]++;
149 return entry;
150 }
151
152 void swap_free(unsigned long entry)
153 {
154 struct swap_info_struct * p;
155 unsigned long offset, type;
156
157 if (!entry)
158 return;
159 type = SWP_TYPE(entry);
160 if (type == SHM_SWP_TYPE)
161 return;
162 if (type >= nr_swapfiles) {
163 printk("Trying to free nonexistent swap-page\n");
164 return;
165 }
166 p = & swap_info[type];
167 offset = SWP_OFFSET(entry);
168 if (offset >= p->max) {
169 printk("swap_free: weirdness\n");
170 return;
171 }
172 if (!(p->flags & SWP_USED)) {
173 printk("Trying to free swap from unused swap-device\n");
174 return;
175 }
176 while (set_bit(offset,p->swap_lockmap))
177 sleep_on(&lock_queue);
178 if (offset < p->lowest_bit)
179 p->lowest_bit = offset;
180 if (offset > p->highest_bit)
181 p->highest_bit = offset;
182 if (!p->swap_map[offset])
183 printk("swap_free: swap-space map bad (entry %08lx)\n",entry);
184 else
185 if (!--p->swap_map[offset])
186 nr_swap_pages++;
187 if (!clear_bit(offset,p->swap_lockmap))
188 printk("swap_free: lock already cleared\n");
189 wake_up(&lock_queue);
190 }
191
192 void swap_in(unsigned long *table_ptr)
193 {
194 unsigned long entry;
195 unsigned long page;
196
197 entry = *table_ptr;
198 if (PAGE_PRESENT & entry) {
199 printk("trying to swap in present page\n");
200 return;
201 }
202 if (!entry) {
203 printk("No swap page in swap_in\n");
204 return;
205 }
206 if (SWP_TYPE(entry) == SHM_SWP_TYPE) {
207 shm_no_page ((unsigned long *) table_ptr);
208 return;
209 }
210 if (!(page = get_free_page(GFP_KERNEL))) {
211 oom(current);
212 page = BAD_PAGE;
213 } else
214 read_swap_page(entry, (char *) page);
215 if (*table_ptr != entry) {
216 free_page(page);
217 return;
218 }
219 *table_ptr = page | (PAGE_DIRTY | PAGE_PRIVATE);
220 swap_free(entry);
221 }
222
223 static inline int try_to_swap_out(unsigned long * table_ptr)
224 {
225 int i;
226 unsigned long page;
227 unsigned long entry;
228
229 page = *table_ptr;
230 if (!(PAGE_PRESENT & page))
231 return 0;
232 if (page >= high_memory)
233 return 0;
234 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
235 return 0;
236 if (PAGE_ACCESSED & page) {
237 *table_ptr &= ~PAGE_ACCESSED;
238 return 0;
239 }
240 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
241 if (last_free_pages[i] == (page & PAGE_MASK))
242 return 0;
243 if (PAGE_DIRTY & page) {
244 page &= PAGE_MASK;
245 if (mem_map[MAP_NR(page)] != 1)
246 return 0;
247 if (!(entry = get_swap_page()))
248 return 0;
249 *table_ptr = entry;
250 invalidate();
251 write_swap_page(entry, (char *) page);
252 free_page(page);
253 return 1;
254 }
255 page &= PAGE_MASK;
256 *table_ptr = 0;
257 invalidate();
258 free_page(page);
259 return 1 + mem_map[MAP_NR(page)];
260 }
261
262
263
264
265
266
267 asmlinkage int sys_idle(void)
268 {
269 need_resched = 1;
270 return 0;
271 }
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286 #ifdef NEW_SWAP
287
288
289
290
291 #define SWAP_MIN 4
292 #define SWAP_MAX 32
293
294
295
296
297
298 #define SWAP_RATIO 128
299
300 static int swap_out(unsigned int priority)
301 {
302 static int swap_task;
303 int table;
304 int page;
305 long pg_table;
306 int loop;
307 int counter = NR_TASKS * 2 >> priority;
308 struct task_struct *p;
309
310 counter = NR_TASKS * 2 >> priority;
311 for(; counter >= 0; counter--, swap_task++) {
312
313
314
315
316 loop = 0;
317 while(1) {
318 if(swap_task >= NR_TASKS) {
319 swap_task = 1;
320 if(loop)
321
322 return 0;
323 loop = 1;
324 }
325
326 p = task[swap_task];
327 if(p && p->swappable && p->rss)
328 break;
329
330 swap_task++;
331 }
332
333
334
335
336 if(! p -> swap_cnt) {
337 p->dec_flt = (p->dec_flt * 3) / 4 + p->maj_flt - p->old_maj_flt;
338 p->old_maj_flt = p->maj_flt;
339
340 if(p->dec_flt >= SWAP_RATIO / SWAP_MIN) {
341 p->dec_flt = SWAP_RATIO / SWAP_MIN;
342 p->swap_cnt = SWAP_MIN;
343 } else if(p->dec_flt <= SWAP_RATIO / SWAP_MAX)
344 p->swap_cnt = SWAP_MAX;
345 else
346 p->swap_cnt = SWAP_RATIO / p->dec_flt;
347 }
348
349
350
351
352 for(table = p->swap_table; table < 1024; table++) {
353 pg_table = ((unsigned long *) p->tss.cr3)[table];
354 if(pg_table >= high_memory)
355 continue;
356 if(mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)
357 continue;
358 if(!(PAGE_PRESENT & pg_table)) {
359 printk("swap_out: bad page-table at pg_dir[%d]: %08lx\n",
360 table, pg_table);
361 ((unsigned long *) p->tss.cr3)[table] = 0;
362 continue;
363 }
364 pg_table &= 0xfffff000;
365
366
367
368
369 for(page = p->swap_page; page < 1024; page++) {
370 switch(try_to_swap_out(page + (unsigned long *) pg_table)) {
371 case 0:
372 break;
373
374 case 1:
375 p->rss--;
376
377 p->swap_table = table;
378 p->swap_page = page + 1;
379 if((--p->swap_cnt) == 0)
380 swap_task++;
381 return 1;
382
383 default:
384 p->rss--;
385 break;
386 }
387 }
388
389 p->swap_page = 0;
390 }
391
392
393
394
395
396 p->swap_table = 0;
397 }
398 return 0;
399 }
400
401 #else
402
403
404
405
406
407
408
409
410
411 static int swap_out(unsigned int priority)
412 {
413 static int swap_task = 1;
414 static int swap_table = 0;
415 static int swap_page = 0;
416 int counter = NR_TASKS*8;
417 int pg_table;
418 struct task_struct * p;
419
420 counter >>= priority;
421 check_task:
422 if (counter-- < 0)
423 return 0;
424 if (swap_task >= NR_TASKS) {
425 swap_task = 1;
426 goto check_task;
427 }
428 p = task[swap_task];
429 if (!p || !p->swappable) {
430 swap_task++;
431 goto check_task;
432 }
433 check_dir:
434 if (swap_table >= PTRS_PER_PAGE) {
435 swap_table = 0;
436 swap_task++;
437 goto check_task;
438 }
439 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
440 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
441 swap_table++;
442 goto check_dir;
443 }
444 if (!(PAGE_PRESENT & pg_table)) {
445 printk("bad page-table at pg_dir[%d]: %08x\n",
446 swap_table,pg_table);
447 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
448 swap_table++;
449 goto check_dir;
450 }
451 pg_table &= PAGE_MASK;
452 check_table:
453 if (swap_page >= PTRS_PER_PAGE) {
454 swap_page = 0;
455 swap_table++;
456 goto check_dir;
457 }
458 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
459 case 0: break;
460 case 1: p->rss--; return 1;
461 default: p->rss--;
462 }
463 swap_page++;
464 goto check_table;
465 }
466
467 #endif
468
469 static int try_to_free_page(void)
470 {
471 int i=6;
472
473 while (i--) {
474 if (shrink_buffers(i))
475 return 1;
476 if (shm_swap(i))
477 return 1;
478 if (swap_out(i))
479 return 1;
480 }
481 return 0;
482 }
483
484
485
486
487
488
489 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
490 {
491 addr &= PAGE_MASK;
492 *(unsigned long *) addr = *queue;
493 *queue = addr;
494 }
495
496
497
498
499
500
501
502
503
504
505
506
507 void free_page(unsigned long addr)
508 {
509 if (addr < high_memory) {
510 unsigned short * map = mem_map + MAP_NR(addr);
511
512 if (*map) {
513 if (!(*map & MAP_PAGE_RESERVED)) {
514 unsigned long flag;
515
516 save_flags(flag);
517 cli();
518 if (!--*map) {
519 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
520 add_mem_queue(addr,&secondary_page_list);
521 nr_secondary_pages++;
522 restore_flags(flag);
523 return;
524 }
525 add_mem_queue(addr,&free_page_list);
526 nr_free_pages++;
527 }
528 restore_flags(flag);
529 }
530 return;
531 }
532 printk("Trying to free free memory (%08lx): memory probabably corrupted\n",addr);
533 printk("PC = %08lx\n",*(((unsigned long *)&addr)-1));
534 return;
535 }
536 }
537
538
539
540
541
542
543
544
545
546
547
548 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
549 cli(); \
550 if ((result = queue) != 0) { \
551 if (!(result & ~PAGE_MASK) && result < high_memory) { \
552 queue = *(unsigned long *) result; \
553 if (!mem_map[MAP_NR(result)]) { \
554 mem_map[MAP_NR(result)] = 1; \
555 nr--; \
556 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
557 restore_flags(flag); \
558 return result; \
559 } \
560 printk("Free page %08lx has mem_map = %d\n", \
561 result,mem_map[MAP_NR(result)]); \
562 } else \
563 printk("Result = 0x%08lx - memory map destroyed\n", result); \
564 queue = 0; \
565 nr = 0; \
566 } else if (nr) { \
567 printk(#nr " is %d, but " #queue " is empty\n",nr); \
568 nr = 0; \
569 } \
570 restore_flags(flag)
571
572
573
574
575
576
577
578
579
580
581 unsigned long __get_free_page(int priority)
582 {
583 unsigned long result, flag;
584 static unsigned long index = 0;
585
586
587
588
589
590
591 save_flags(flag);
592 repeat:
593 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
594 if (priority == GFP_BUFFER)
595 return 0;
596 if (priority != GFP_ATOMIC)
597 if (try_to_free_page())
598 goto repeat;
599 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
600 return 0;
601 }
602
603
604
605
606
607
608 static int try_to_unuse(unsigned int type)
609 {
610 int nr, pgt, pg;
611 unsigned long page, *ppage;
612 unsigned long tmp = 0;
613 struct task_struct *p;
614
615 nr = 0;
616
617
618
619
620 repeat:
621 for (; nr < NR_TASKS ; nr++) {
622 p = task[nr];
623 if (!p)
624 continue;
625 for (pgt = 0 ; pgt < PTRS_PER_PAGE ; pgt++) {
626 ppage = pgt + ((unsigned long *) p->tss.cr3);
627 page = *ppage;
628 if (!page)
629 continue;
630 if (!(page & PAGE_PRESENT) || (page >= high_memory))
631 continue;
632 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
633 continue;
634 ppage = (unsigned long *) (page & PAGE_MASK);
635 for (pg = 0 ; pg < PTRS_PER_PAGE ; pg++,ppage++) {
636 page = *ppage;
637 if (!page)
638 continue;
639 if (page & PAGE_PRESENT)
640 continue;
641 if (SWP_TYPE(page) != type)
642 continue;
643 if (!tmp) {
644 if (!(tmp = __get_free_page(GFP_KERNEL)))
645 return -ENOMEM;
646 goto repeat;
647 }
648 read_swap_page(page, (char *) tmp);
649 if (*ppage == page) {
650 *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
651 ++p->rss;
652 swap_free(page);
653 tmp = 0;
654 }
655 goto repeat;
656 }
657 }
658 }
659 free_page(tmp);
660 return 0;
661 }
662
663 asmlinkage int sys_swapoff(const char * specialfile)
664 {
665 struct swap_info_struct * p;
666 struct inode * inode;
667 unsigned int type;
668 int i;
669
670 if (!suser())
671 return -EPERM;
672 i = namei(specialfile,&inode);
673 if (i)
674 return i;
675 p = swap_info;
676 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
677 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
678 continue;
679 if (p->swap_file) {
680 if (p->swap_file == inode)
681 break;
682 } else {
683 if (!S_ISBLK(inode->i_mode))
684 continue;
685 if (p->swap_device == inode->i_rdev)
686 break;
687 }
688 }
689 iput(inode);
690 if (type >= nr_swapfiles)
691 return -EINVAL;
692 p->flags = SWP_USED;
693 i = try_to_unuse(type);
694 if (i) {
695 p->flags = SWP_WRITEOK;
696 return i;
697 }
698 nr_swap_pages -= p->pages;
699 iput(p->swap_file);
700 p->swap_file = NULL;
701 p->swap_device = 0;
702 vfree(p->swap_map);
703 p->swap_map = NULL;
704 free_page((long) p->swap_lockmap);
705 p->swap_lockmap = NULL;
706 p->flags = 0;
707 return 0;
708 }
709
710
711
712
713
714
715 asmlinkage int sys_swapon(const char * specialfile)
716 {
717 struct swap_info_struct * p;
718 struct inode * swap_inode;
719 unsigned int type;
720 int i,j;
721 int error;
722
723 if (!suser())
724 return -EPERM;
725 p = swap_info;
726 for (type = 0 ; type < nr_swapfiles ; type++,p++)
727 if (!(p->flags & SWP_USED))
728 break;
729 if (type >= MAX_SWAPFILES)
730 return -EPERM;
731 if (type >= nr_swapfiles)
732 nr_swapfiles = type+1;
733 p->flags = SWP_USED;
734 p->swap_file = NULL;
735 p->swap_device = 0;
736 p->swap_map = NULL;
737 p->swap_lockmap = NULL;
738 p->lowest_bit = 0;
739 p->highest_bit = 0;
740 p->max = 1;
741 error = namei(specialfile,&swap_inode);
742 if (error)
743 goto bad_swap;
744 error = -EBUSY;
745 if (swap_inode->i_count != 1)
746 goto bad_swap;
747 error = -EINVAL;
748 if (S_ISBLK(swap_inode->i_mode)) {
749 p->swap_device = swap_inode->i_rdev;
750 iput(swap_inode);
751 error = -ENODEV;
752 if (!p->swap_device)
753 goto bad_swap;
754 error = -EBUSY;
755 for (i = 0 ; i < nr_swapfiles ; i++) {
756 if (i == type)
757 continue;
758 if (p->swap_device == swap_info[i].swap_device)
759 goto bad_swap;
760 }
761 } else if (S_ISREG(swap_inode->i_mode))
762 p->swap_file = swap_inode;
763 else
764 goto bad_swap;
765 p->swap_lockmap = (unsigned char *) get_free_page(GFP_USER);
766 if (!p->swap_lockmap) {
767 printk("Unable to start swapping: out of memory :-)\n");
768 error = -ENOMEM;
769 goto bad_swap;
770 }
771 read_swap_page(SWP_ENTRY(type,0), (char *) p->swap_lockmap);
772 if (memcmp("SWAP-SPACE",p->swap_lockmap+4086,10)) {
773 printk("Unable to find swap-space signature\n");
774 error = -EINVAL;
775 goto bad_swap;
776 }
777 memset(p->swap_lockmap+PAGE_SIZE-10,0,10);
778 j = 0;
779 p->lowest_bit = 0;
780 p->highest_bit = 0;
781 for (i = 1 ; i < 8*PAGE_SIZE ; i++) {
782 if (test_bit(i,p->swap_lockmap)) {
783 if (!p->lowest_bit)
784 p->lowest_bit = i;
785 p->highest_bit = i;
786 p->max = i+1;
787 j++;
788 }
789 }
790 if (!j) {
791 printk("Empty swap-file\n");
792 error = -EINVAL;
793 goto bad_swap;
794 }
795 p->swap_map = (unsigned char *) vmalloc(p->max);
796 if (!p->swap_map) {
797 error = -ENOMEM;
798 goto bad_swap;
799 }
800 for (i = 1 ; i < p->max ; i++) {
801 if (test_bit(i,p->swap_lockmap))
802 p->swap_map[i] = 0;
803 else
804 p->swap_map[i] = 0x80;
805 }
806 p->swap_map[0] = 0x80;
807 memset(p->swap_lockmap,0,PAGE_SIZE);
808 p->flags = SWP_WRITEOK;
809 p->pages = j;
810 nr_swap_pages += j;
811 printk("Adding Swap: %dk swap-space\n",j<<2);
812 return 0;
813 bad_swap:
814 free_page((long) p->swap_lockmap);
815 vfree(p->swap_map);
816 iput(p->swap_file);
817 p->swap_device = 0;
818 p->swap_file = NULL;
819 p->swap_map = NULL;
820 p->swap_lockmap = NULL;
821 p->flags = 0;
822 return error;
823 }
824
825 void si_swapinfo(struct sysinfo *val)
826 {
827 unsigned int i, j;
828
829 val->freeswap = val->totalswap = 0;
830 for (i = 0; i < nr_swapfiles; i++) {
831 if (!(swap_info[i].flags & SWP_USED))
832 continue;
833 for (j = 0; j < swap_info[i].max; ++j)
834 switch (swap_info[i].swap_map[j]) {
835 case 128:
836 continue;
837 case 0:
838 ++val->freeswap;
839 default:
840 ++val->totalswap;
841 }
842 }
843 val->freeswap <<= PAGE_SHIFT;
844 val->totalswap <<= PAGE_SHIFT;
845 return;
846 }