This source file includes following definitions.
- show_swap_cache_info
- add_to_swap_cache
- init_swap_cache
- swap_setup
- buff_setup
- rw_swap_page
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- swap_out_pmd
- swap_out_pgd
- swap_out_vma
- swap_out_process
- swap_out
- try_to_free_page
- add_mem_queue
- remove_mem_queue
- free_pages_ok
- check_free_buffers
- free_pages
- mark_used
- __get_free_pages
- show_free_areas
- unuse_pte
- unuse_pmd
- unuse_pgd
- unuse_vma
- unuse_process
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
- free_area_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #include <linux/mm.h>
15 #include <linux/sched.h>
16 #include <linux/head.h>
17 #include <linux/kernel.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/errno.h>
20 #include <linux/string.h>
21 #include <linux/stat.h>
22 #include <linux/swap.h>
23 #include <linux/fs.h>
24 #include <linux/swapctl.h>
25
26 #include <asm/dma.h>
27 #include <asm/system.h>
28 #include <asm/segment.h>
29 #include <asm/bitops.h>
30 #include <asm/pgtable.h>
31
32 #define MAX_SWAPFILES 8
33
34 #define SWP_USED 1
35 #define SWP_WRITEOK 3
36
37 int min_free_pages = 20;
38
39
40
41
42
43
44
45 swap_control_t swap_control = {
46 20, 3, 1, 3,
47 10, 2, 2, 0,
48 32, 4,
49 8192, 4096,
50 -200,
51 1, 1,
52 RCL_ROUND_ROBIN
53 };
54
55 static int nr_swapfiles = 0;
56 static struct wait_queue * lock_queue = NULL;
57 static struct {
58 int head;
59 int next;
60 } swap_list = {-1, -1};
61
62 static struct swap_info_struct {
63 unsigned int flags;
64 kdev_t swap_device;
65 struct inode * swap_file;
66 unsigned char * swap_map;
67 unsigned char * swap_lockmap;
68 int lowest_bit;
69 int highest_bit;
70 int prio;
71 int pages;
72 unsigned long max;
73 int next;
74 } swap_info[MAX_SWAPFILES];
75
76 extern int shm_swap (int, unsigned long);
77
78
79
80
81
82
83
84
85 unsigned long *swap_cache;
86
87 #ifdef SWAP_CACHE_INFO
88 unsigned long swap_cache_add_total = 0;
89 unsigned long swap_cache_add_success = 0;
90 unsigned long swap_cache_del_total = 0;
91 unsigned long swap_cache_del_success = 0;
92 unsigned long swap_cache_find_total = 0;
93 unsigned long swap_cache_find_success = 0;
94
95 extern inline void show_swap_cache_info(void)
96 {
97 printk("Swap cache: add %ld/%ld, delete %ld/%ld, find %ld/%ld\n",
98 swap_cache_add_total, swap_cache_add_success,
99 swap_cache_del_total, swap_cache_del_success,
100 swap_cache_find_total, swap_cache_find_success);
101 }
102 #endif
103
104 static int add_to_swap_cache(unsigned long addr, unsigned long entry)
105 {
106 struct swap_info_struct * p = &swap_info[SWP_TYPE(entry)];
107
108 #ifdef SWAP_CACHE_INFO
109 swap_cache_add_total++;
110 #endif
111 if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
112 entry = xchg(swap_cache + MAP_NR(addr), entry);
113 if (entry) {
114 printk("swap_cache: replacing non-NULL entry\n");
115 }
116 #ifdef SWAP_CACHE_INFO
117 swap_cache_add_success++;
118 #endif
119 return 1;
120 }
121 return 0;
122 }
123
124 static unsigned long init_swap_cache(unsigned long mem_start,
125 unsigned long mem_end)
126 {
127 unsigned long swap_cache_size;
128
129 mem_start = (mem_start + 15) & ~15;
130 swap_cache = (unsigned long *) mem_start;
131 swap_cache_size = MAP_NR(mem_end);
132 memset(swap_cache, 0, swap_cache_size * sizeof (unsigned long));
133 return (unsigned long) (swap_cache + swap_cache_size);
134 }
135
136
137
138
139 void swap_setup(char *str, int *ints)
140 {
141 int * swap_vars[8] = {
142 &MAX_PAGE_AGE,
143 &PAGE_ADVANCE,
144 &PAGE_DECLINE,
145 &PAGE_INITIAL_AGE,
146 &AGE_CLUSTER_FRACT,
147 &AGE_CLUSTER_MIN,
148 &PAGEOUT_WEIGHT,
149 &BUFFEROUT_WEIGHT
150 };
151 int i;
152 for (i=0; i < ints[0] && i < 8; i++) {
153 if (ints[i+1])
154 *(swap_vars[i]) = ints[i+1];
155 }
156 }
157
158
159 void buff_setup(char *str, int *ints)
160 {
161 int * buff_vars[6] = {
162 &MAX_BUFF_AGE,
163 &BUFF_ADVANCE,
164 &BUFF_DECLINE,
165 &BUFF_INITIAL_AGE,
166 &BUFFEROUT_WEIGHT,
167 &BUFFERMEM_GRACE
168 };
169 int i;
170 for (i=0; i < ints[0] && i < 6; i++) {
171 if (ints[i+1])
172 *(buff_vars[i]) = ints[i+1];
173 }
174 }
175
176
177
178 void rw_swap_page(int rw, unsigned long entry, char * buf)
179 {
180 unsigned long type, offset;
181 struct swap_info_struct * p;
182
183 type = SWP_TYPE(entry);
184 if (type >= nr_swapfiles) {
185 printk("Internal error: bad swap-device\n");
186 return;
187 }
188 p = &swap_info[type];
189 offset = SWP_OFFSET(entry);
190 if (offset >= p->max) {
191 printk("rw_swap_page: weirdness\n");
192 return;
193 }
194 if (p->swap_map && !p->swap_map[offset]) {
195 printk("Hmm.. Trying to use unallocated swap (%08lx)\n", entry);
196 return;
197 }
198 if (!(p->flags & SWP_USED)) {
199 printk("Trying to swap to unused swap-device\n");
200 return;
201 }
202 while (set_bit(offset,p->swap_lockmap))
203 sleep_on(&lock_queue);
204 if (rw == READ)
205 kstat.pswpin++;
206 else
207 kstat.pswpout++;
208 if (p->swap_device) {
209 ll_rw_page(rw,p->swap_device,offset,buf);
210 } else if (p->swap_file) {
211 struct inode *swapf = p->swap_file;
212 unsigned int zones[PAGE_SIZE/512];
213 int i;
214 if (swapf->i_op->bmap == NULL
215 && swapf->i_op->smap != NULL){
216
217
218
219
220
221
222
223
224
225
226
227
228 int j;
229 unsigned int block = offset << 3;
230
231 for (i=0, j=0; j< PAGE_SIZE ; i++, j += 512){
232 if (!(zones[i] = swapf->i_op->smap(swapf,block++))) {
233 printk("rw_swap_page: bad swap file\n");
234 return;
235 }
236 }
237 }else{
238 int j;
239 unsigned int block = offset
240 << (PAGE_SHIFT - swapf->i_sb->s_blocksize_bits);
241
242 for (i=0, j=0; j< PAGE_SIZE ; i++, j +=swapf->i_sb->s_blocksize)
243 if (!(zones[i] = bmap(swapf,block++))) {
244 printk("rw_swap_page: bad swap file\n");
245 return;
246 }
247 }
248 ll_rw_swap_file(rw,swapf->i_dev, zones, i,buf);
249 } else
250 printk("re_swap_page: no swap file or device\n");
251 if (offset && !clear_bit(offset,p->swap_lockmap))
252 printk("rw_swap_page: lock already cleared\n");
253 wake_up(&lock_queue);
254 }
255
256 unsigned long get_swap_page(void)
257 {
258 struct swap_info_struct * p;
259 unsigned long offset, entry;
260 int type, wrapped = 0;
261
262 type = swap_list.next;
263 if (type < 0)
264 return 0;
265
266 while (1) {
267 p = &swap_info[type];
268 if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
269 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
270 if (p->swap_map[offset])
271 continue;
272 if (test_bit(offset, p->swap_lockmap))
273 continue;
274 p->swap_map[offset] = 1;
275 nr_swap_pages--;
276 if (offset == p->highest_bit)
277 p->highest_bit--;
278 p->lowest_bit = offset;
279 entry = SWP_ENTRY(type,offset);
280
281 type = swap_info[type].next;
282 if (type < 0 || p->prio != swap_info[type].prio) {
283 swap_list.next = swap_list.head;
284 } else {
285 swap_list.next = type;
286 }
287 return entry;
288 }
289 }
290 type = p->next;
291 if (!wrapped) {
292 if (type < 0 || p->prio != swap_info[type].prio) {
293 type = swap_list.head;
294 wrapped = 1;
295 }
296 } else if (type < 0) {
297 return 0;
298 }
299 }
300 }
301
302 void swap_duplicate(unsigned long entry)
303 {
304 struct swap_info_struct * p;
305 unsigned long offset, type;
306
307 if (!entry)
308 return;
309 offset = SWP_OFFSET(entry);
310 type = SWP_TYPE(entry);
311 if (type & SHM_SWP_TYPE)
312 return;
313 if (type >= nr_swapfiles) {
314 printk("Trying to duplicate nonexistent swap-page\n");
315 return;
316 }
317 p = type + swap_info;
318 if (offset >= p->max) {
319 printk("swap_duplicate: weirdness\n");
320 return;
321 }
322 if (!p->swap_map[offset]) {
323 printk("swap_duplicate: trying to duplicate unused page\n");
324 return;
325 }
326 p->swap_map[offset]++;
327 return;
328 }
329
330 void swap_free(unsigned long entry)
331 {
332 struct swap_info_struct * p;
333 unsigned long offset, type;
334
335 if (!entry)
336 return;
337 type = SWP_TYPE(entry);
338 if (type & SHM_SWP_TYPE)
339 return;
340 if (type >= nr_swapfiles) {
341 printk("Trying to free nonexistent swap-page\n");
342 return;
343 }
344 p = & swap_info[type];
345 offset = SWP_OFFSET(entry);
346 if (offset >= p->max) {
347 printk("swap_free: weirdness\n");
348 return;
349 }
350 if (!(p->flags & SWP_USED)) {
351 printk("Trying to free swap from unused swap-device\n");
352 return;
353 }
354 if (offset < p->lowest_bit)
355 p->lowest_bit = offset;
356 if (offset > p->highest_bit)
357 p->highest_bit = offset;
358 if (!p->swap_map[offset])
359 printk("swap_free: swap-space map bad (entry %08lx)\n",entry);
360 else
361 if (!--p->swap_map[offset])
362 nr_swap_pages++;
363 if (p->prio > swap_info[swap_list.next].prio) {
364 swap_list.next = swap_list.head;
365 }
366 }
367
368
369
370
371
372
373
374
375 void swap_in(struct task_struct * tsk, struct vm_area_struct * vma,
376 pte_t * page_table, unsigned long entry, int write_access)
377 {
378 unsigned long page = __get_free_page(GFP_KERNEL);
379
380 if (pte_val(*page_table) != entry) {
381 free_page(page);
382 return;
383 }
384 if (!page) {
385 set_pte(page_table, BAD_PAGE);
386 swap_free(entry);
387 oom(tsk);
388 return;
389 }
390 read_swap_page(entry, (char *) page);
391 if (pte_val(*page_table) != entry) {
392 free_page(page);
393 return;
394 }
395 vma->vm_mm->rss++;
396 tsk->maj_flt++;
397 if (!write_access && add_to_swap_cache(page, entry)) {
398 set_pte(page_table, mk_pte(page, vma->vm_page_prot));
399 return;
400 }
401 set_pte(page_table, pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))));
402 swap_free(entry);
403 return;
404 }
405
406
407
408
409
410
411
412
413
414
415
416
417 static inline int try_to_swap_out(struct task_struct * tsk, struct vm_area_struct* vma,
418 unsigned long address, pte_t * page_table, unsigned long limit)
419 {
420 pte_t pte;
421 unsigned long entry;
422 unsigned long page;
423
424 pte = *page_table;
425 if (!pte_present(pte))
426 return 0;
427 page = pte_page(pte);
428 if (page >= high_memory)
429 return 0;
430 if (page >= limit)
431 return 0;
432
433 if (mem_map[MAP_NR(page)].reserved)
434 return 0;
435
436
437
438 if ((pte_dirty(pte) && delete_from_swap_cache(page))
439 || pte_young(pte)) {
440 set_pte(page_table, pte_mkold(pte));
441 touch_page(page);
442 return 0;
443 }
444 age_page(page);
445 if (age_of(page))
446 return 0;
447 if (pte_dirty(pte)) {
448 if (vma->vm_ops && vma->vm_ops->swapout) {
449 pid_t pid = tsk->pid;
450 vma->vm_mm->rss--;
451 if (vma->vm_ops->swapout(vma, address - vma->vm_start + vma->vm_offset, page_table))
452 kill_proc(pid, SIGBUS, 1);
453 } else {
454 if (mem_map[MAP_NR(page)].count != 1)
455 return 0;
456 if (!(entry = get_swap_page()))
457 return 0;
458 vma->vm_mm->rss--;
459 set_pte(page_table, __pte(entry));
460 invalidate();
461 tsk->nswap++;
462 write_swap_page(entry, (char *) page);
463 }
464 free_page(page);
465 return 1;
466 }
467 if ((entry = find_in_swap_cache(page))) {
468 if (mem_map[MAP_NR(page)].count != 1) {
469 set_pte(page_table, pte_mkdirty(pte));
470 printk("Aiee.. duplicated cached swap-cache entry\n");
471 return 0;
472 }
473 vma->vm_mm->rss--;
474 set_pte(page_table, __pte(entry));
475 invalidate();
476 free_page(page);
477 return 1;
478 }
479 vma->vm_mm->rss--;
480 pte_clear(page_table);
481 invalidate();
482 entry = mem_map[MAP_NR(page)].count;
483 free_page(page);
484 return entry;
485 }
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501 static inline int swap_out_pmd(struct task_struct * tsk, struct vm_area_struct * vma,
502 pmd_t *dir, unsigned long address, unsigned long end, unsigned long limit)
503 {
504 pte_t * pte;
505 unsigned long pmd_end;
506
507 if (pmd_none(*dir))
508 return 0;
509 if (pmd_bad(*dir)) {
510 printk("swap_out_pmd: bad pmd (%08lx)\n", pmd_val(*dir));
511 pmd_clear(dir);
512 return 0;
513 }
514
515 pte = pte_offset(dir, address);
516
517 pmd_end = (address + PMD_SIZE) & PMD_MASK;
518 if (end > pmd_end)
519 end = pmd_end;
520
521 do {
522 int result;
523 tsk->swap_address = address + PAGE_SIZE;
524 result = try_to_swap_out(tsk, vma, address, pte, limit);
525 if (result)
526 return result;
527 address += PAGE_SIZE;
528 pte++;
529 } while (address < end);
530 return 0;
531 }
532
533 static inline int swap_out_pgd(struct task_struct * tsk, struct vm_area_struct * vma,
534 pgd_t *dir, unsigned long address, unsigned long end, unsigned long limit)
535 {
536 pmd_t * pmd;
537 unsigned long pgd_end;
538
539 if (pgd_none(*dir))
540 return 0;
541 if (pgd_bad(*dir)) {
542 printk("swap_out_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
543 pgd_clear(dir);
544 return 0;
545 }
546
547 pmd = pmd_offset(dir, address);
548
549 pgd_end = (address + PGDIR_SIZE) & PGDIR_MASK;
550 if (end > pgd_end)
551 end = pgd_end;
552
553 do {
554 int result = swap_out_pmd(tsk, vma, pmd, address, end, limit);
555 if (result)
556 return result;
557 address = (address + PMD_SIZE) & PMD_MASK;
558 pmd++;
559 } while (address < end);
560 return 0;
561 }
562
563 static int swap_out_vma(struct task_struct * tsk, struct vm_area_struct * vma,
564 pgd_t *pgdir, unsigned long start, unsigned long limit)
565 {
566 unsigned long end;
567
568
569
570 if (vma->vm_flags & VM_SHM)
571 return 0;
572
573
574
575 if (vma->vm_flags & VM_DONTSWAP)
576 return 0;
577
578 end = vma->vm_end;
579 while (start < end) {
580 int result = swap_out_pgd(tsk, vma, pgdir, start, end, limit);
581 if (result)
582 return result;
583 start = (start + PGDIR_SIZE) & PGDIR_MASK;
584 pgdir++;
585 }
586 return 0;
587 }
588
589 static int swap_out_process(struct task_struct * p, unsigned long limit)
590 {
591 unsigned long address;
592 struct vm_area_struct* vma;
593
594
595
596
597 address = p->swap_address;
598 p->swap_address = 0;
599
600
601
602
603 vma = find_vma(p, address);
604 if (!vma)
605 return 0;
606 if (address < vma->vm_start)
607 address = vma->vm_start;
608
609 for (;;) {
610 int result = swap_out_vma(p, vma, pgd_offset(p->mm, address), address, limit);
611 if (result)
612 return result;
613 vma = vma->vm_next;
614 if (!vma)
615 break;
616 address = vma->vm_start;
617 }
618 p->swap_address = 0;
619 return 0;
620 }
621
622 static int swap_out(unsigned int priority, unsigned long limit)
623 {
624 static int swap_task;
625 int loop, counter;
626 struct task_struct *p;
627
628 counter = ((PAGEOUT_WEIGHT * nr_tasks) >> 10) >> priority;
629 for(; counter >= 0; counter--) {
630
631
632
633
634 loop = 0;
635 while(1) {
636 if (swap_task >= NR_TASKS) {
637 swap_task = 1;
638 if (loop)
639
640 return 0;
641 loop = 1;
642 }
643
644 p = task[swap_task];
645 if (p && p->swappable && p->mm->rss)
646 break;
647
648 swap_task++;
649 }
650
651
652
653
654 if (!p->swap_cnt) {
655
656
657 p->swap_cnt = AGE_CLUSTER_SIZE(p->mm->rss);
658 }
659 if (!--p->swap_cnt)
660 swap_task++;
661 switch (swap_out_process(p, limit)) {
662 case 0:
663 if (p->swap_cnt)
664 swap_task++;
665 break;
666 case 1:
667 return 1;
668 default:
669 break;
670 }
671 }
672 return 0;
673 }
674
675
676
677
678
679
680 static int try_to_free_page(int priority, unsigned long limit)
681 {
682 static int state = 0;
683 int i=6;
684
685 switch (state) {
686 do {
687 case 0:
688 if (priority != GFP_NOBUFFER && shrink_buffers(i, limit))
689 return 1;
690 state = 1;
691 case 1:
692 if (shm_swap(i, limit))
693 return 1;
694 state = 2;
695 default:
696 if (swap_out(i, limit))
697 return 1;
698 state = 0;
699 } while(i--);
700 }
701 return 0;
702 }
703
704 static inline void add_mem_queue(struct mem_list * head, struct mem_list * entry)
705 {
706 entry->prev = head;
707 (entry->next = head->next)->prev = entry;
708 head->next = entry;
709 }
710
711 static inline void remove_mem_queue(struct mem_list * head, struct mem_list * entry)
712 {
713 entry->next->prev = entry->prev;
714 entry->prev->next = entry->next;
715 }
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736 static inline void free_pages_ok(unsigned long addr, unsigned long order)
737 {
738 unsigned long index = MAP_NR(addr) >> (1 + order);
739 unsigned long mask = PAGE_MASK << order;
740
741 addr &= mask;
742 nr_free_pages += 1 << order;
743 while (order < NR_MEM_LISTS-1) {
744 if (!change_bit(index, free_area_map[order]))
745 break;
746 remove_mem_queue(free_area_list+order, (struct mem_list *) (addr ^ (1+~mask)));
747 order++;
748 index >>= 1;
749 mask <<= 1;
750 addr &= mask;
751 }
752 add_mem_queue(free_area_list+order, (struct mem_list *) addr);
753 }
754
755 static inline void check_free_buffers(unsigned long addr)
756 {
757 struct buffer_head * bh;
758
759 bh = buffer_pages[MAP_NR(addr)];
760 if (bh) {
761 struct buffer_head *tmp = bh;
762 do {
763 if (tmp->b_list == BUF_SHARED
764 && tmp->b_dev != B_FREE)
765 refile_buffer(tmp);
766 tmp = tmp->b_this_page;
767 } while (tmp != bh);
768 }
769 }
770
771 void free_pages(unsigned long addr, unsigned long order)
772 {
773 if (MAP_NR(addr) < MAP_NR(high_memory)) {
774 unsigned long flag;
775 mem_map_t * map = mem_map + MAP_NR(addr);
776 if (map->reserved)
777 return;
778 if (map->count) {
779 save_flags(flag);
780 cli();
781 if (!--map->count) {
782 free_pages_ok(addr, order);
783 delete_from_swap_cache(addr);
784 }
785 restore_flags(flag);
786 if (map->count == 1)
787 check_free_buffers(addr);
788 return;
789 }
790 printk("Trying to free free memory (%08lx): memory probably corrupted\n",addr);
791 printk("PC = %p\n", __builtin_return_address(0));
792 return;
793 }
794 }
795
796
797
798
799 #define RMQUEUE(order, limit) \
800 do { struct mem_list * queue = free_area_list+order; \
801 unsigned long new_order = order; \
802 do { struct mem_list *prev = queue, *ret; \
803 while (queue != (ret = prev->next)) { \
804 if ((unsigned long) ret < (limit)) { \
805 (prev->next = ret->next)->prev = prev; \
806 mark_used((unsigned long) ret, new_order); \
807 nr_free_pages -= 1 << order; \
808 restore_flags(flags); \
809 EXPAND(ret, order, new_order); \
810 return (unsigned long) ret; \
811 } \
812 prev = ret; \
813 } \
814 new_order++; queue++; \
815 } while (new_order < NR_MEM_LISTS); \
816 } while (0)
817
818 static inline int mark_used(unsigned long addr, unsigned long order)
819 {
820 return change_bit(MAP_NR(addr) >> (1+order), free_area_map[order]);
821 }
822
823 #define EXPAND(addr,low,high) \
824 do { unsigned long size = PAGE_SIZE << high; \
825 while (high > low) { \
826 high--; size >>= 1; cli(); \
827 add_mem_queue(free_area_list+high, addr); \
828 mark_used((unsigned long) addr, high); \
829 restore_flags(flags); \
830 addr = (struct mem_list *) (size + (unsigned long) addr); \
831 } mem_map[MAP_NR((unsigned long) addr)].count = 1; \
832 mem_map[MAP_NR((unsigned long) addr)].age = PAGE_INITIAL_AGE; \
833 } while (0)
834
835 unsigned long __get_free_pages(int priority, unsigned long order, unsigned long limit)
836 {
837 unsigned long flags;
838 int reserved_pages;
839
840 if (intr_count && priority != GFP_ATOMIC) {
841 static int count = 0;
842 if (++count < 5) {
843 printk("gfp called nonatomically from interrupt %p\n",
844 __builtin_return_address(0));
845 priority = GFP_ATOMIC;
846 }
847 }
848 reserved_pages = 5;
849 if (priority != GFP_NFS)
850 reserved_pages = min_free_pages;
851 save_flags(flags);
852 repeat:
853 cli();
854 if ((priority==GFP_ATOMIC) || nr_free_pages > reserved_pages) {
855 RMQUEUE(order, limit);
856 restore_flags(flags);
857 return 0;
858 }
859 restore_flags(flags);
860 if (priority != GFP_BUFFER && try_to_free_page(priority, limit))
861 goto repeat;
862 return 0;
863 }
864
865
866
867
868
869
870 void show_free_areas(void)
871 {
872 unsigned long order, flags;
873 unsigned long total = 0;
874
875 printk("Free pages: %6dkB\n ( ",nr_free_pages<<(PAGE_SHIFT-10));
876 save_flags(flags);
877 cli();
878 for (order=0 ; order < NR_MEM_LISTS; order++) {
879 struct mem_list * tmp;
880 unsigned long nr = 0;
881 for (tmp = free_area_list[order].next ; tmp != free_area_list + order ; tmp = tmp->next) {
882 nr ++;
883 }
884 total += nr * ((PAGE_SIZE>>10) << order);
885 printk("%lu*%lukB ", nr, (PAGE_SIZE>>10) << order);
886 }
887 restore_flags(flags);
888 printk("= %lukB)\n", total);
889 #ifdef SWAP_CACHE_INFO
890 show_swap_cache_info();
891 #endif
892 }
893
894
895
896
897
898
899
900
901
902
903 static inline int unuse_pte(struct vm_area_struct * vma, unsigned long address,
904 pte_t *dir, unsigned int type, unsigned long page)
905 {
906 pte_t pte = *dir;
907
908 if (pte_none(pte))
909 return 0;
910 if (pte_present(pte)) {
911 unsigned long page = pte_page(pte);
912 if (page >= high_memory)
913 return 0;
914 if (!in_swap_cache(page))
915 return 0;
916 if (SWP_TYPE(in_swap_cache(page)) != type)
917 return 0;
918 delete_from_swap_cache(page);
919 set_pte(dir, pte_mkdirty(pte));
920 return 0;
921 }
922 if (SWP_TYPE(pte_val(pte)) != type)
923 return 0;
924 read_swap_page(pte_val(pte), (char *) page);
925 if (pte_val(*dir) != pte_val(pte)) {
926 free_page(page);
927 return 1;
928 }
929 set_pte(dir, pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))));
930 ++vma->vm_mm->rss;
931 swap_free(pte_val(pte));
932 return 1;
933 }
934
935 static inline int unuse_pmd(struct vm_area_struct * vma, pmd_t *dir,
936 unsigned long address, unsigned long size, unsigned long offset,
937 unsigned int type, unsigned long page)
938 {
939 pte_t * pte;
940 unsigned long end;
941
942 if (pmd_none(*dir))
943 return 0;
944 if (pmd_bad(*dir)) {
945 printk("unuse_pmd: bad pmd (%08lx)\n", pmd_val(*dir));
946 pmd_clear(dir);
947 return 0;
948 }
949 pte = pte_offset(dir, address);
950 offset += address & PMD_MASK;
951 address &= ~PMD_MASK;
952 end = address + size;
953 if (end > PMD_SIZE)
954 end = PMD_SIZE;
955 do {
956 if (unuse_pte(vma, offset+address-vma->vm_start, pte, type, page))
957 return 1;
958 address += PAGE_SIZE;
959 pte++;
960 } while (address < end);
961 return 0;
962 }
963
964 static inline int unuse_pgd(struct vm_area_struct * vma, pgd_t *dir,
965 unsigned long address, unsigned long size,
966 unsigned int type, unsigned long page)
967 {
968 pmd_t * pmd;
969 unsigned long offset, end;
970
971 if (pgd_none(*dir))
972 return 0;
973 if (pgd_bad(*dir)) {
974 printk("unuse_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
975 pgd_clear(dir);
976 return 0;
977 }
978 pmd = pmd_offset(dir, address);
979 offset = address & PGDIR_MASK;
980 address &= ~PGDIR_MASK;
981 end = address + size;
982 if (end > PGDIR_SIZE)
983 end = PGDIR_SIZE;
984 do {
985 if (unuse_pmd(vma, pmd, address, end - address, offset, type, page))
986 return 1;
987 address = (address + PMD_SIZE) & PMD_MASK;
988 pmd++;
989 } while (address < end);
990 return 0;
991 }
992
993 static int unuse_vma(struct vm_area_struct * vma, pgd_t *pgdir,
994 unsigned long start, unsigned long end,
995 unsigned int type, unsigned long page)
996 {
997 while (start < end) {
998 if (unuse_pgd(vma, pgdir, start, end - start, type, page))
999 return 1;
1000 start = (start + PGDIR_SIZE) & PGDIR_MASK;
1001 pgdir++;
1002 }
1003 return 0;
1004 }
1005
1006 static int unuse_process(struct task_struct * p, unsigned int type, unsigned long page)
1007 {
1008 struct vm_area_struct* vma;
1009
1010
1011
1012
1013 if (!p->mm || pgd_inuse(p->mm->pgd))
1014 return 0;
1015 vma = p->mm->mmap;
1016 while (vma) {
1017 pgd_t * pgd = pgd_offset(p->mm, vma->vm_start);
1018 if (unuse_vma(vma, pgd, vma->vm_start, vma->vm_end, type, page))
1019 return 1;
1020 vma = vma->vm_next;
1021 }
1022 return 0;
1023 }
1024
1025
1026
1027
1028
1029
1030 static int try_to_unuse(unsigned int type)
1031 {
1032 int nr;
1033 unsigned long page = get_free_page(GFP_KERNEL);
1034
1035 if (!page)
1036 return -ENOMEM;
1037 nr = 0;
1038 while (nr < NR_TASKS) {
1039 if (task[nr]) {
1040 if (unuse_process(task[nr], type, page)) {
1041 page = get_free_page(GFP_KERNEL);
1042 if (!page)
1043 return -ENOMEM;
1044 continue;
1045 }
1046 }
1047 nr++;
1048 }
1049 free_page(page);
1050 return 0;
1051 }
1052
1053 asmlinkage int sys_swapoff(const char * specialfile)
1054 {
1055 struct swap_info_struct * p;
1056 struct inode * inode;
1057 struct file filp;
1058 int i, type, prev;
1059
1060 if (!suser())
1061 return -EPERM;
1062 i = namei(specialfile,&inode);
1063 if (i)
1064 return i;
1065 prev = -1;
1066 for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
1067 p = swap_info + type;
1068 if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
1069 if (p->swap_file) {
1070 if (p->swap_file == inode)
1071 break;
1072 } else {
1073 if (S_ISBLK(inode->i_mode)
1074 && (p->swap_device == inode->i_rdev))
1075 break;
1076 }
1077 }
1078 prev = type;
1079 }
1080 if (type < 0){
1081 iput(inode);
1082 return -EINVAL;
1083 }
1084 if (prev < 0) {
1085 swap_list.head = p->next;
1086 } else {
1087 swap_info[prev].next = p->next;
1088 }
1089 if (type == swap_list.next) {
1090
1091 swap_list.next = swap_list.head;
1092 }
1093 p->flags = SWP_USED;
1094 i = try_to_unuse(type);
1095 if (i) {
1096 iput(inode);
1097 p->flags = SWP_WRITEOK;
1098 return i;
1099 }
1100
1101 if(p->swap_device){
1102 memset(&filp, 0, sizeof(filp));
1103 filp.f_inode = inode;
1104 filp.f_mode = 3;
1105
1106 if( !blkdev_open(inode, &filp) &&
1107 filp.f_op && filp.f_op->release){
1108 filp.f_op->release(inode,&filp);
1109 filp.f_op->release(inode,&filp);
1110 }
1111 }
1112 iput(inode);
1113
1114 nr_swap_pages -= p->pages;
1115 iput(p->swap_file);
1116 p->swap_file = NULL;
1117 p->swap_device = 0;
1118 vfree(p->swap_map);
1119 p->swap_map = NULL;
1120 free_page((long) p->swap_lockmap);
1121 p->swap_lockmap = NULL;
1122 p->flags = 0;
1123 return 0;
1124 }
1125
1126
1127
1128
1129
1130
1131 asmlinkage int sys_swapon(const char * specialfile, int swap_flags)
1132 {
1133 struct swap_info_struct * p;
1134 struct inode * swap_inode;
1135 unsigned int type;
1136 int i, j, prev;
1137 int error;
1138 struct file filp;
1139 static int least_priority = 0;
1140
1141 memset(&filp, 0, sizeof(filp));
1142 if (!suser())
1143 return -EPERM;
1144 p = swap_info;
1145 for (type = 0 ; type < nr_swapfiles ; type++,p++)
1146 if (!(p->flags & SWP_USED))
1147 break;
1148 if (type >= MAX_SWAPFILES)
1149 return -EPERM;
1150 if (type >= nr_swapfiles)
1151 nr_swapfiles = type+1;
1152 p->flags = SWP_USED;
1153 p->swap_file = NULL;
1154 p->swap_device = 0;
1155 p->swap_map = NULL;
1156 p->swap_lockmap = NULL;
1157 p->lowest_bit = 0;
1158 p->highest_bit = 0;
1159 p->max = 1;
1160 p->next = -1;
1161 if (swap_flags & SWAP_FLAG_PREFER) {
1162 p->prio =
1163 (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
1164 } else {
1165 p->prio = --least_priority;
1166 }
1167 error = namei(specialfile,&swap_inode);
1168 if (error)
1169 goto bad_swap_2;
1170 p->swap_file = swap_inode;
1171 error = -EBUSY;
1172 if (swap_inode->i_count != 1)
1173 goto bad_swap_2;
1174 error = -EINVAL;
1175
1176 if (S_ISBLK(swap_inode->i_mode)) {
1177 p->swap_device = swap_inode->i_rdev;
1178
1179 filp.f_inode = swap_inode;
1180 filp.f_mode = 3;
1181 error = blkdev_open(swap_inode, &filp);
1182 p->swap_file = NULL;
1183 iput(swap_inode);
1184 if(error)
1185 goto bad_swap_2;
1186 error = -ENODEV;
1187 if (!p->swap_device)
1188 goto bad_swap;
1189 error = -EBUSY;
1190 for (i = 0 ; i < nr_swapfiles ; i++) {
1191 if (i == type)
1192 continue;
1193 if (p->swap_device == swap_info[i].swap_device)
1194 goto bad_swap;
1195 }
1196 } else if (!S_ISREG(swap_inode->i_mode))
1197 goto bad_swap;
1198 p->swap_lockmap = (unsigned char *) get_free_page(GFP_USER);
1199 if (!p->swap_lockmap) {
1200 printk("Unable to start swapping: out of memory :-)\n");
1201 error = -ENOMEM;
1202 goto bad_swap;
1203 }
1204 read_swap_page(SWP_ENTRY(type,0), (char *) p->swap_lockmap);
1205 if (memcmp("SWAP-SPACE",p->swap_lockmap+PAGE_SIZE-10,10)) {
1206 printk("Unable to find swap-space signature\n");
1207 error = -EINVAL;
1208 goto bad_swap;
1209 }
1210 memset(p->swap_lockmap+PAGE_SIZE-10,0,10);
1211 j = 0;
1212 p->lowest_bit = 0;
1213 p->highest_bit = 0;
1214 for (i = 1 ; i < 8*PAGE_SIZE ; i++) {
1215 if (test_bit(i,p->swap_lockmap)) {
1216 if (!p->lowest_bit)
1217 p->lowest_bit = i;
1218 p->highest_bit = i;
1219 p->max = i+1;
1220 j++;
1221 }
1222 }
1223 if (!j) {
1224 printk("Empty swap-file\n");
1225 error = -EINVAL;
1226 goto bad_swap;
1227 }
1228 p->swap_map = (unsigned char *) vmalloc(p->max);
1229 if (!p->swap_map) {
1230 error = -ENOMEM;
1231 goto bad_swap;
1232 }
1233 for (i = 1 ; i < p->max ; i++) {
1234 if (test_bit(i,p->swap_lockmap))
1235 p->swap_map[i] = 0;
1236 else
1237 p->swap_map[i] = 0x80;
1238 }
1239 p->swap_map[0] = 0x80;
1240 memset(p->swap_lockmap,0,PAGE_SIZE);
1241 p->flags = SWP_WRITEOK;
1242 p->pages = j;
1243 nr_swap_pages += j;
1244 printk("Adding Swap: %dk swap-space\n",j<<(PAGE_SHIFT-10));
1245
1246
1247 prev = -1;
1248 for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
1249 if (p->prio >= swap_info[i].prio) {
1250 break;
1251 }
1252 prev = i;
1253 }
1254 p->next = i;
1255 if (prev < 0) {
1256 swap_list.head = swap_list.next = p - swap_info;
1257 } else {
1258 swap_info[prev].next = p - swap_info;
1259 }
1260 return 0;
1261 bad_swap:
1262 if(filp.f_op && filp.f_op->release)
1263 filp.f_op->release(filp.f_inode,&filp);
1264 bad_swap_2:
1265 free_page((long) p->swap_lockmap);
1266 vfree(p->swap_map);
1267 iput(p->swap_file);
1268 p->swap_device = 0;
1269 p->swap_file = NULL;
1270 p->swap_map = NULL;
1271 p->swap_lockmap = NULL;
1272 p->flags = 0;
1273 return error;
1274 }
1275
1276 void si_swapinfo(struct sysinfo *val)
1277 {
1278 unsigned int i, j;
1279
1280 val->freeswap = val->totalswap = 0;
1281 for (i = 0; i < nr_swapfiles; i++) {
1282 if ((swap_info[i].flags & SWP_WRITEOK) != SWP_WRITEOK)
1283 continue;
1284 for (j = 0; j < swap_info[i].max; ++j)
1285 switch (swap_info[i].swap_map[j]) {
1286 case 128:
1287 continue;
1288 case 0:
1289 ++val->freeswap;
1290 default:
1291 ++val->totalswap;
1292 }
1293 }
1294 val->freeswap <<= PAGE_SHIFT;
1295 val->totalswap <<= PAGE_SHIFT;
1296 return;
1297 }
1298
1299 #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
1300
1301
1302
1303
1304
1305
1306
1307 unsigned long free_area_init(unsigned long start_mem, unsigned long end_mem)
1308 {
1309 mem_map_t * p;
1310 unsigned long mask = PAGE_MASK;
1311 int i;
1312
1313
1314
1315
1316
1317 i = (end_mem - PAGE_OFFSET) >> (PAGE_SHIFT+6);
1318 if (i < 16)
1319 i = 16;
1320 min_free_pages = i;
1321 start_mem = init_swap_cache(start_mem, end_mem);
1322 mem_map = (mem_map_t *) start_mem;
1323 p = mem_map + MAP_NR(end_mem);
1324 start_mem = LONG_ALIGN((unsigned long) p);
1325 while (p > mem_map) {
1326 --p;
1327 p->count = 0;
1328 p->dirty = 0;
1329 p->reserved = 1;
1330 }
1331
1332 for (i = 0 ; i < NR_MEM_LISTS ; i++) {
1333 unsigned long bitmap_size;
1334 free_area_list[i].prev = free_area_list[i].next = &free_area_list[i];
1335 mask += mask;
1336 end_mem = (end_mem + ~mask) & mask;
1337 bitmap_size = (end_mem - PAGE_OFFSET) >> (PAGE_SHIFT + i);
1338 bitmap_size = (bitmap_size + 7) >> 3;
1339 bitmap_size = LONG_ALIGN(bitmap_size);
1340 free_area_map[i] = (unsigned char *) start_mem;
1341 memset((void *) start_mem, 0, bitmap_size);
1342 start_mem += bitmap_size;
1343 }
1344 return start_mem;
1345 }