This source file includes following definitions.
- bitop
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19 #include <asm/system.h>
20
21 #define MAX_SWAPFILES 8
22
23 #define SWP_USED 1
24 #define SWP_WRITEOK 3
25
26 static int nr_swapfiles = 0;
27 static struct wait_queue * lock_queue = NULL;
28
29 static struct swap_info_struct {
30 unsigned long flags;
31 struct inode * swap_file;
32 unsigned int swap_device;
33 unsigned char * swap_map;
34 char * swap_lockmap;
35 int lowest_bit;
36 int highest_bit;
37 } swap_info[MAX_SWAPFILES];
38
39 extern unsigned long free_page_list;
40
41
42
43
44
45 #define NR_LAST_FREE_PAGES 32
46 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
47
48 #define SWAP_BITS 4096
49
50 #define bitop(name,op) \
51 static inline int name(char * addr,unsigned int nr) \
52 { \
53 int __res; \
54 __asm__ __volatile__("bt" op " %1,%2; adcl $0,%0" \
55 :"=g" (__res) \
56 :"r" (nr),"m" (*(addr)),"0" (0)); \
57 return __res; \
58 }
59
60 bitop(bit,"")
61 bitop(setbit,"s")
62 bitop(clrbit,"r")
63
64 void rw_swap_page(int rw, unsigned int nr, char * buf)
65 {
66 struct swap_info_struct * p;
67
68 if ((nr >> 24) >= nr_swapfiles) {
69 printk("Internal error: bad swap-device\n");
70 return;
71 }
72 p = swap_info + (nr >> 24);
73 nr &= 0x00ffffff;
74 if (nr >= SWAP_BITS) {
75 printk("rw_swap_page: weirdness\n");
76 return;
77 }
78 if (!(p->flags & SWP_USED)) {
79 printk("Trying to swap to unused swap-device\n");
80 return;
81 }
82 while (setbit(p->swap_lockmap,nr))
83 sleep_on(&lock_queue);
84 if (p->swap_device) {
85 ll_rw_page(rw,p->swap_device,nr,buf);
86 } else if (p->swap_file) {
87 unsigned int zones[4];
88 unsigned int block = nr << 2;
89 int i;
90
91 for (i = 0; i < 4; i++)
92 if (!(zones[i] = bmap(p->swap_file,block++))) {
93 printk("rw_swap_page: bad swap file\n");
94 return;
95 }
96 ll_rw_swap_file(rw,p->swap_file->i_dev, zones,4,buf);
97 } else
98 printk("re_swap_page: no swap file or device\n");
99 if (!clrbit(p->swap_lockmap,nr))
100 printk("rw_swap_page: lock already cleared\n");
101 wake_up(&lock_queue);
102 }
103
104 static unsigned int get_swap_page(void)
105 {
106 struct swap_info_struct * p;
107 unsigned int block_nr, swap_nr;
108
109 p = swap_info;
110 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++) {
111 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
112 continue;
113 for (block_nr = p->lowest_bit; block_nr <= p->highest_bit ; block_nr++) {
114 if (p->swap_map[block_nr])
115 continue;
116 p->swap_map[block_nr] = 1;
117 if (block_nr == p->highest_bit)
118 p->highest_bit--;
119 p->lowest_bit = block_nr;
120 return block_nr + (swap_nr << 24);
121 }
122 }
123 return 0;
124 }
125
126 void swap_duplicate(unsigned int nr)
127 {
128 struct swap_info_struct * p;
129
130 if (!nr)
131 return;
132 if ((nr >> 24) >= nr_swapfiles) {
133 printk("Trying to free nonexistent swap-page\n");
134 return;
135 }
136 p = (nr >> 24) + swap_info;
137 nr &= 0x00ffffff;
138 if (nr >= SWAP_BITS) {
139 printk("swap_free: weirness\n");
140 return;
141 }
142 if (!p->swap_map[nr]) {
143 printk("swap_duplicate: trying to duplicate unused page\n");
144 return;
145 }
146 p->swap_map[nr]++;
147 }
148
149 void swap_free(unsigned int nr)
150 {
151 struct swap_info_struct * p;
152
153 if (!nr)
154 return;
155 if ((nr >> 24) >= nr_swapfiles) {
156 printk("Trying to free nonexistent swap-page\n");
157 return;
158 }
159 p = (nr >> 24) + swap_info;
160 nr &= 0x00ffffff;
161 if (nr >= SWAP_BITS) {
162 printk("swap_free: weirness\n");
163 return;
164 }
165 if (!(p->flags & SWP_USED)) {
166 printk("Trying to free swap from unused swap-device\n");
167 return;
168 }
169 while (setbit(p->swap_lockmap,nr))
170 sleep_on(&lock_queue);
171 if (nr < p->lowest_bit)
172 p->lowest_bit = nr;
173 if (nr > p->highest_bit)
174 p->highest_bit = nr;
175 if (!p->swap_map[nr])
176 printk("swap_free: swap-space map bad (page %d)\n",nr);
177 else
178 p->swap_map[nr]--;
179 if (!clrbit(p->swap_lockmap,nr))
180 printk("swap_free: lock already cleared\n");
181 wake_up(&lock_queue);
182 }
183
184 void swap_in(unsigned long *table_ptr)
185 {
186 unsigned long swap_nr;
187 unsigned long page;
188
189 swap_nr = *table_ptr;
190 if (1 & swap_nr) {
191 printk("trying to swap in present page\n\r");
192 return;
193 }
194 if (!swap_nr) {
195 printk("No swap page in swap_in\n\r");
196 return;
197 }
198 page = get_free_page(GFP_KERNEL);
199 if (!page) {
200 oom(current);
201 page = BAD_PAGE;
202 } else
203 read_swap_page(swap_nr>>1, (char *) page);
204 if (*table_ptr != swap_nr) {
205 free_page(page);
206 return;
207 }
208 *table_ptr = page | (PAGE_DIRTY | 7);
209 swap_free(swap_nr>>1);
210 }
211
212 static int try_to_swap_out(unsigned long * table_ptr)
213 {
214 int i;
215 unsigned long page;
216 unsigned long swap_nr;
217
218 page = *table_ptr;
219 if (!(PAGE_PRESENT & page))
220 return 0;
221 if (page >= high_memory) {
222 printk("try_to_swap_out: bad page (%08x)\n",page);
223 *table_ptr = 0;
224 return 0;
225 }
226 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
227 return 0;
228 if (PAGE_ACCESSED & page) {
229 *table_ptr &= ~PAGE_ACCESSED;
230 return 0;
231 }
232 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
233 if (last_free_pages[i] == (page & 0xfffff000))
234 return 0;
235 if (PAGE_DIRTY & page) {
236 page &= 0xfffff000;
237 if (mem_map[MAP_NR(page)] != 1)
238 return 0;
239 if (!(swap_nr = get_swap_page()))
240 return 0;
241 *table_ptr = swap_nr<<1;
242 invalidate();
243 write_swap_page(swap_nr, (char *) page);
244 free_page(page);
245 return 1;
246 }
247 page &= 0xfffff000;
248 *table_ptr = 0;
249 invalidate();
250 free_page(page);
251 return 1 + mem_map[MAP_NR(page)];
252 }
253
254
255
256
257
258
259 int sys_idle(void)
260 {
261 need_resched = 1;
262 return 0;
263 }
264
265
266
267
268
269
270
271
272
273 static int swap_out(unsigned int priority)
274 {
275 static int swap_task = 1;
276 static int swap_table = 0;
277 static int swap_page = 0;
278 int counter = NR_TASKS*8;
279 int pg_table;
280 struct task_struct * p;
281
282 counter >>= priority;
283 check_task:
284 if (counter-- < 0)
285 return 0;
286 if (swap_task >= NR_TASKS) {
287 swap_task = 1;
288 goto check_task;
289 }
290 p = task[swap_task];
291 if (!p || !p->swappable) {
292 swap_task++;
293 goto check_task;
294 }
295 check_dir:
296 if (swap_table >= 1024) {
297 swap_table = 0;
298 swap_task++;
299 goto check_task;
300 }
301 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
302 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
303 swap_table++;
304 goto check_dir;
305 }
306 if (!(1 & pg_table)) {
307 printk("bad page-table at pg_dir[%d]: %08x\n\r",
308 swap_table,pg_table);
309 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
310 swap_table++;
311 goto check_dir;
312 }
313 pg_table &= 0xfffff000;
314 check_table:
315 if (swap_page >= 1024) {
316 swap_page = 0;
317 swap_table++;
318 goto check_dir;
319 }
320 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
321 case 0: break;
322 case 1: p->rss--; return 1;
323 default: p->rss--;
324 }
325 swap_page++;
326 goto check_table;
327 }
328
329 static int try_to_free_page(void)
330 {
331 int i=6;
332
333 while (i--) {
334 if (shrink_buffers(i))
335 return 1;
336 if (swap_out(i))
337 return 1;
338 }
339 return 0;
340 }
341
342
343
344
345
346
347 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
348 {
349 addr &= 0xfffff000;
350 *(unsigned long *) addr = *queue;
351 *queue = addr;
352 }
353
354
355
356
357
358
359
360
361
362
363
364
365 void free_page(unsigned long addr)
366 {
367 if (addr < high_memory) {
368 unsigned short * map = mem_map + MAP_NR(addr);
369
370 if (*map) {
371 if (!(*map & MAP_PAGE_RESERVED)) {
372 unsigned long flag;
373
374 save_flags(flag);
375 cli();
376 if (!--*map) {
377 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
378 add_mem_queue(addr,&secondary_page_list);
379 nr_secondary_pages++;
380 restore_flags(flag);
381 return;
382 }
383 add_mem_queue(addr,&free_page_list);
384 nr_free_pages++;
385 }
386 restore_flags(flag);
387 }
388 return;
389 }
390 printk("Trying to free free memory (%08x): memory probabably corrupted\n",addr);
391 return;
392 }
393 printk("Trying to free nonexistent page %08x\n",addr);
394 return;
395 }
396
397
398
399
400
401
402
403
404
405
406
407 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
408 cli(); \
409 if ((result = queue) != 0) { \
410 if (!(result & 0xfff) && result < high_memory) { \
411 queue = *(unsigned long *) result; \
412 if (!mem_map[MAP_NR(result)]) { \
413 mem_map[MAP_NR(result)] = 1; \
414 nr--; \
415 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
416 restore_flags(flag); \
417 __asm__ __volatile__("cld ; rep ; stosl" \
418 ::"a" (0),"c" (1024),"D" (result) \
419 :"di","cx"); \
420 return result; \
421 } \
422 printk("Free page %08x has mem_map = %d\n", \
423 result,mem_map[MAP_NR(result)]); \
424 } else \
425 printk("Result = 0x%08x - memory map destroyed\n", result); \
426 queue = 0; \
427 nr = 0; \
428 } else if (nr) { \
429 printk(#nr " is %d, but " #queue " is empty\n",nr); \
430 nr = 0; \
431 } \
432 restore_flags(flag)
433
434
435
436
437
438
439
440
441
442
443 unsigned long get_free_page(int priority)
444 {
445 unsigned long result, flag;
446 static unsigned long index = 0;
447
448
449
450
451
452
453 save_flags(flag);
454 repeat:
455 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
456 if (priority == GFP_BUFFER)
457 return 0;
458 if (priority != GFP_ATOMIC)
459 if (try_to_free_page())
460 goto repeat;
461 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
462 return 0;
463 }
464
465
466
467
468
469
470
471
472
473
474 static int try_to_unuse(unsigned int swap_nr)
475 {
476 int nr, pgt, pg;
477 unsigned long page, *ppage;
478 unsigned long tmp = 0;
479 struct task_struct *p;
480
481 nr = 0;
482
483
484
485
486 repeat:
487 for (; nr < NR_TASKS ; nr++) {
488 p = task[nr];
489 if (!p)
490 continue;
491 for (pgt = 0 ; pgt < 1024 ; pgt++) {
492 ppage = pgt + ((unsigned long *) p->tss.cr3);
493 page = *ppage;
494 if (!page)
495 continue;
496 if (!(page & PAGE_PRESENT) || (page >= high_memory)) {
497 printk("try_to_unuse: bad page directory (%d,%d:%08x)\n",nr,pgt,page);
498 *ppage = 0;
499 continue;
500 }
501 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
502 continue;
503 ppage = (unsigned long *) (page & 0xfffff000);
504 for (pg = 0 ; pg < 1024 ; pg++,ppage++) {
505 page = *ppage;
506 if (!page)
507 continue;
508 if (page & PAGE_PRESENT) {
509 if (page >= high_memory) {
510 printk("try_to_unuse: bad page table (%d,%d,%d:%08x)\n",nr,pgt,pg,page);
511 *ppage = 0;
512 }
513 continue;
514 }
515 if ((page >> 25) != swap_nr)
516 continue;
517 if (!tmp) {
518 tmp = get_free_page(GFP_KERNEL);
519 if (!tmp)
520 return -ENOMEM;
521 goto repeat;
522 }
523 read_swap_page(page>>1, (char *) tmp);
524 if (*ppage == page) {
525 *ppage = tmp | (PAGE_DIRTY | 7);
526 ++p->rss;
527 swap_free(page>>1);
528 tmp = 0;
529 }
530 goto repeat;
531 }
532 }
533 }
534 free_page(tmp);
535 return 0;
536 }
537
538 int sys_swapoff(const char * specialfile)
539 {
540 struct swap_info_struct * p;
541 struct inode * inode;
542 unsigned int swap_nr;
543 int i;
544
545 if (!suser())
546 return -EPERM;
547 i = namei(specialfile,&inode);
548 if (i)
549 return i;
550 p = swap_info;
551 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++) {
552 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
553 continue;
554 if (p->swap_file) {
555 if (p->swap_file == inode)
556 break;
557 } else {
558 if (!S_ISBLK(inode->i_mode))
559 continue;
560 if (p->swap_device == inode->i_rdev)
561 break;
562 }
563 }
564 iput(inode);
565 if (swap_nr >= nr_swapfiles)
566 return -EINVAL;
567 p->flags = SWP_USED;
568 i = try_to_unuse(swap_nr);
569 if (i) {
570 p->flags = SWP_WRITEOK;
571 return i;
572 }
573 iput(p->swap_file);
574 p->swap_file = NULL;
575 p->swap_device = 0;
576 free_page((long) p->swap_map);
577 p->swap_map = NULL;
578 free_page((long) p->swap_lockmap);
579 p->swap_lockmap = NULL;
580 p->flags = 0;
581 return 0;
582 }
583
584
585
586
587
588
589 int sys_swapon(const char * specialfile)
590 {
591 struct swap_info_struct * p;
592 struct inode * swap_inode;
593 unsigned int swap_nr;
594 char * tmp;
595 int i,j;
596
597 if (!suser())
598 return -EPERM;
599 p = swap_info;
600 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++)
601 if (!(p->flags & SWP_USED))
602 break;
603 if (swap_nr >= MAX_SWAPFILES)
604 return -EPERM;
605 if (swap_nr >= nr_swapfiles)
606 nr_swapfiles = swap_nr+1;
607 p->flags = SWP_USED;
608 p->swap_file = NULL;
609 p->swap_device = 0;
610 p->swap_map = NULL;
611 p->swap_lockmap = NULL;
612 p->lowest_bit = 0;
613 p->highest_bit = 0;
614 i = namei(specialfile,&swap_inode);
615 if (i) {
616 p->flags = 0;
617 return i;
618 }
619 if (swap_inode->i_count != 1) {
620 iput(swap_inode);
621 p->flags = 0;
622 return -EBUSY;
623 }
624 if (S_ISBLK(swap_inode->i_mode)) {
625 p->swap_device = swap_inode->i_rdev;
626 iput(swap_inode);
627 if (!p->swap_device) {
628 p->flags = 0;
629 return -ENODEV;
630 }
631 for (i = 0 ; i < nr_swapfiles ; i++) {
632 if (i == swap_nr)
633 continue;
634 if (p->swap_device == swap_info[i].swap_device) {
635 p->swap_device = 0;
636 p->flags = 0;
637 return -EBUSY;
638 }
639 }
640 } else if (S_ISREG(swap_inode->i_mode))
641 p->swap_file = swap_inode;
642 else {
643 iput(swap_inode);
644 p->flags = 0;
645 return -EINVAL;
646 }
647 tmp = (char *) get_free_page(GFP_USER);
648 p->swap_lockmap = (char *) get_free_page(GFP_USER);
649 if (!tmp || !p->swap_lockmap) {
650 printk("Unable to start swapping: out of memory :-)\n");
651 free_page((long) tmp);
652 free_page((long) p->swap_lockmap);
653 iput(p->swap_file);
654 p->swap_device = 0;
655 p->swap_file = NULL;
656 p->swap_map = NULL;
657 p->swap_lockmap = NULL;
658 p->flags = 0;
659 return -ENOMEM;
660 }
661 read_swap_page(swap_nr << 24,tmp);
662 if (strncmp("SWAP-SPACE",tmp+4086,10)) {
663 printk("Unable to find swap-space signature\n\r");
664 free_page((long) tmp);
665 free_page((long) p->swap_lockmap);
666 iput(p->swap_file);
667 p->swap_device = 0;
668 p->swap_file = NULL;
669 p->swap_map = NULL;
670 p->swap_lockmap = NULL;
671 p->flags = 0;
672 return -EINVAL;
673 }
674 memset(tmp+4086,0,10);
675 j = 0;
676 p->lowest_bit = 0;
677 p->highest_bit = 0;
678 for (i = 1 ; i < SWAP_BITS ; i++)
679 if (bit(tmp,i)) {
680 if (!p->lowest_bit)
681 p->lowest_bit = i;
682 p->highest_bit = i;
683 j++;
684 }
685 if (!j) {
686 printk("Empty swap-file\n");
687 free_page((long) tmp);
688 free_page((long) p->swap_lockmap);
689 iput(p->swap_file);
690 p->swap_device = 0;
691 p->swap_file = NULL;
692 p->swap_map = NULL;
693 p->swap_lockmap = NULL;
694 p->flags = 0;
695 return -EINVAL;
696 }
697 i = SWAP_BITS;
698 while (i--)
699 if (bit(tmp,i))
700 tmp[i] = 0;
701 else
702 tmp[i] = 128;
703 tmp[0] = 128;
704 p->swap_map = tmp;
705 p->flags = SWP_WRITEOK;
706 printk("Adding Swap: %dk swap-space\n\r",j<<2);
707 return 0;
708 }
709
710 void si_swapinfo(struct sysinfo *val)
711 {
712 unsigned int i, j;
713
714 val->freeswap = val->totalswap = 0;
715 for (i = 0; i < nr_swapfiles; i++) {
716 if (!(swap_info[i].flags & SWP_USED))
717 continue;
718 for (j = 0; j < 4096; ++j)
719 switch (swap_info[i].swap_map[j]) {
720 case 128:
721 continue;
722 case 0:
723 ++val->freeswap;
724 default:
725 ++val->totalswap;
726 }
727 }
728 val->freeswap <<= PAGE_SHIFT;
729 val->totalswap <<= PAGE_SHIFT;
730 return;
731 }