This source file includes following definitions.
- bitop
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- remove_from_mem_queue
- get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19 #include <asm/system.h>
20
21 #define MAX_SWAPFILES 8
22
23 #define SWP_USED 1
24 #define SWP_WRITEOK 3
25
26 static int nr_swapfiles = 0;
27 static struct wait_queue * lock_queue = NULL;
28
29 static struct swap_info_struct {
30 unsigned long flags;
31 struct inode * swap_file;
32 unsigned int swap_device;
33 unsigned char * swap_map;
34 char * swap_lockmap;
35 int lowest_bit;
36 int highest_bit;
37 } swap_info[MAX_SWAPFILES];
38
39 extern unsigned long free_page_list;
40
41
42
43
44 #define NR_LAST_FREE_PAGES 32
45 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
46
47 #define SWAP_BITS 4096
48
49 #define bitop(name,op) \
50 static inline int name(char * addr,unsigned int nr) \
51 { \
52 int __res; \
53 __asm__ __volatile__("bt" op " %1,%2; adcl $0,%0" \
54 :"=g" (__res) \
55 :"r" (nr),"m" (*(addr)),"0" (0)); \
56 return __res; \
57 }
58
59 bitop(bit,"")
60 bitop(setbit,"s")
61 bitop(clrbit,"r")
62
63 void rw_swap_page(int rw, unsigned int nr, char * buf)
64 {
65 struct swap_info_struct * p;
66
67 if ((nr >> 24) >= nr_swapfiles) {
68 printk("Internal error: bad swap-device\n");
69 return;
70 }
71 p = swap_info + (nr >> 24);
72 nr &= 0x00ffffff;
73 if (nr >= SWAP_BITS) {
74 printk("rw_swap_page: weirdness\n");
75 return;
76 }
77 if (!(p->flags & SWP_USED)) {
78 printk("Trying to swap to unused swap-device\n");
79 return;
80 }
81 while (setbit(p->swap_lockmap,nr))
82 sleep_on(&lock_queue);
83 if (p->swap_device) {
84 ll_rw_page(rw,p->swap_device,nr,buf);
85 } else if (p->swap_file) {
86 unsigned int zones[4];
87 unsigned int block = nr << 2;
88 int i;
89
90 for (i = 0; i < 4; i++)
91 if (!(zones[i] = bmap(p->swap_file,block++))) {
92 printk("rw_swap_page: bad swap file\n");
93 return;
94 }
95 ll_rw_swap_file(rw,p->swap_file->i_dev, zones,4,buf);
96 } else
97 printk("re_swap_page: no swap file or device\n");
98 if (!clrbit(p->swap_lockmap,nr))
99 printk("rw_swap_page: lock already cleared\n");
100 wake_up(&lock_queue);
101 }
102
103 static unsigned int get_swap_page(void)
104 {
105 struct swap_info_struct * p;
106 unsigned int block_nr, swap_nr;
107
108 p = swap_info;
109 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++) {
110 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
111 continue;
112 for (block_nr = p->lowest_bit; block_nr <= p->highest_bit ; block_nr++) {
113 if (p->swap_map[block_nr])
114 continue;
115 p->swap_map[block_nr] = 1;
116 if (block_nr == p->highest_bit)
117 p->highest_bit--;
118 p->lowest_bit = block_nr;
119 return block_nr + (swap_nr << 24);
120 }
121 }
122 return 0;
123 }
124
125 void swap_duplicate(unsigned int nr)
126 {
127 struct swap_info_struct * p;
128
129 if (!nr)
130 return;
131 if ((nr >> 24) >= nr_swapfiles) {
132 printk("Trying to free nonexistent swap-page\n");
133 return;
134 }
135 p = (nr >> 24) + swap_info;
136 nr &= 0x00ffffff;
137 if (nr >= SWAP_BITS) {
138 printk("swap_free: weirness\n");
139 return;
140 }
141 if (!p->swap_map[nr]) {
142 printk("swap_duplicate: trying to duplicate unused page\n");
143 return;
144 }
145 p->swap_map[nr]++;
146 }
147
148 void swap_free(unsigned int nr)
149 {
150 struct swap_info_struct * p;
151
152 if (!nr)
153 return;
154 if ((nr >> 24) >= nr_swapfiles) {
155 printk("Trying to free nonexistent swap-page\n");
156 return;
157 }
158 p = (nr >> 24) + swap_info;
159 nr &= 0x00ffffff;
160 if (nr >= SWAP_BITS) {
161 printk("swap_free: weirness\n");
162 return;
163 }
164 if (!(p->flags & SWP_USED)) {
165 printk("Trying to free swap from unused swap-device\n");
166 return;
167 }
168 while (setbit(p->swap_lockmap,nr))
169 sleep_on(&lock_queue);
170 if (nr < p->lowest_bit)
171 p->lowest_bit = nr;
172 if (nr > p->highest_bit)
173 p->highest_bit = nr;
174 if (!p->swap_map[nr])
175 printk("swap_free: swap-space map bad (page %d)\n",nr);
176 else
177 p->swap_map[nr]--;
178 if (!clrbit(p->swap_lockmap,nr))
179 printk("swap_free: lock already cleared\n");
180 wake_up(&lock_queue);
181 }
182
183 void swap_in(unsigned long *table_ptr)
184 {
185 unsigned long swap_nr;
186 unsigned long page;
187
188 swap_nr = *table_ptr;
189 if (1 & swap_nr) {
190 printk("trying to swap in present page\n\r");
191 return;
192 }
193 if (!swap_nr) {
194 printk("No swap page in swap_in\n\r");
195 return;
196 }
197 page = get_free_page(GFP_KERNEL);
198 if (!page) {
199 oom(current);
200 page = BAD_PAGE;
201 } else
202 read_swap_page(swap_nr>>1, (char *) page);
203 if (*table_ptr != swap_nr) {
204 free_page(page);
205 return;
206 }
207 *table_ptr = page | (PAGE_DIRTY | 7);
208 swap_free(swap_nr>>1);
209 }
210
211 static int try_to_swap_out(unsigned long * table_ptr)
212 {
213 int i;
214 unsigned long page;
215 unsigned long swap_nr;
216
217 page = *table_ptr;
218 if (!(PAGE_PRESENT & page))
219 return 0;
220 if (page >= high_memory) {
221 printk("try_to_swap_out: bad page (%08x)\n",page);
222 *table_ptr = 0;
223 return 0;
224 }
225 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
226 return 0;
227 if (PAGE_ACCESSED & page) {
228 *table_ptr &= ~PAGE_ACCESSED;
229 return 0;
230 }
231 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
232 if (last_free_pages[i] == (page & 0xfffff000))
233 return 0;
234 if (PAGE_DIRTY & page) {
235 page &= 0xfffff000;
236 if (mem_map[MAP_NR(page)] != 1)
237 return 0;
238 if (!(swap_nr = get_swap_page()))
239 return 0;
240 *table_ptr = swap_nr<<1;
241 invalidate();
242 write_swap_page(swap_nr, (char *) page);
243 free_page(page);
244 return 1;
245 }
246 page &= 0xfffff000;
247 *table_ptr = 0;
248 invalidate();
249 free_page(page);
250 return 1 + mem_map[MAP_NR(page)];
251 }
252
253
254
255
256
257
258 int sys_idle(void)
259 {
260 need_resched = 1;
261 return 0;
262 }
263
264
265
266
267
268
269
270
271
272 static int swap_out(unsigned int priority)
273 {
274 static int swap_task = 1;
275 static int swap_table = 0;
276 static int swap_page = 0;
277 int counter = NR_TASKS*8;
278 int pg_table;
279 struct task_struct * p;
280
281 counter >>= priority;
282 check_task:
283 if (counter-- < 0)
284 return 0;
285 if (swap_task >= NR_TASKS) {
286 swap_task = 1;
287 goto check_task;
288 }
289 p = task[swap_task];
290 if (!p || !p->swappable) {
291 swap_task++;
292 goto check_task;
293 }
294 check_dir:
295 if (swap_table >= 1024) {
296 swap_table = 0;
297 swap_task++;
298 goto check_task;
299 }
300 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
301 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
302 swap_table++;
303 goto check_dir;
304 }
305 if (!(1 & pg_table)) {
306 printk("bad page-table at pg_dir[%d]: %08x\n\r",
307 swap_table,pg_table);
308 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
309 swap_table++;
310 goto check_dir;
311 }
312 pg_table &= 0xfffff000;
313 check_table:
314 if (swap_page >= 1024) {
315 swap_page = 0;
316 swap_table++;
317 goto check_dir;
318 }
319 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
320 case 0: break;
321 case 1: p->rss--; return 1;
322 default: p->rss--;
323 }
324 swap_page++;
325 goto check_table;
326 }
327
328 static int try_to_free_page(void)
329 {
330 int i=6;
331
332 while (i--) {
333 if (shrink_buffers(i))
334 return 1;
335 if (swap_out(i))
336 return 1;
337 }
338 return 0;
339 }
340
341
342
343
344
345
346 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
347 {
348 addr &= 0xfffff000;
349 *(unsigned long *) addr = *queue;
350 *queue = addr;
351 }
352
353 void free_page(unsigned long addr)
354 {
355 unsigned long i;
356 unsigned long flag;
357
358 if (addr >= high_memory) {
359 printk("Trying to free nonexistent page %08x\n",addr);
360 return;
361 }
362 i = MAP_NR(addr);
363 if (mem_map[i] & MAP_PAGE_RESERVED)
364 return;
365 __asm__ __volatile__("pushfl ; popl %0 ; cli":"=r" (flag));
366 if (!mem_map[i])
367 goto bad_free_page;
368 if (!--mem_map[i])
369 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
370 add_mem_queue(addr,&secondary_page_list);
371 nr_secondary_pages++;
372 } else {
373 add_mem_queue(addr,&free_page_list);
374 nr_free_pages++;
375 }
376 __asm__ __volatile__("pushl %0 ; popfl"::"r" (flag));
377 return;
378 bad_free_page:
379 __asm__ __volatile__("pushl %0 ; popfl"::"r" (flag));
380 printk("Trying to free free memory (%08x): memory probabably corrupted\n");
381 }
382
383 static unsigned long remove_from_mem_queue(unsigned long * queue)
384 {
385 unsigned long result;
386 static unsigned long index = 0;
387
388 cli();
389 result = *queue;
390 if (!result) {
391 sti();
392 return 0;
393 }
394 if ((result & 0xfff) || result >= high_memory) {
395 *queue = 0;
396 printk("Result = %08x - memory map destroyed\n");
397 sti();
398 panic("mm error");
399 }
400 *queue = *(unsigned long *) result;
401 sti();
402 if (mem_map[MAP_NR(result)]) {
403 printk("Free page %08x has mem_map = %d\n",
404 result,mem_map[MAP_NR(result)]);
405 return 0;
406 }
407 mem_map[MAP_NR(result)] = 1;
408 cli();
409 if (index >= NR_LAST_FREE_PAGES)
410 index = 0;
411 last_free_pages[index] = result;
412 index++;
413 sti();
414 __asm__ __volatile__("cld ; rep ; stosl"
415 ::"a" (0),"c" (1024),"D" (result)
416 :"di","cx");
417 return result;
418 }
419
420
421
422
423
424 unsigned long get_free_page(int priority)
425 {
426 unsigned long result;
427
428
429
430
431
432
433 repeat:
434 result = remove_from_mem_queue(&free_page_list);
435 if (result) {
436 nr_free_pages--;
437 return result;
438 }
439 if (nr_free_pages) {
440 printk("nr_free_pages is %d, but no free memory found\n",nr_free_pages);
441 nr_free_pages = 0;
442 }
443 if (priority == GFP_BUFFER)
444 return 0;
445 if (priority != GFP_ATOMIC)
446 if (try_to_free_page())
447 goto repeat;
448 result = remove_from_mem_queue(&secondary_page_list);
449 if (result) {
450 nr_secondary_pages--;
451 return result;
452 }
453 if (nr_secondary_pages) {
454 printk("nr_secondary_pages is %d, but no free memory found\n",nr_secondary_pages);
455 nr_secondary_pages = 0;
456 }
457 return 0;
458 }
459
460
461
462
463
464
465
466
467
468
469 static int try_to_unuse(unsigned int swap_nr)
470 {
471 int nr, pgt, pg;
472 unsigned long page, *ppage;
473 unsigned long tmp = 0;
474 struct task_struct *p;
475
476 nr = 0;
477
478
479
480
481 repeat:
482 for (; nr < NR_TASKS ; nr++) {
483 p = task[nr];
484 if (!p)
485 continue;
486 for (pgt = 0 ; pgt < 1024 ; pgt++) {
487 ppage = pgt + ((unsigned long *) p->tss.cr3);
488 page = *ppage;
489 if (!page)
490 continue;
491 if (!(page & PAGE_PRESENT) || (page >= high_memory)) {
492 printk("try_to_unuse: bad page directory (%d,%d:%08x)\n",nr,pgt,page);
493 *ppage = 0;
494 continue;
495 }
496 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
497 continue;
498 ppage = (unsigned long *) (page & 0xfffff000);
499 for (pg = 0 ; pg < 1024 ; pg++,ppage++) {
500 page = *ppage;
501 if (!page)
502 continue;
503 if (page & PAGE_PRESENT) {
504 if (page >= high_memory) {
505 printk("try_to_unuse: bad page table (%d,%d,%d:%08x)\n",nr,pgt,pg,page);
506 *ppage = 0;
507 }
508 continue;
509 }
510 if ((page >> 25) != swap_nr)
511 continue;
512 if (!tmp) {
513 tmp = get_free_page(GFP_KERNEL);
514 if (!tmp)
515 return -ENOMEM;
516 goto repeat;
517 }
518 read_swap_page(page>>1, (char *) tmp);
519 if (*ppage == page) {
520 *ppage = tmp | (PAGE_DIRTY | 7);
521 ++p->rss;
522 swap_free(page>>1);
523 tmp = 0;
524 }
525 goto repeat;
526 }
527 }
528 }
529 free_page(tmp);
530 return 0;
531 }
532
533 int sys_swapoff(const char * specialfile)
534 {
535 struct swap_info_struct * p;
536 struct inode * inode;
537 unsigned int swap_nr;
538 int i;
539
540 if (!suser())
541 return -EPERM;
542 i = namei(specialfile,&inode);
543 if (i)
544 return i;
545 p = swap_info;
546 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++) {
547 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
548 continue;
549 if (p->swap_file) {
550 if (p->swap_file == inode)
551 break;
552 } else {
553 if (!S_ISBLK(inode->i_mode))
554 continue;
555 if (p->swap_device == inode->i_rdev)
556 break;
557 }
558 }
559 iput(inode);
560 if (swap_nr >= nr_swapfiles)
561 return -EINVAL;
562 p->flags = SWP_USED;
563 i = try_to_unuse(swap_nr);
564 if (i) {
565 p->flags = SWP_WRITEOK;
566 return i;
567 }
568 iput(p->swap_file);
569 p->swap_file = NULL;
570 p->swap_device = 0;
571 free_page((long) p->swap_map);
572 p->swap_map = NULL;
573 free_page((long) p->swap_lockmap);
574 p->swap_lockmap = NULL;
575 p->flags = 0;
576 return 0;
577 }
578
579
580
581
582
583
584 int sys_swapon(const char * specialfile)
585 {
586 struct swap_info_struct * p;
587 struct inode * swap_inode;
588 unsigned int swap_nr;
589 char * tmp;
590 int i,j;
591
592 if (!suser())
593 return -EPERM;
594 p = swap_info;
595 for (swap_nr = 0 ; swap_nr < nr_swapfiles ; swap_nr++,p++)
596 if (!(p->flags & SWP_USED))
597 break;
598 if (swap_nr >= MAX_SWAPFILES)
599 return -EPERM;
600 if (swap_nr >= nr_swapfiles)
601 nr_swapfiles = swap_nr+1;
602 p->flags = SWP_USED;
603 p->swap_file = NULL;
604 p->swap_device = 0;
605 p->swap_map = NULL;
606 p->swap_lockmap = NULL;
607 p->lowest_bit = 0;
608 p->highest_bit = 0;
609 i = namei(specialfile,&swap_inode);
610 if (i) {
611 p->flags = 0;
612 return i;
613 }
614 if (swap_inode->i_count != 1) {
615 iput(swap_inode);
616 p->flags = 0;
617 return -EBUSY;
618 }
619 if (S_ISBLK(swap_inode->i_mode)) {
620 p->swap_device = swap_inode->i_rdev;
621 iput(swap_inode);
622 if (!p->swap_device) {
623 p->flags = 0;
624 return -ENODEV;
625 }
626 for (i = 0 ; i < nr_swapfiles ; i++) {
627 if (i == swap_nr)
628 continue;
629 if (p->swap_device == swap_info[i].swap_device) {
630 p->swap_device = 0;
631 p->flags = 0;
632 return -EBUSY;
633 }
634 }
635 } else if (S_ISREG(swap_inode->i_mode))
636 p->swap_file = swap_inode;
637 else {
638 iput(swap_inode);
639 p->flags = 0;
640 return -EINVAL;
641 }
642 tmp = (char *) get_free_page(GFP_USER);
643 p->swap_lockmap = (char *) get_free_page(GFP_USER);
644 if (!tmp || !p->swap_lockmap) {
645 printk("Unable to start swapping: out of memory :-)\n");
646 free_page((long) tmp);
647 free_page((long) p->swap_lockmap);
648 iput(p->swap_file);
649 p->swap_device = 0;
650 p->swap_file = NULL;
651 p->swap_map = NULL;
652 p->swap_lockmap = NULL;
653 p->flags = 0;
654 return -ENOMEM;
655 }
656 read_swap_page(swap_nr << 24,tmp);
657 if (strncmp("SWAP-SPACE",tmp+4086,10)) {
658 printk("Unable to find swap-space signature\n\r");
659 free_page((long) tmp);
660 free_page((long) p->swap_lockmap);
661 iput(p->swap_file);
662 p->swap_device = 0;
663 p->swap_file = NULL;
664 p->swap_map = NULL;
665 p->swap_lockmap = NULL;
666 p->flags = 0;
667 return -EINVAL;
668 }
669 memset(tmp+4086,0,10);
670 j = 0;
671 p->lowest_bit = 0;
672 p->highest_bit = 0;
673 for (i = 1 ; i < SWAP_BITS ; i++)
674 if (bit(tmp,i)) {
675 if (!p->lowest_bit)
676 p->lowest_bit = i;
677 p->highest_bit = i;
678 j++;
679 }
680 if (!j) {
681 printk("Empty swap-file\n");
682 free_page((long) tmp);
683 free_page((long) p->swap_lockmap);
684 iput(p->swap_file);
685 p->swap_device = 0;
686 p->swap_file = NULL;
687 p->swap_map = NULL;
688 p->swap_lockmap = NULL;
689 p->flags = 0;
690 return -EINVAL;
691 }
692 i = SWAP_BITS;
693 while (i--)
694 if (bit(tmp,i))
695 tmp[i] = 0;
696 else
697 tmp[i] = 128;
698 tmp[0] = 128;
699 p->swap_map = tmp;
700 p->flags = SWP_WRITEOK;
701 printk("Adding Swap: %dk swap-space\n\r",j<<2);
702 return 0;
703 }