This source file includes following definitions.
- bitop
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- __get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19 #include <asm/system.h>
20
21 #define MAX_SWAPFILES 8
22
23 #define SWP_USED 1
24 #define SWP_WRITEOK 3
25
26 #define SWP_TYPE(entry) (((entry) & 0xffe) >> 1)
27 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
28 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
29
30 static int nr_swapfiles = 0;
31 static struct wait_queue * lock_queue = NULL;
32
33 static struct swap_info_struct {
34 unsigned long flags;
35 struct inode * swap_file;
36 unsigned int swap_device;
37 unsigned char * swap_map;
38 char * swap_lockmap;
39 int lowest_bit;
40 int highest_bit;
41 } swap_info[MAX_SWAPFILES];
42
43 extern unsigned long free_page_list;
44
45
46
47
48
49 #define NR_LAST_FREE_PAGES 32
50 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
51
52 #define SWAP_BITS 4096
53
54 #define bitop(name,op) \
55 static inline int name(char * addr,unsigned int nr) \
56 { \
57 int __res; \
58 __asm__ __volatile__("bt" op " %1,%2; adcl $0,%0" \
59 :"=g" (__res) \
60 :"r" (nr),"m" (*(addr)),"0" (0)); \
61 return __res; \
62 }
63
64 bitop(bit,"")
65 bitop(setbit,"s")
66 bitop(clrbit,"r")
67
68 void rw_swap_page(int rw, unsigned long entry, char * buf)
69 {
70 unsigned long type, offset;
71 struct swap_info_struct * p;
72
73 type = SWP_TYPE(entry);
74 if (type >= nr_swapfiles) {
75 printk("Internal error: bad swap-device\n");
76 return;
77 }
78 p = &swap_info[type];
79 offset = SWP_OFFSET(entry);
80 if (offset >= SWAP_BITS) {
81 printk("rw_swap_page: weirdness\n");
82 return;
83 }
84 if (!(p->flags & SWP_USED)) {
85 printk("Trying to swap to unused swap-device\n");
86 return;
87 }
88 while (setbit(p->swap_lockmap,offset))
89 sleep_on(&lock_queue);
90 if (p->swap_device) {
91 ll_rw_page(rw,p->swap_device,offset,buf);
92 } else if (p->swap_file) {
93 unsigned int zones[4];
94 unsigned int block = offset << 2;
95 int i;
96
97 for (i = 0; i < 4; i++)
98 if (!(zones[i] = bmap(p->swap_file,block++))) {
99 printk("rw_swap_page: bad swap file\n");
100 return;
101 }
102 ll_rw_swap_file(rw,p->swap_file->i_dev, zones,4,buf);
103 } else
104 printk("re_swap_page: no swap file or device\n");
105 if (!clrbit(p->swap_lockmap,offset))
106 printk("rw_swap_page: lock already cleared\n");
107 wake_up(&lock_queue);
108 }
109
110 static unsigned int get_swap_page(void)
111 {
112 struct swap_info_struct * p;
113 unsigned int offset, type;
114
115 p = swap_info;
116 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
117 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
118 continue;
119 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
120 if (p->swap_map[offset])
121 continue;
122 p->swap_map[offset] = 1;
123 if (offset == p->highest_bit)
124 p->highest_bit--;
125 p->lowest_bit = offset;
126 return SWP_ENTRY(type,offset);
127 }
128 }
129 return 0;
130 }
131
132 unsigned long swap_duplicate(unsigned long entry)
133 {
134 struct swap_info_struct * p;
135 unsigned long offset, type;
136
137 if (!entry)
138 return 0;
139 offset = SWP_OFFSET(entry);
140 type = SWP_TYPE(entry);
141 if (type >= nr_swapfiles) {
142 printk("Trying to duplicate nonexistent swap-page\n");
143 return 0;
144 }
145 p = type + swap_info;
146 if (offset >= SWAP_BITS) {
147 printk("swap_free: weirness\n");
148 return 0;
149 }
150 if (!p->swap_map[offset]) {
151 printk("swap_duplicate: trying to duplicate unused page\n");
152 return 0;
153 }
154 p->swap_map[offset]++;
155 return entry;
156 }
157
158 void swap_free(unsigned long entry)
159 {
160 struct swap_info_struct * p;
161 unsigned long offset, type;
162
163 if (!entry)
164 return;
165 type = SWP_TYPE(entry);
166 if (type >= nr_swapfiles) {
167 printk("Trying to free nonexistent swap-page\n");
168 return;
169 }
170 p = & swap_info[type];
171 offset = SWP_OFFSET(entry);
172 if (offset >= SWAP_BITS) {
173 printk("swap_free: weirness\n");
174 return;
175 }
176 if (!(p->flags & SWP_USED)) {
177 printk("Trying to free swap from unused swap-device\n");
178 return;
179 }
180 while (setbit(p->swap_lockmap,offset))
181 sleep_on(&lock_queue);
182 if (offset < p->lowest_bit)
183 p->lowest_bit = offset;
184 if (offset > p->highest_bit)
185 p->highest_bit = offset;
186 if (!p->swap_map[offset])
187 printk("swap_free: swap-space map bad (entry %08x)\n",entry);
188 else
189 p->swap_map[offset]--;
190 if (!clrbit(p->swap_lockmap,offset))
191 printk("swap_free: lock already cleared\n");
192 wake_up(&lock_queue);
193 }
194
195 void swap_in(unsigned long *table_ptr)
196 {
197 unsigned long entry;
198 unsigned long page;
199
200 entry = *table_ptr;
201 if (PAGE_PRESENT & entry) {
202 printk("trying to swap in present page\n");
203 return;
204 }
205 if (!entry) {
206 printk("No swap page in swap_in\n");
207 return;
208 }
209 page = get_free_page(GFP_KERNEL);
210 if (!page) {
211 oom(current);
212 page = BAD_PAGE;
213 } else
214 read_swap_page(entry, (char *) page);
215 if (*table_ptr != entry) {
216 free_page(page);
217 return;
218 }
219 *table_ptr = page | (PAGE_DIRTY | PAGE_PRIVATE);
220 swap_free(entry);
221 }
222
223 static int try_to_swap_out(unsigned long * table_ptr)
224 {
225 int i;
226 unsigned long page;
227 unsigned long entry;
228
229 page = *table_ptr;
230 if (!(PAGE_PRESENT & page))
231 return 0;
232 if (page >= high_memory) {
233 printk("try_to_swap_out: bad page (%08x)\n",page);
234 *table_ptr = 0;
235 return 0;
236 }
237 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
238 return 0;
239 if (PAGE_ACCESSED & page) {
240 *table_ptr &= ~PAGE_ACCESSED;
241 return 0;
242 }
243 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
244 if (last_free_pages[i] == (page & 0xfffff000))
245 return 0;
246 if (PAGE_DIRTY & page) {
247 page &= 0xfffff000;
248 if (mem_map[MAP_NR(page)] != 1)
249 return 0;
250 if (!(entry = get_swap_page()))
251 return 0;
252 *table_ptr = entry;
253 invalidate();
254 write_swap_page(entry, (char *) page);
255 free_page(page);
256 return 1;
257 }
258 page &= 0xfffff000;
259 *table_ptr = 0;
260 invalidate();
261 free_page(page);
262 return 1 + mem_map[MAP_NR(page)];
263 }
264
265
266
267
268
269
270 int sys_idle(void)
271 {
272 need_resched = 1;
273 return 0;
274 }
275
276
277
278
279
280
281
282
283
284 static int swap_out(unsigned int priority)
285 {
286 static int swap_task = 1;
287 static int swap_table = 0;
288 static int swap_page = 0;
289 int counter = NR_TASKS*8;
290 int pg_table;
291 struct task_struct * p;
292
293 counter >>= priority;
294 check_task:
295 if (counter-- < 0)
296 return 0;
297 if (swap_task >= NR_TASKS) {
298 swap_task = 1;
299 goto check_task;
300 }
301 p = task[swap_task];
302 if (!p || !p->swappable) {
303 swap_task++;
304 goto check_task;
305 }
306 check_dir:
307 if (swap_table >= 1024) {
308 swap_table = 0;
309 swap_task++;
310 goto check_task;
311 }
312 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
313 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
314 swap_table++;
315 goto check_dir;
316 }
317 if (!(PAGE_PRESENT & pg_table)) {
318 printk("bad page-table at pg_dir[%d]: %08x\n",
319 swap_table,pg_table);
320 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
321 swap_table++;
322 goto check_dir;
323 }
324 pg_table &= 0xfffff000;
325 check_table:
326 if (swap_page >= 1024) {
327 swap_page = 0;
328 swap_table++;
329 goto check_dir;
330 }
331 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
332 case 0: break;
333 case 1: p->rss--; return 1;
334 default: p->rss--;
335 }
336 swap_page++;
337 goto check_table;
338 }
339
340 static int try_to_free_page(void)
341 {
342 int i=6;
343
344 while (i--) {
345 if (shrink_buffers(i))
346 return 1;
347 if (swap_out(i))
348 return 1;
349 }
350 return 0;
351 }
352
353
354
355
356
357
358 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
359 {
360 addr &= 0xfffff000;
361 *(unsigned long *) addr = *queue;
362 *queue = addr;
363 }
364
365
366
367
368
369
370
371
372
373
374
375
376 void free_page(unsigned long addr)
377 {
378 if (addr < high_memory) {
379 unsigned short * map = mem_map + MAP_NR(addr);
380
381 if (*map) {
382 if (!(*map & MAP_PAGE_RESERVED)) {
383 unsigned long flag;
384
385 save_flags(flag);
386 cli();
387 if (!--*map) {
388 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
389 add_mem_queue(addr,&secondary_page_list);
390 nr_secondary_pages++;
391 restore_flags(flag);
392 return;
393 }
394 add_mem_queue(addr,&free_page_list);
395 nr_free_pages++;
396 }
397 restore_flags(flag);
398 }
399 return;
400 }
401 printk("Trying to free free memory (%08x): memory probabably corrupted\n",addr);
402 return;
403 }
404 printk("Trying to free nonexistent page %08x\n",addr);
405 return;
406 }
407
408
409
410
411
412
413
414
415
416
417
418 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
419 cli(); \
420 if ((result = queue) != 0) { \
421 if (!(result & 0xfff) && result < high_memory) { \
422 queue = *(unsigned long *) result; \
423 if (!mem_map[MAP_NR(result)]) { \
424 mem_map[MAP_NR(result)] = 1; \
425 nr--; \
426 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
427 restore_flags(flag); \
428 return result; \
429 } \
430 printk("Free page %08x has mem_map = %d\n", \
431 result,mem_map[MAP_NR(result)]); \
432 } else \
433 printk("Result = 0x%08x - memory map destroyed\n", result); \
434 queue = 0; \
435 nr = 0; \
436 } else if (nr) { \
437 printk(#nr " is %d, but " #queue " is empty\n",nr); \
438 nr = 0; \
439 } \
440 restore_flags(flag)
441
442
443
444
445
446
447
448
449
450
451 unsigned long __get_free_page(int priority)
452 {
453 unsigned long result, flag;
454 static unsigned long index = 0;
455
456
457
458
459
460
461 save_flags(flag);
462 repeat:
463 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
464 if (priority == GFP_BUFFER)
465 return 0;
466 if (priority != GFP_ATOMIC)
467 if (try_to_free_page())
468 goto repeat;
469 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
470 return 0;
471 }
472
473
474
475
476
477
478 static int try_to_unuse(unsigned int type)
479 {
480 int nr, pgt, pg;
481 unsigned long page, *ppage;
482 unsigned long tmp = 0;
483 struct task_struct *p;
484
485 nr = 0;
486
487
488
489
490 repeat:
491 for (; nr < NR_TASKS ; nr++) {
492 p = task[nr];
493 if (!p)
494 continue;
495 for (pgt = 0 ; pgt < 1024 ; pgt++) {
496 ppage = pgt + ((unsigned long *) p->tss.cr3);
497 page = *ppage;
498 if (!page)
499 continue;
500 if (!(page & PAGE_PRESENT) || (page >= high_memory)) {
501 printk("try_to_unuse: bad page directory (%d,%d:%08x)\n",nr,pgt,page);
502 *ppage = 0;
503 continue;
504 }
505 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
506 continue;
507 ppage = (unsigned long *) (page & 0xfffff000);
508 for (pg = 0 ; pg < 1024 ; pg++,ppage++) {
509 page = *ppage;
510 if (!page)
511 continue;
512 if (page & PAGE_PRESENT) {
513 if (page >= high_memory) {
514 printk("try_to_unuse: bad page table (%d,%d,%d:%08x)\n",nr,pgt,pg,page);
515 *ppage = 0;
516 }
517 continue;
518 }
519 if (SWP_TYPE(page) != type)
520 continue;
521 if (!tmp) {
522 tmp = get_free_page(GFP_KERNEL);
523 if (!tmp)
524 return -ENOMEM;
525 goto repeat;
526 }
527 read_swap_page(page, (char *) tmp);
528 if (*ppage == page) {
529 *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
530 ++p->rss;
531 swap_free(page);
532 tmp = 0;
533 }
534 goto repeat;
535 }
536 }
537 }
538 free_page(tmp);
539 return 0;
540 }
541
542 int sys_swapoff(const char * specialfile)
543 {
544 struct swap_info_struct * p;
545 struct inode * inode;
546 unsigned int type;
547 int i;
548
549 if (!suser())
550 return -EPERM;
551 i = namei(specialfile,&inode);
552 if (i)
553 return i;
554 p = swap_info;
555 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
556 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
557 continue;
558 if (p->swap_file) {
559 if (p->swap_file == inode)
560 break;
561 } else {
562 if (!S_ISBLK(inode->i_mode))
563 continue;
564 if (p->swap_device == inode->i_rdev)
565 break;
566 }
567 }
568 iput(inode);
569 if (type >= nr_swapfiles)
570 return -EINVAL;
571 p->flags = SWP_USED;
572 i = try_to_unuse(type);
573 if (i) {
574 p->flags = SWP_WRITEOK;
575 return i;
576 }
577 iput(p->swap_file);
578 p->swap_file = NULL;
579 p->swap_device = 0;
580 free_page((long) p->swap_map);
581 p->swap_map = NULL;
582 free_page((long) p->swap_lockmap);
583 p->swap_lockmap = NULL;
584 p->flags = 0;
585 return 0;
586 }
587
588
589
590
591
592
593 int sys_swapon(const char * specialfile)
594 {
595 struct swap_info_struct * p;
596 struct inode * swap_inode;
597 unsigned int type;
598 char * tmp;
599 int i,j;
600
601 if (!suser())
602 return -EPERM;
603 p = swap_info;
604 for (type = 0 ; type < nr_swapfiles ; type++,p++)
605 if (!(p->flags & SWP_USED))
606 break;
607 if (type >= MAX_SWAPFILES)
608 return -EPERM;
609 if (type >= nr_swapfiles)
610 nr_swapfiles = type+1;
611 p->flags = SWP_USED;
612 p->swap_file = NULL;
613 p->swap_device = 0;
614 p->swap_map = NULL;
615 p->swap_lockmap = NULL;
616 p->lowest_bit = 0;
617 p->highest_bit = 0;
618 i = namei(specialfile,&swap_inode);
619 if (i) {
620 p->flags = 0;
621 return i;
622 }
623 if (swap_inode->i_count != 1) {
624 iput(swap_inode);
625 p->flags = 0;
626 return -EBUSY;
627 }
628 if (S_ISBLK(swap_inode->i_mode)) {
629 p->swap_device = swap_inode->i_rdev;
630 iput(swap_inode);
631 if (!p->swap_device) {
632 p->flags = 0;
633 return -ENODEV;
634 }
635 for (i = 0 ; i < nr_swapfiles ; i++) {
636 if (i == type)
637 continue;
638 if (p->swap_device == swap_info[i].swap_device) {
639 p->swap_device = 0;
640 p->flags = 0;
641 return -EBUSY;
642 }
643 }
644 } else if (S_ISREG(swap_inode->i_mode))
645 p->swap_file = swap_inode;
646 else {
647 iput(swap_inode);
648 p->flags = 0;
649 return -EINVAL;
650 }
651 tmp = (char *) get_free_page(GFP_USER);
652 p->swap_lockmap = (char *) get_free_page(GFP_USER);
653 if (!tmp || !p->swap_lockmap) {
654 printk("Unable to start swapping: out of memory :-)\n");
655 free_page((long) tmp);
656 free_page((long) p->swap_lockmap);
657 iput(p->swap_file);
658 p->swap_device = 0;
659 p->swap_file = NULL;
660 p->swap_map = NULL;
661 p->swap_lockmap = NULL;
662 p->flags = 0;
663 return -ENOMEM;
664 }
665 read_swap_page(SWP_ENTRY(type,0),tmp);
666 if (strncmp("SWAP-SPACE",tmp+4086,10)) {
667 printk("Unable to find swap-space signature\n");
668 free_page((long) tmp);
669 free_page((long) p->swap_lockmap);
670 iput(p->swap_file);
671 p->swap_device = 0;
672 p->swap_file = NULL;
673 p->swap_map = NULL;
674 p->swap_lockmap = NULL;
675 p->flags = 0;
676 return -EINVAL;
677 }
678 memset(tmp+4086,0,10);
679 j = 0;
680 p->lowest_bit = 0;
681 p->highest_bit = 0;
682 for (i = 1 ; i < SWAP_BITS ; i++)
683 if (bit(tmp,i)) {
684 if (!p->lowest_bit)
685 p->lowest_bit = i;
686 p->highest_bit = i;
687 j++;
688 }
689 if (!j) {
690 printk("Empty swap-file\n");
691 free_page((long) tmp);
692 free_page((long) p->swap_lockmap);
693 iput(p->swap_file);
694 p->swap_device = 0;
695 p->swap_file = NULL;
696 p->swap_map = NULL;
697 p->swap_lockmap = NULL;
698 p->flags = 0;
699 return -EINVAL;
700 }
701 i = SWAP_BITS;
702 while (i--)
703 if (bit(tmp,i))
704 tmp[i] = 0;
705 else
706 tmp[i] = 128;
707 tmp[0] = 128;
708 p->swap_map = tmp;
709 p->flags = SWP_WRITEOK;
710 printk("Adding Swap: %dk swap-space\n",j<<2);
711 return 0;
712 }
713
714 void si_swapinfo(struct sysinfo *val)
715 {
716 unsigned int i, j;
717
718 val->freeswap = val->totalswap = 0;
719 for (i = 0; i < nr_swapfiles; i++) {
720 if (!(swap_info[i].flags & SWP_USED))
721 continue;
722 for (j = 0; j < 4096; ++j)
723 switch (swap_info[i].swap_map[j]) {
724 case 128:
725 continue;
726 case 0:
727 ++val->freeswap;
728 default:
729 ++val->totalswap;
730 }
731 }
732 val->freeswap <<= PAGE_SHIFT;
733 val->totalswap <<= PAGE_SHIFT;
734 return;
735 }