This source file includes following definitions.
- rw_swap_page
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- __get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19
20 #include <asm/system.h>
21 #include <asm/bitops.h>
22
23 #define MAX_SWAPFILES 8
24
25 #define SWP_USED 1
26 #define SWP_WRITEOK 3
27
28 #define SWP_TYPE(entry) (((entry) & 0xfe) >> 1)
29 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
30 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
31
32 static int nr_swapfiles = 0;
33 static struct wait_queue * lock_queue = NULL;
34
35 static struct swap_info_struct {
36 unsigned long flags;
37 struct inode * swap_file;
38 unsigned int swap_device;
39 unsigned char * swap_map;
40 char * swap_lockmap;
41 int pages;
42 int lowest_bit;
43 int highest_bit;
44 } swap_info[MAX_SWAPFILES];
45
46 extern unsigned long free_page_list;
47 extern int shm_swap (int);
48
49
50
51
52
53 #define NR_LAST_FREE_PAGES 32
54 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
55
56 #define SWAP_BITS 4096
57
58 void rw_swap_page(int rw, unsigned long entry, char * buf)
59 {
60 unsigned long type, offset;
61 struct swap_info_struct * p;
62
63 type = SWP_TYPE(entry);
64 if (type >= nr_swapfiles) {
65 printk("Internal error: bad swap-device\n");
66 return;
67 }
68 p = &swap_info[type];
69 offset = SWP_OFFSET(entry);
70 if (offset >= SWAP_BITS) {
71 printk("rw_swap_page: weirdness\n");
72 return;
73 }
74 if (!(p->flags & SWP_USED)) {
75 printk("Trying to swap to unused swap-device\n");
76 return;
77 }
78 while (set_bit(offset,p->swap_lockmap))
79 sleep_on(&lock_queue);
80 if (p->swap_device) {
81 ll_rw_page(rw,p->swap_device,offset,buf);
82 } else if (p->swap_file) {
83 unsigned int zones[8];
84 unsigned int block;
85 int i, j;
86
87 block = offset << (12 - p->swap_file->i_sb->s_blocksize_bits);
88
89 for (i=0, j=0; j< PAGE_SIZE ; i++, j +=p->swap_file->i_sb->s_blocksize)
90 if (!(zones[i] = bmap(p->swap_file,block++))) {
91 printk("rw_swap_page: bad swap file\n");
92 return;
93 }
94 ll_rw_swap_file(rw,p->swap_file->i_dev, zones, i,buf);
95 } else
96 printk("re_swap_page: no swap file or device\n");
97 if (!clear_bit(offset,p->swap_lockmap))
98 printk("rw_swap_page: lock already cleared\n");
99 wake_up(&lock_queue);
100 }
101
102 unsigned int get_swap_page(void)
103 {
104 struct swap_info_struct * p;
105 unsigned int offset, type;
106
107 p = swap_info;
108 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
109 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
110 continue;
111 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
112 if (p->swap_map[offset])
113 continue;
114 p->swap_map[offset] = 1;
115 nr_swap_pages--;
116 if (offset == p->highest_bit)
117 p->highest_bit--;
118 p->lowest_bit = offset;
119 return SWP_ENTRY(type,offset);
120 }
121 }
122 return 0;
123 }
124
125 unsigned long swap_duplicate(unsigned long entry)
126 {
127 struct swap_info_struct * p;
128 unsigned long offset, type;
129
130 if (!entry)
131 return 0;
132 offset = SWP_OFFSET(entry);
133 type = SWP_TYPE(entry);
134 if (type == SHM_SWP_TYPE)
135 return entry;
136 if (type >= nr_swapfiles) {
137 printk("Trying to duplicate nonexistent swap-page\n");
138 return 0;
139 }
140 p = type + swap_info;
141 if (offset >= SWAP_BITS) {
142 printk("swap_free: weirness\n");
143 return 0;
144 }
145 if (!p->swap_map[offset]) {
146 printk("swap_duplicate: trying to duplicate unused page\n");
147 return 0;
148 }
149 p->swap_map[offset]++;
150 return entry;
151 }
152
153 void swap_free(unsigned long entry)
154 {
155 struct swap_info_struct * p;
156 unsigned long offset, type;
157
158 if (!entry)
159 return;
160 type = SWP_TYPE(entry);
161 if (type == SHM_SWP_TYPE)
162 return;
163 if (type >= nr_swapfiles) {
164 printk("Trying to free nonexistent swap-page\n");
165 return;
166 }
167 p = & swap_info[type];
168 offset = SWP_OFFSET(entry);
169 if (offset >= SWAP_BITS) {
170 printk("swap_free: weirness\n");
171 return;
172 }
173 if (!(p->flags & SWP_USED)) {
174 printk("Trying to free swap from unused swap-device\n");
175 return;
176 }
177 while (set_bit(offset,p->swap_lockmap))
178 sleep_on(&lock_queue);
179 if (offset < p->lowest_bit)
180 p->lowest_bit = offset;
181 if (offset > p->highest_bit)
182 p->highest_bit = offset;
183 if (!p->swap_map[offset])
184 printk("swap_free: swap-space map bad (entry %08x)\n",entry);
185 else
186 if (!--p->swap_map[offset])
187 nr_swap_pages++;
188 if (!clear_bit(offset,p->swap_lockmap))
189 printk("swap_free: lock already cleared\n");
190 wake_up(&lock_queue);
191 }
192
193 void swap_in(unsigned long *table_ptr)
194 {
195 unsigned long entry;
196 unsigned long page;
197
198 entry = *table_ptr;
199 if (PAGE_PRESENT & entry) {
200 printk("trying to swap in present page\n");
201 return;
202 }
203 if (!entry) {
204 printk("No swap page in swap_in\n");
205 return;
206 }
207 if (SWP_TYPE(entry) == SHM_SWP_TYPE) {
208 shm_no_page ((unsigned long *) table_ptr);
209 return;
210 }
211 page = get_free_page(GFP_KERNEL);
212 if (!page) {
213 oom(current);
214 page = BAD_PAGE;
215 } else
216 read_swap_page(entry, (char *) page);
217 if (*table_ptr != entry) {
218 free_page(page);
219 return;
220 }
221 *table_ptr = page | (PAGE_DIRTY | PAGE_PRIVATE);
222 swap_free(entry);
223 }
224
225 static int try_to_swap_out(unsigned long * table_ptr)
226 {
227 int i;
228 unsigned long page;
229 unsigned long entry;
230
231 page = *table_ptr;
232 if (!(PAGE_PRESENT & page))
233 return 0;
234 if (page >= high_memory)
235 return 0;
236 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
237 return 0;
238 if (PAGE_ACCESSED & page) {
239 *table_ptr &= ~PAGE_ACCESSED;
240 return 0;
241 }
242 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
243 if (last_free_pages[i] == (page & 0xfffff000))
244 return 0;
245 if (PAGE_DIRTY & page) {
246 page &= 0xfffff000;
247 if (mem_map[MAP_NR(page)] != 1)
248 return 0;
249 if (!(entry = get_swap_page()))
250 return 0;
251 *table_ptr = entry;
252 invalidate();
253 write_swap_page(entry, (char *) page);
254 free_page(page);
255 return 1;
256 }
257 page &= 0xfffff000;
258 *table_ptr = 0;
259 invalidate();
260 free_page(page);
261 return 1 + mem_map[MAP_NR(page)];
262 }
263
264
265
266
267
268
269 extern "C" int sys_idle(void)
270 {
271 need_resched = 1;
272 return 0;
273 }
274
275
276
277
278
279
280
281
282
283 static int swap_out(unsigned int priority)
284 {
285 static int swap_task = 1;
286 static int swap_table = 0;
287 static int swap_page = 0;
288 int counter = NR_TASKS*8;
289 int pg_table;
290 struct task_struct * p;
291
292 counter >>= priority;
293 check_task:
294 if (counter-- < 0)
295 return 0;
296 if (swap_task >= NR_TASKS) {
297 swap_task = 1;
298 goto check_task;
299 }
300 p = task[swap_task];
301 if (!p || !p->swappable) {
302 swap_task++;
303 goto check_task;
304 }
305 check_dir:
306 if (swap_table >= 1024) {
307 swap_table = 0;
308 swap_task++;
309 goto check_task;
310 }
311 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
312 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
313 swap_table++;
314 goto check_dir;
315 }
316 if (!(PAGE_PRESENT & pg_table)) {
317 printk("bad page-table at pg_dir[%d]: %08x\n",
318 swap_table,pg_table);
319 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
320 swap_table++;
321 goto check_dir;
322 }
323 pg_table &= 0xfffff000;
324 check_table:
325 if (swap_page >= 1024) {
326 swap_page = 0;
327 swap_table++;
328 goto check_dir;
329 }
330 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
331 case 0: break;
332 case 1: p->rss--; return 1;
333 default: p->rss--;
334 }
335 swap_page++;
336 goto check_table;
337 }
338
339 static int try_to_free_page(void)
340 {
341 int i=6;
342
343 while (i--) {
344 if (shrink_buffers(i))
345 return 1;
346 if (shm_swap(i))
347 return 1;
348 if (swap_out(i))
349 return 1;
350 }
351 return 0;
352 }
353
354
355
356
357
358
359 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
360 {
361 addr &= 0xfffff000;
362 *(unsigned long *) addr = *queue;
363 *queue = addr;
364 }
365
366
367
368
369
370
371
372
373
374
375
376
377 void free_page(unsigned long addr)
378 {
379 if (addr < high_memory) {
380 unsigned short * map = mem_map + MAP_NR(addr);
381
382 if (*map) {
383 if (!(*map & MAP_PAGE_RESERVED)) {
384 unsigned long flag;
385
386 save_flags(flag);
387 cli();
388 if (!--*map) {
389 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
390 add_mem_queue(addr,&secondary_page_list);
391 nr_secondary_pages++;
392 restore_flags(flag);
393 return;
394 }
395 add_mem_queue(addr,&free_page_list);
396 nr_free_pages++;
397 }
398 restore_flags(flag);
399 }
400 return;
401 }
402 printk("Trying to free free memory (%08x): memory probabably corrupted\n",addr);
403 printk("PC = %08x\n",*(((unsigned long *)&addr)-1));
404 return;
405 }
406 }
407
408
409
410
411
412
413
414
415
416
417
418 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
419 cli(); \
420 if ((result = queue) != 0) { \
421 if (!(result & 0xfff) && result < high_memory) { \
422 queue = *(unsigned long *) result; \
423 if (!mem_map[MAP_NR(result)]) { \
424 mem_map[MAP_NR(result)] = 1; \
425 nr--; \
426 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
427 restore_flags(flag); \
428 return result; \
429 } \
430 printk("Free page %08x has mem_map = %d\n", \
431 result,mem_map[MAP_NR(result)]); \
432 } else \
433 printk("Result = 0x%08x - memory map destroyed\n", result); \
434 queue = 0; \
435 nr = 0; \
436 } else if (nr) { \
437 printk(#nr " is %d, but " #queue " is empty\n",nr); \
438 nr = 0; \
439 } \
440 restore_flags(flag)
441
442
443
444
445
446
447
448
449
450
451 unsigned long __get_free_page(int priority)
452 {
453 unsigned long result, flag;
454 static unsigned long index = 0;
455
456
457
458
459
460
461 save_flags(flag);
462 repeat:
463 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
464 if (priority == GFP_BUFFER)
465 return 0;
466 if (priority != GFP_ATOMIC)
467 if (try_to_free_page())
468 goto repeat;
469 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
470 return 0;
471 }
472
473
474
475
476
477
478 static int try_to_unuse(unsigned int type)
479 {
480 int nr, pgt, pg;
481 unsigned long page, *ppage;
482 unsigned long tmp = 0;
483 struct task_struct *p;
484
485 nr = 0;
486
487
488
489
490 repeat:
491 for (; nr < NR_TASKS ; nr++) {
492 p = task[nr];
493 if (!p)
494 continue;
495 for (pgt = 0 ; pgt < 1024 ; pgt++) {
496 ppage = pgt + ((unsigned long *) p->tss.cr3);
497 page = *ppage;
498 if (!page)
499 continue;
500 if (!(page & PAGE_PRESENT) || (page >= high_memory))
501 continue;
502 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
503 continue;
504 ppage = (unsigned long *) (page & 0xfffff000);
505 for (pg = 0 ; pg < 1024 ; pg++,ppage++) {
506 page = *ppage;
507 if (!page)
508 continue;
509 if (page & PAGE_PRESENT)
510 continue;
511 if (SWP_TYPE(page) != type)
512 continue;
513 if (!tmp) {
514 tmp = get_free_page(GFP_KERNEL);
515 if (!tmp)
516 return -ENOMEM;
517 goto repeat;
518 }
519 read_swap_page(page, (char *) tmp);
520 if (*ppage == page) {
521 *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
522 ++p->rss;
523 swap_free(page);
524 tmp = 0;
525 }
526 goto repeat;
527 }
528 }
529 }
530 free_page(tmp);
531 return 0;
532 }
533
534 extern "C" int sys_swapoff(const char * specialfile)
535 {
536 struct swap_info_struct * p;
537 struct inode * inode;
538 unsigned int type;
539 int i;
540
541 if (!suser())
542 return -EPERM;
543 i = namei(specialfile,&inode);
544 if (i)
545 return i;
546 p = swap_info;
547 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
548 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
549 continue;
550 if (p->swap_file) {
551 if (p->swap_file == inode)
552 break;
553 } else {
554 if (!S_ISBLK(inode->i_mode))
555 continue;
556 if (p->swap_device == inode->i_rdev)
557 break;
558 }
559 }
560 iput(inode);
561 if (type >= nr_swapfiles)
562 return -EINVAL;
563 p->flags = SWP_USED;
564 i = try_to_unuse(type);
565 if (i) {
566 p->flags = SWP_WRITEOK;
567 return i;
568 }
569 nr_swap_pages -= p->pages;
570 iput(p->swap_file);
571 p->swap_file = NULL;
572 p->swap_device = 0;
573 free_page((long) p->swap_map);
574 p->swap_map = NULL;
575 free_page((long) p->swap_lockmap);
576 p->swap_lockmap = NULL;
577 p->flags = 0;
578 return 0;
579 }
580
581
582
583
584
585
586 extern "C" int sys_swapon(const char * specialfile)
587 {
588 struct swap_info_struct * p;
589 struct inode * swap_inode;
590 unsigned int type;
591 char * tmp;
592 int i,j;
593
594 if (!suser())
595 return -EPERM;
596 p = swap_info;
597 for (type = 0 ; type < nr_swapfiles ; type++,p++)
598 if (!(p->flags & SWP_USED))
599 break;
600 if (type >= MAX_SWAPFILES)
601 return -EPERM;
602 if (type >= nr_swapfiles)
603 nr_swapfiles = type+1;
604 p->flags = SWP_USED;
605 p->swap_file = NULL;
606 p->swap_device = 0;
607 p->swap_map = NULL;
608 p->swap_lockmap = NULL;
609 p->lowest_bit = 0;
610 p->highest_bit = 0;
611 i = namei(specialfile,&swap_inode);
612 if (i) {
613 p->flags = 0;
614 return i;
615 }
616 if (swap_inode->i_count != 1) {
617 iput(swap_inode);
618 p->flags = 0;
619 return -EBUSY;
620 }
621 if (S_ISBLK(swap_inode->i_mode)) {
622 p->swap_device = swap_inode->i_rdev;
623 iput(swap_inode);
624 if (!p->swap_device) {
625 p->flags = 0;
626 return -ENODEV;
627 }
628 for (i = 0 ; i < nr_swapfiles ; i++) {
629 if (i == type)
630 continue;
631 if (p->swap_device == swap_info[i].swap_device) {
632 p->swap_device = 0;
633 p->flags = 0;
634 return -EBUSY;
635 }
636 }
637 } else if (S_ISREG(swap_inode->i_mode))
638 p->swap_file = swap_inode;
639 else {
640 iput(swap_inode);
641 p->flags = 0;
642 return -EINVAL;
643 }
644 tmp = (char *) get_free_page(GFP_USER);
645 p->swap_lockmap = (char *) get_free_page(GFP_USER);
646 if (!tmp || !p->swap_lockmap) {
647 printk("Unable to start swapping: out of memory :-)\n");
648 free_page((long) tmp);
649 free_page((long) p->swap_lockmap);
650 iput(p->swap_file);
651 p->swap_device = 0;
652 p->swap_file = NULL;
653 p->swap_map = NULL;
654 p->swap_lockmap = NULL;
655 p->flags = 0;
656 return -ENOMEM;
657 }
658 read_swap_page(SWP_ENTRY(type,0),tmp);
659 if (strncmp("SWAP-SPACE",tmp+4086,10)) {
660 printk("Unable to find swap-space signature\n");
661 free_page((long) tmp);
662 free_page((long) p->swap_lockmap);
663 iput(p->swap_file);
664 p->swap_device = 0;
665 p->swap_file = NULL;
666 p->swap_map = NULL;
667 p->swap_lockmap = NULL;
668 p->flags = 0;
669 return -EINVAL;
670 }
671 memset(tmp+4086,0,10);
672 j = 0;
673 p->lowest_bit = 0;
674 p->highest_bit = 0;
675 for (i = 1 ; i < SWAP_BITS ; i++)
676 if (test_bit(i,tmp)) {
677 if (!p->lowest_bit)
678 p->lowest_bit = i;
679 p->highest_bit = i;
680 j++;
681 }
682 if (!j) {
683 printk("Empty swap-file\n");
684 free_page((long) tmp);
685 free_page((long) p->swap_lockmap);
686 iput(p->swap_file);
687 p->swap_device = 0;
688 p->swap_file = NULL;
689 p->swap_map = NULL;
690 p->swap_lockmap = NULL;
691 p->flags = 0;
692 return -EINVAL;
693 }
694 i = SWAP_BITS;
695 while (i--)
696 if (test_bit(i,tmp))
697 tmp[i] = 0;
698 else
699 tmp[i] = 128;
700 tmp[0] = 128;
701 p->swap_map = (unsigned char *) tmp;
702 p->flags = SWP_WRITEOK;
703 p->pages = j;
704 nr_swap_pages += j;
705 printk("Adding Swap: %dk swap-space\n",j<<2);
706 return 0;
707 }
708
709 void si_swapinfo(struct sysinfo *val)
710 {
711 unsigned int i, j;
712
713 val->freeswap = val->totalswap = 0;
714 for (i = 0; i < nr_swapfiles; i++) {
715 if (!(swap_info[i].flags & SWP_USED))
716 continue;
717 for (j = 0; j < 4096; ++j)
718 switch (swap_info[i].swap_map[j]) {
719 case 128:
720 continue;
721 case 0:
722 ++val->freeswap;
723 default:
724 ++val->totalswap;
725 }
726 }
727 val->freeswap <<= PAGE_SHIFT;
728 val->totalswap <<= PAGE_SHIFT;
729 return;
730 }