This source file includes following definitions.
- rw_swap_page
- get_swap_page
- swap_duplicate
- swap_free
- swap_in
- try_to_swap_out
- sys_idle
- swap_out
- try_to_free_page
- add_mem_queue
- free_page
- __get_free_page
- try_to_unuse
- sys_swapoff
- sys_swapon
- si_swapinfo
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/mm.h>
13 #include <linux/sched.h>
14 #include <linux/head.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/stat.h>
19
20 #include <asm/system.h>
21 #include <asm/bitops.h>
22
23 #define MAX_SWAPFILES 8
24
25 #define SWP_USED 1
26 #define SWP_WRITEOK 3
27
28 #define SWP_TYPE(entry) (((entry) & 0xfe) >> 1)
29 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
30 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
31
32 static int nr_swapfiles = 0;
33 static struct wait_queue * lock_queue = NULL;
34
35 static struct swap_info_struct {
36 unsigned long flags;
37 struct inode * swap_file;
38 unsigned int swap_device;
39 unsigned char * swap_map;
40 char * swap_lockmap;
41 int lowest_bit;
42 int highest_bit;
43 } swap_info[MAX_SWAPFILES];
44
45 extern unsigned long free_page_list;
46 extern int shm_swap (int);
47
48
49
50
51
52 #define NR_LAST_FREE_PAGES 32
53 static unsigned long last_free_pages[NR_LAST_FREE_PAGES] = {0,};
54
55 #define SWAP_BITS 4096
56
57 void rw_swap_page(int rw, unsigned long entry, char * buf)
58 {
59 unsigned long type, offset;
60 struct swap_info_struct * p;
61
62 type = SWP_TYPE(entry);
63 if (type >= nr_swapfiles) {
64 printk("Internal error: bad swap-device\n");
65 return;
66 }
67 p = &swap_info[type];
68 offset = SWP_OFFSET(entry);
69 if (offset >= SWAP_BITS) {
70 printk("rw_swap_page: weirdness\n");
71 return;
72 }
73 if (!(p->flags & SWP_USED)) {
74 printk("Trying to swap to unused swap-device\n");
75 return;
76 }
77 while (set_bit(offset,p->swap_lockmap))
78 sleep_on(&lock_queue);
79 if (p->swap_device) {
80 ll_rw_page(rw,p->swap_device,offset,buf);
81 } else if (p->swap_file) {
82 unsigned int zones[4];
83 unsigned int block = offset << 2;
84 int i;
85
86 for (i = 0; i < 4; i++)
87 if (!(zones[i] = bmap(p->swap_file,block++))) {
88 printk("rw_swap_page: bad swap file\n");
89 return;
90 }
91 ll_rw_swap_file(rw,p->swap_file->i_dev, zones,4,buf);
92 } else
93 printk("re_swap_page: no swap file or device\n");
94 if (clear_bit(offset,p->swap_lockmap))
95 printk("rw_swap_page: lock already cleared\n");
96 wake_up(&lock_queue);
97 }
98
99 unsigned int get_swap_page(void)
100 {
101 struct swap_info_struct * p;
102 unsigned int offset, type;
103
104 p = swap_info;
105 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
106 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
107 continue;
108 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
109 if (p->swap_map[offset])
110 continue;
111 p->swap_map[offset] = 1;
112 if (offset == p->highest_bit)
113 p->highest_bit--;
114 p->lowest_bit = offset;
115 return SWP_ENTRY(type,offset);
116 }
117 }
118 return 0;
119 }
120
121 unsigned long swap_duplicate(unsigned long entry)
122 {
123 struct swap_info_struct * p;
124 unsigned long offset, type;
125
126 if (!entry)
127 return 0;
128 offset = SWP_OFFSET(entry);
129 type = SWP_TYPE(entry);
130 if (type == SHM_SWP_TYPE)
131 return entry;
132 if (type >= nr_swapfiles) {
133 printk("Trying to duplicate nonexistent swap-page\n");
134 return 0;
135 }
136 p = type + swap_info;
137 if (offset >= SWAP_BITS) {
138 printk("swap_free: weirness\n");
139 return 0;
140 }
141 if (!p->swap_map[offset]) {
142 printk("swap_duplicate: trying to duplicate unused page\n");
143 return 0;
144 }
145 p->swap_map[offset]++;
146 return entry;
147 }
148
149 void swap_free(unsigned long entry)
150 {
151 struct swap_info_struct * p;
152 unsigned long offset, type;
153
154 if (!entry)
155 return;
156 type = SWP_TYPE(entry);
157 if (type == SHM_SWP_TYPE)
158 return;
159 if (type >= nr_swapfiles) {
160 printk("Trying to free nonexistent swap-page\n");
161 return;
162 }
163 p = & swap_info[type];
164 offset = SWP_OFFSET(entry);
165 if (offset >= SWAP_BITS) {
166 printk("swap_free: weirness\n");
167 return;
168 }
169 if (!(p->flags & SWP_USED)) {
170 printk("Trying to free swap from unused swap-device\n");
171 return;
172 }
173 while (set_bit(offset,p->swap_lockmap))
174 sleep_on(&lock_queue);
175 if (offset < p->lowest_bit)
176 p->lowest_bit = offset;
177 if (offset > p->highest_bit)
178 p->highest_bit = offset;
179 if (!p->swap_map[offset])
180 printk("swap_free: swap-space map bad (entry %08x)\n",entry);
181 else
182 p->swap_map[offset]--;
183 if (clear_bit(offset,p->swap_lockmap))
184 printk("swap_free: lock already cleared\n");
185 wake_up(&lock_queue);
186 }
187
188 void swap_in(unsigned long *table_ptr)
189 {
190 unsigned long entry;
191 unsigned long page;
192
193 entry = *table_ptr;
194 if (PAGE_PRESENT & entry) {
195 printk("trying to swap in present page\n");
196 return;
197 }
198 if (!entry) {
199 printk("No swap page in swap_in\n");
200 return;
201 }
202 if (SWP_TYPE(entry) == SHM_SWP_TYPE)
203 return shm_no_page ((unsigned long *) table_ptr);
204 page = get_free_page(GFP_KERNEL);
205 if (!page) {
206 oom(current);
207 page = BAD_PAGE;
208 } else
209 read_swap_page(entry, (char *) page);
210 if (*table_ptr != entry) {
211 free_page(page);
212 return;
213 }
214 *table_ptr = page | (PAGE_DIRTY | PAGE_PRIVATE);
215 swap_free(entry);
216 }
217
218 static int try_to_swap_out(unsigned long * table_ptr)
219 {
220 int i;
221 unsigned long page;
222 unsigned long entry;
223
224 page = *table_ptr;
225 if (!(PAGE_PRESENT & page))
226 return 0;
227 if (page >= high_memory) {
228 printk("try_to_swap_out: bad page (%08x)\n",page);
229 *table_ptr = 0;
230 return 0;
231 }
232 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
233 return 0;
234 if (PAGE_ACCESSED & page) {
235 *table_ptr &= ~PAGE_ACCESSED;
236 return 0;
237 }
238 for (i = 0; i < NR_LAST_FREE_PAGES; i++)
239 if (last_free_pages[i] == (page & 0xfffff000))
240 return 0;
241 if (PAGE_DIRTY & page) {
242 page &= 0xfffff000;
243 if (mem_map[MAP_NR(page)] != 1)
244 return 0;
245 if (!(entry = get_swap_page()))
246 return 0;
247 *table_ptr = entry;
248 invalidate();
249 write_swap_page(entry, (char *) page);
250 free_page(page);
251 return 1;
252 }
253 page &= 0xfffff000;
254 *table_ptr = 0;
255 invalidate();
256 free_page(page);
257 return 1 + mem_map[MAP_NR(page)];
258 }
259
260
261
262
263
264
265 int sys_idle(void)
266 {
267 need_resched = 1;
268 return 0;
269 }
270
271
272
273
274
275
276
277
278
279 static int swap_out(unsigned int priority)
280 {
281 static int swap_task = 1;
282 static int swap_table = 0;
283 static int swap_page = 0;
284 int counter = NR_TASKS*8;
285 int pg_table;
286 struct task_struct * p;
287
288 counter >>= priority;
289 check_task:
290 if (counter-- < 0)
291 return 0;
292 if (swap_task >= NR_TASKS) {
293 swap_task = 1;
294 goto check_task;
295 }
296 p = task[swap_task];
297 if (!p || !p->swappable) {
298 swap_task++;
299 goto check_task;
300 }
301 check_dir:
302 if (swap_table >= 1024) {
303 swap_table = 0;
304 swap_task++;
305 goto check_task;
306 }
307 pg_table = ((unsigned long *) p->tss.cr3)[swap_table];
308 if (pg_table >= high_memory || (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)) {
309 swap_table++;
310 goto check_dir;
311 }
312 if (!(PAGE_PRESENT & pg_table)) {
313 printk("bad page-table at pg_dir[%d]: %08x\n",
314 swap_table,pg_table);
315 ((unsigned long *) p->tss.cr3)[swap_table] = 0;
316 swap_table++;
317 goto check_dir;
318 }
319 pg_table &= 0xfffff000;
320 check_table:
321 if (swap_page >= 1024) {
322 swap_page = 0;
323 swap_table++;
324 goto check_dir;
325 }
326 switch (try_to_swap_out(swap_page + (unsigned long *) pg_table)) {
327 case 0: break;
328 case 1: p->rss--; return 1;
329 default: p->rss--;
330 }
331 swap_page++;
332 goto check_table;
333 }
334
335 static int try_to_free_page(void)
336 {
337 int i=6;
338
339 while (i--) {
340 if (shrink_buffers(i))
341 return 1;
342 if (shm_swap(i))
343 return 1;
344 if (swap_out(i))
345 return 1;
346 }
347 return 0;
348 }
349
350
351
352
353
354
355 static inline void add_mem_queue(unsigned long addr, unsigned long * queue)
356 {
357 addr &= 0xfffff000;
358 *(unsigned long *) addr = *queue;
359 *queue = addr;
360 }
361
362
363
364
365
366
367
368
369
370
371
372
373 void free_page(unsigned long addr)
374 {
375 if (addr < high_memory) {
376 unsigned short * map = mem_map + MAP_NR(addr);
377
378 if (*map) {
379 if (!(*map & MAP_PAGE_RESERVED)) {
380 unsigned long flag;
381
382 save_flags(flag);
383 cli();
384 if (!--*map) {
385 if (nr_secondary_pages < MAX_SECONDARY_PAGES) {
386 add_mem_queue(addr,&secondary_page_list);
387 nr_secondary_pages++;
388 restore_flags(flag);
389 return;
390 }
391 add_mem_queue(addr,&free_page_list);
392 nr_free_pages++;
393 }
394 restore_flags(flag);
395 }
396 return;
397 }
398 printk("Trying to free free memory (%08x): memory probabably corrupted\n",addr);
399 printk("PC = %08x\n",*(((unsigned long *)&addr)-1));
400 return;
401 }
402 printk("Trying to free nonexistent page %08x\n",addr);
403 return;
404 }
405
406
407
408
409
410
411
412
413
414
415
416 #define REMOVE_FROM_MEM_QUEUE(queue,nr) \
417 cli(); \
418 if ((result = queue) != 0) { \
419 if (!(result & 0xfff) && result < high_memory) { \
420 queue = *(unsigned long *) result; \
421 if (!mem_map[MAP_NR(result)]) { \
422 mem_map[MAP_NR(result)] = 1; \
423 nr--; \
424 last_free_pages[index = (index + 1) & (NR_LAST_FREE_PAGES - 1)] = result; \
425 restore_flags(flag); \
426 return result; \
427 } \
428 printk("Free page %08x has mem_map = %d\n", \
429 result,mem_map[MAP_NR(result)]); \
430 } else \
431 printk("Result = 0x%08x - memory map destroyed\n", result); \
432 queue = 0; \
433 nr = 0; \
434 } else if (nr) { \
435 printk(#nr " is %d, but " #queue " is empty\n",nr); \
436 nr = 0; \
437 } \
438 restore_flags(flag)
439
440
441
442
443
444
445
446
447
448
449 unsigned long __get_free_page(int priority)
450 {
451 unsigned long result, flag;
452 static unsigned long index = 0;
453
454
455
456
457
458
459 save_flags(flag);
460 repeat:
461 REMOVE_FROM_MEM_QUEUE(free_page_list,nr_free_pages);
462 if (priority == GFP_BUFFER)
463 return 0;
464 if (priority != GFP_ATOMIC)
465 if (try_to_free_page())
466 goto repeat;
467 REMOVE_FROM_MEM_QUEUE(secondary_page_list,nr_secondary_pages);
468 return 0;
469 }
470
471
472
473
474
475
476 static int try_to_unuse(unsigned int type)
477 {
478 int nr, pgt, pg;
479 unsigned long page, *ppage;
480 unsigned long tmp = 0;
481 struct task_struct *p;
482
483 nr = 0;
484
485
486
487
488 repeat:
489 for (; nr < NR_TASKS ; nr++) {
490 p = task[nr];
491 if (!p)
492 continue;
493 for (pgt = 0 ; pgt < 1024 ; pgt++) {
494 ppage = pgt + ((unsigned long *) p->tss.cr3);
495 page = *ppage;
496 if (!page)
497 continue;
498 if (!(page & PAGE_PRESENT) || (page >= high_memory)) {
499 printk("try_to_unuse: bad page directory (%d,%d:%08x)\n",nr,pgt,page);
500 *ppage = 0;
501 continue;
502 }
503 if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
504 continue;
505 ppage = (unsigned long *) (page & 0xfffff000);
506 for (pg = 0 ; pg < 1024 ; pg++,ppage++) {
507 page = *ppage;
508 if (!page)
509 continue;
510 if (page & PAGE_PRESENT) {
511 if (page >= high_memory) {
512 printk("try_to_unuse: bad page table (%d,%d,%d:%08x)\n",nr,pgt,pg,page);
513 *ppage = 0;
514 }
515 continue;
516 }
517 if (SWP_TYPE(page) != type)
518 continue;
519 if (!tmp) {
520 tmp = get_free_page(GFP_KERNEL);
521 if (!tmp)
522 return -ENOMEM;
523 goto repeat;
524 }
525 read_swap_page(page, (char *) tmp);
526 if (*ppage == page) {
527 *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
528 ++p->rss;
529 swap_free(page);
530 tmp = 0;
531 }
532 goto repeat;
533 }
534 }
535 }
536 free_page(tmp);
537 return 0;
538 }
539
540 int sys_swapoff(const char * specialfile)
541 {
542 struct swap_info_struct * p;
543 struct inode * inode;
544 unsigned int type;
545 int i;
546
547 if (!suser())
548 return -EPERM;
549 i = namei(specialfile,&inode);
550 if (i)
551 return i;
552 p = swap_info;
553 for (type = 0 ; type < nr_swapfiles ; type++,p++) {
554 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
555 continue;
556 if (p->swap_file) {
557 if (p->swap_file == inode)
558 break;
559 } else {
560 if (!S_ISBLK(inode->i_mode))
561 continue;
562 if (p->swap_device == inode->i_rdev)
563 break;
564 }
565 }
566 iput(inode);
567 if (type >= nr_swapfiles)
568 return -EINVAL;
569 p->flags = SWP_USED;
570 i = try_to_unuse(type);
571 if (i) {
572 p->flags = SWP_WRITEOK;
573 return i;
574 }
575 iput(p->swap_file);
576 p->swap_file = NULL;
577 p->swap_device = 0;
578 free_page((long) p->swap_map);
579 p->swap_map = NULL;
580 free_page((long) p->swap_lockmap);
581 p->swap_lockmap = NULL;
582 p->flags = 0;
583 return 0;
584 }
585
586
587
588
589
590
591 int sys_swapon(const char * specialfile)
592 {
593 struct swap_info_struct * p;
594 struct inode * swap_inode;
595 unsigned int type;
596 char * tmp;
597 int i,j;
598
599 if (!suser())
600 return -EPERM;
601 p = swap_info;
602 for (type = 0 ; type < nr_swapfiles ; type++,p++)
603 if (!(p->flags & SWP_USED))
604 break;
605 if (type >= MAX_SWAPFILES)
606 return -EPERM;
607 if (type >= nr_swapfiles)
608 nr_swapfiles = type+1;
609 p->flags = SWP_USED;
610 p->swap_file = NULL;
611 p->swap_device = 0;
612 p->swap_map = NULL;
613 p->swap_lockmap = NULL;
614 p->lowest_bit = 0;
615 p->highest_bit = 0;
616 i = namei(specialfile,&swap_inode);
617 if (i) {
618 p->flags = 0;
619 return i;
620 }
621 if (swap_inode->i_count != 1) {
622 iput(swap_inode);
623 p->flags = 0;
624 return -EBUSY;
625 }
626 if (S_ISBLK(swap_inode->i_mode)) {
627 p->swap_device = swap_inode->i_rdev;
628 iput(swap_inode);
629 if (!p->swap_device) {
630 p->flags = 0;
631 return -ENODEV;
632 }
633 for (i = 0 ; i < nr_swapfiles ; i++) {
634 if (i == type)
635 continue;
636 if (p->swap_device == swap_info[i].swap_device) {
637 p->swap_device = 0;
638 p->flags = 0;
639 return -EBUSY;
640 }
641 }
642 } else if (S_ISREG(swap_inode->i_mode))
643 p->swap_file = swap_inode;
644 else {
645 iput(swap_inode);
646 p->flags = 0;
647 return -EINVAL;
648 }
649 tmp = (char *) get_free_page(GFP_USER);
650 p->swap_lockmap = (char *) get_free_page(GFP_USER);
651 if (!tmp || !p->swap_lockmap) {
652 printk("Unable to start swapping: out of memory :-)\n");
653 free_page((long) tmp);
654 free_page((long) p->swap_lockmap);
655 iput(p->swap_file);
656 p->swap_device = 0;
657 p->swap_file = NULL;
658 p->swap_map = NULL;
659 p->swap_lockmap = NULL;
660 p->flags = 0;
661 return -ENOMEM;
662 }
663 read_swap_page(SWP_ENTRY(type,0),tmp);
664 if (strncmp("SWAP-SPACE",tmp+4086,10)) {
665 printk("Unable to find swap-space signature\n");
666 free_page((long) tmp);
667 free_page((long) p->swap_lockmap);
668 iput(p->swap_file);
669 p->swap_device = 0;
670 p->swap_file = NULL;
671 p->swap_map = NULL;
672 p->swap_lockmap = NULL;
673 p->flags = 0;
674 return -EINVAL;
675 }
676 memset(tmp+4086,0,10);
677 j = 0;
678 p->lowest_bit = 0;
679 p->highest_bit = 0;
680 for (i = 1 ; i < SWAP_BITS ; i++)
681 if (test_bit(i,tmp)) {
682 if (!p->lowest_bit)
683 p->lowest_bit = i;
684 p->highest_bit = i;
685 j++;
686 }
687 if (!j) {
688 printk("Empty swap-file\n");
689 free_page((long) tmp);
690 free_page((long) p->swap_lockmap);
691 iput(p->swap_file);
692 p->swap_device = 0;
693 p->swap_file = NULL;
694 p->swap_map = NULL;
695 p->swap_lockmap = NULL;
696 p->flags = 0;
697 return -EINVAL;
698 }
699 i = SWAP_BITS;
700 while (i--)
701 if (test_bit(i,tmp))
702 tmp[i] = 0;
703 else
704 tmp[i] = 128;
705 tmp[0] = 128;
706 p->swap_map = tmp;
707 p->flags = SWP_WRITEOK;
708 printk("Adding Swap: %dk swap-space\n",j<<2);
709 return 0;
710 }
711
712 void si_swapinfo(struct sysinfo *val)
713 {
714 unsigned int i, j;
715
716 val->freeswap = val->totalswap = 0;
717 for (i = 0; i < nr_swapfiles; i++) {
718 if (!(swap_info[i].flags & SWP_USED))
719 continue;
720 for (j = 0; j < 4096; ++j)
721 switch (swap_info[i].swap_map[j]) {
722 case 128:
723 continue;
724 case 0:
725 ++val->freeswap;
726 default:
727 ++val->totalswap;
728 }
729 }
730 val->freeswap <<= PAGE_SHIFT;
731 val->totalswap <<= PAGE_SHIFT;
732 return;
733 }