This source file includes following definitions.
- oom
- free_one_pte
- free_one_pmd
- free_one_pgd
- clear_page_tables
- free_page_tables
- clone_page_tables
- copy_one_pte
- copy_one_pmd
- copy_one_pgd
- copy_page_tables
- forget_pte
- unmap_pte_range
- unmap_pmd_range
- unmap_page_range
- zeromap_pte_range
- zeromap_pmd_range
- zeromap_page_range
- remap_pte_range
- remap_pmd_range
- remap_page_range
- put_page
- put_dirty_page
- do_wp_page
- verify_area
- get_empty_page
- try_to_share
- share_page
- get_empty_pgtable
- do_swap_page
- do_no_page
1 #define THREE_LEVEL
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37 #include <linux/config.h>
38 #include <linux/signal.h>
39 #include <linux/sched.h>
40 #include <linux/head.h>
41 #include <linux/kernel.h>
42 #include <linux/errno.h>
43 #include <linux/string.h>
44 #include <linux/types.h>
45 #include <linux/ptrace.h>
46 #include <linux/mman.h>
47 #include <linux/mm.h>
48
49 #include <asm/system.h>
50 #include <asm/segment.h>
51 #include <asm/pgtable.h>
52
53 unsigned long high_memory = 0;
54
55
56
57
58
59 int nr_swap_pages = 0;
60 int nr_free_pages = 0;
61 struct mem_list free_area_list[NR_MEM_LISTS];
62 unsigned char * free_area_map[NR_MEM_LISTS];
63
64 #define copy_page(from,to) memcpy((void *) to, (void *) from, PAGE_SIZE)
65
66 #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE)
67
68 mem_map_t * mem_map = NULL;
69
70
71
72
73
74 void oom(struct task_struct * task)
75 {
76 printk("\nOut of memory for %s.\n", current->comm);
77 task->sigaction[SIGKILL-1].sa_handler = NULL;
78 task->blocked &= ~(1<<(SIGKILL-1));
79 send_sig(SIGKILL,task,1);
80 }
81
82 static inline void free_one_pte(pte_t * page_table)
83 {
84 pte_t page = *page_table;
85
86 if (pte_none(page))
87 return;
88 pte_clear(page_table);
89 if (!pte_present(page)) {
90 swap_free(pte_val(page));
91 return;
92 }
93 free_page(pte_page(page));
94 return;
95 }
96
97 static inline void free_one_pmd(pmd_t * dir)
98 {
99 int j;
100 pte_t * pte;
101
102 if (pmd_none(*dir))
103 return;
104 if (pmd_bad(*dir)) {
105 printk("free_one_pmd: bad directory entry %08lx\n", pmd_val(*dir));
106 pmd_clear(dir);
107 return;
108 }
109 pte = pte_offset(dir, 0);
110 pmd_clear(dir);
111 if (pte_inuse(pte)) {
112 pte_free(pte);
113 return;
114 }
115 for (j = 0; j < PTRS_PER_PTE ; j++)
116 free_one_pte(pte+j);
117 pte_free(pte);
118 }
119
120 static inline void free_one_pgd(pgd_t * dir)
121 {
122 int j;
123 pmd_t * pmd;
124
125 if (pgd_none(*dir))
126 return;
127 if (pgd_bad(*dir)) {
128 printk("free_one_pgd: bad directory entry %08lx\n", pgd_val(*dir));
129 pgd_clear(dir);
130 return;
131 }
132 pmd = pmd_offset(dir, 0);
133 pgd_clear(dir);
134 if (pmd_inuse(pmd)) {
135 pmd_free(pmd);
136 return;
137 }
138 for (j = 0; j < PTRS_PER_PMD ; j++)
139 free_one_pmd(pmd+j);
140 pmd_free(pmd);
141 }
142
143
144
145
146
147
148
149
150
151 void clear_page_tables(struct task_struct * tsk)
152 {
153 int i;
154 pgd_t * page_dir;
155
156 if (!tsk)
157 return;
158 if (tsk == task[0])
159 panic("task[0] (swapper) doesn't support exec()\n");
160 page_dir = pgd_offset(tsk, 0);
161 if (!page_dir || page_dir == swapper_pg_dir) {
162 printk("Trying to clear kernel page-directory: not good\n");
163 return;
164 }
165 if (pgd_inuse(page_dir)) {
166 pgd_t * new_pg;
167
168 if (!(new_pg = pgd_alloc())) {
169 oom(tsk);
170 return;
171 }
172 for (i = USER_PTRS_PER_PGD ; i < PTRS_PER_PGD ; i++)
173 new_pg[i] = page_dir[i];
174 SET_PAGE_DIR(tsk, new_pg);
175 pgd_free(page_dir);
176 return;
177 }
178 for (i = 0 ; i < USER_PTRS_PER_PGD ; i++)
179 free_one_pgd(page_dir + i);
180 invalidate();
181 return;
182 }
183
184
185
186
187 void free_page_tables(struct task_struct * tsk)
188 {
189 int i;
190 pgd_t * page_dir;
191
192 if (!tsk)
193 return;
194 if (tsk == task[0]) {
195 printk("task[0] (swapper) killed: unable to recover\n");
196 panic("Trying to free up swapper memory space");
197 }
198 page_dir = pgd_offset(tsk, 0);
199 if (!page_dir || page_dir == swapper_pg_dir) {
200 printk("Trying to free kernel page-directory: not good\n");
201 return;
202 }
203 SET_PAGE_DIR(tsk, swapper_pg_dir);
204 if (pgd_inuse(page_dir)) {
205 pgd_free(page_dir);
206 return;
207 }
208 for (i = 0 ; i < PTRS_PER_PGD ; i++)
209 free_one_pgd(page_dir + i);
210 pgd_free(page_dir);
211 invalidate();
212 }
213
214
215
216
217
218
219
220 int clone_page_tables(struct task_struct * tsk)
221 {
222 pgd_t * pg_dir;
223
224 pg_dir = pgd_offset(current, 0);
225 pgd_reuse(pg_dir);
226 SET_PAGE_DIR(tsk, pg_dir);
227 return 0;
228 }
229
230 static inline void copy_one_pte(pte_t * old_pte, pte_t * new_pte)
231 {
232 pte_t pte = *old_pte;
233
234 if (pte_none(pte))
235 return;
236 if (!pte_present(pte)) {
237 swap_duplicate(pte_val(pte));
238 *new_pte = pte;
239 return;
240 }
241 if (pte_page(pte) > high_memory || (mem_map[MAP_NR(pte_page(pte))] & MAP_PAGE_RESERVED)) {
242 *new_pte = pte;
243 return;
244 }
245 if (pte_cow(pte))
246 pte = pte_wrprotect(pte);
247 if (delete_from_swap_cache(pte_page(pte)))
248 pte = pte_mkdirty(pte);
249 *new_pte = pte;
250 *old_pte = pte;
251 mem_map[MAP_NR(pte_page(pte))]++;
252 }
253
254 static inline int copy_one_pmd(pmd_t * old_pmd, pmd_t * new_pmd)
255 {
256 int j;
257 pte_t *old_pte, *new_pte;
258
259 if (pmd_none(*old_pmd))
260 return 0;
261 if (pmd_bad(*old_pmd)) {
262 printk("copy_one_pmd: bad page table: probable memory corruption\n");
263 pmd_clear(old_pmd);
264 return 0;
265 }
266 old_pte = pte_offset(old_pmd, 0);
267 if (pte_inuse(old_pte)) {
268 pte_reuse(old_pte);
269 *new_pmd = *old_pmd;
270 return 0;
271 }
272 new_pte = pte_alloc(new_pmd, 0);
273 if (!new_pte)
274 return -ENOMEM;
275 for (j = 0 ; j < PTRS_PER_PTE ; j++) {
276 copy_one_pte(old_pte, new_pte);
277 old_pte++;
278 new_pte++;
279 }
280 return 0;
281 }
282
283 static inline int copy_one_pgd(pgd_t * old_pgd, pgd_t * new_pgd)
284 {
285 int j;
286 pmd_t *old_pmd, *new_pmd;
287
288 if (pgd_none(*old_pgd))
289 return 0;
290 if (pgd_bad(*old_pgd)) {
291 printk("copy_one_pgd: bad page table: probable memory corruption\n");
292 pgd_clear(old_pgd);
293 return 0;
294 }
295 old_pmd = pmd_offset(old_pgd, 0);
296 if (pmd_inuse(old_pmd)) {
297 pmd_reuse(old_pmd);
298 *new_pgd = *old_pgd;
299 return 0;
300 }
301 new_pmd = pmd_alloc(new_pgd, 0);
302 if (!new_pmd)
303 return -ENOMEM;
304 for (j = 0 ; j < PTRS_PER_PMD ; j++) {
305 int error = copy_one_pmd(old_pmd, new_pmd);
306 if (error)
307 return error;
308 old_pmd++;
309 new_pmd++;
310 }
311 return 0;
312 }
313
314
315
316
317
318
319 int copy_page_tables(struct task_struct * tsk)
320 {
321 int i;
322 pgd_t *old_pgd;
323 pgd_t *new_pgd;
324
325 new_pgd = pgd_alloc();
326 if (!new_pgd)
327 return -ENOMEM;
328 old_pgd = pgd_offset(current, 0);
329 SET_PAGE_DIR(tsk, new_pgd);
330 for (i = 0 ; i < PTRS_PER_PGD ; i++) {
331 int errno = copy_one_pgd(old_pgd, new_pgd);
332 if (errno) {
333 free_page_tables(tsk);
334 invalidate();
335 return errno;
336 }
337 old_pgd++;
338 new_pgd++;
339 }
340 invalidate();
341 return 0;
342 }
343
344 static inline void forget_pte(pte_t page)
345 {
346 if (pte_none(page))
347 return;
348 if (pte_present(page)) {
349 free_page(pte_page(page));
350 if (mem_map[MAP_NR(pte_page(page))] & MAP_PAGE_RESERVED)
351 return;
352 if (current->mm->rss <= 0)
353 return;
354 current->mm->rss--;
355 return;
356 }
357 swap_free(pte_val(page));
358 }
359
360 static inline void unmap_pte_range(pmd_t * pmd, unsigned long address, unsigned long size)
361 {
362 pte_t * pte;
363 unsigned long end;
364
365 if (pmd_none(*pmd))
366 return;
367 if (pmd_bad(*pmd)) {
368 printk("unmap_pte_range: bad pmd (%08lx)\n", pmd_val(*pmd));
369 pmd_clear(pmd);
370 return;
371 }
372 pte = pte_offset(pmd, address);
373 address &= ~PMD_MASK;
374 end = address + size;
375 if (end >= PMD_SIZE)
376 end = PMD_SIZE;
377 do {
378 pte_t page = *pte;
379 pte_clear(pte);
380 forget_pte(page);
381 address += PAGE_SIZE;
382 pte++;
383 } while (address < end);
384 }
385
386 static inline void unmap_pmd_range(pgd_t * dir, unsigned long address, unsigned long size)
387 {
388 pmd_t * pmd;
389 unsigned long end;
390
391 if (pgd_none(*dir))
392 return;
393 if (pgd_bad(*dir)) {
394 printk("unmap_pmd_range: bad pgd (%08lx)\n", pgd_val(*dir));
395 pgd_clear(dir);
396 return;
397 }
398 pmd = pmd_offset(dir, address);
399 address &= ~PGDIR_MASK;
400 end = address + size;
401 if (end > PGDIR_SIZE)
402 end = PGDIR_SIZE;
403 do {
404 unmap_pte_range(pmd, address, end - address);
405 address = (address + PMD_SIZE) & PMD_MASK;
406 pmd++;
407 } while (address < end);
408 }
409
410
411
412
413
414 int unmap_page_range(unsigned long address, unsigned long size)
415 {
416 pgd_t * dir;
417 unsigned long end = address + size;
418
419 dir = pgd_offset(current, address);
420 while (address < end) {
421 unmap_pmd_range(dir, address, end - address);
422 address = (address + PGDIR_SIZE) & PGDIR_MASK;
423 dir++;
424 }
425 invalidate();
426 return 0;
427 }
428
429 static inline void zeromap_pte_range(pte_t * pte, unsigned long address, unsigned long size, pte_t zero_pte)
430 {
431 unsigned long end;
432
433 address &= ~PMD_MASK;
434 end = address + size;
435 if (end > PMD_SIZE)
436 end = PMD_SIZE;
437 do {
438 pte_t oldpage = *pte;
439 *pte = zero_pte;
440 forget_pte(oldpage);
441 address += PAGE_SIZE;
442 pte++;
443 } while (address < end);
444 }
445
446 static inline int zeromap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, pte_t zero_pte)
447 {
448 unsigned long end;
449
450 address &= ~PGDIR_MASK;
451 end = address + size;
452 if (end > PGDIR_SIZE)
453 end = PGDIR_SIZE;
454 do {
455 pte_t * pte = pte_alloc(pmd, address);
456 if (!pte)
457 return -ENOMEM;
458 zeromap_pte_range(pte, address, end - address, zero_pte);
459 address = (address + PMD_SIZE) & PMD_MASK;
460 pmd++;
461 } while (address < end);
462 return 0;
463 }
464
465 int zeromap_page_range(unsigned long address, unsigned long size, pgprot_t prot)
466 {
467 int error = 0;
468 pgd_t * dir;
469 unsigned long end = address + size;
470 pte_t zero_pte;
471
472 zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE, prot));
473 dir = pgd_offset(current, address);
474 while (address < end) {
475 pmd_t *pmd = pmd_alloc(dir, address);
476 error = -ENOMEM;
477 if (!pmd)
478 break;
479 error = zeromap_pmd_range(pmd, address, end - address, zero_pte);
480 if (error)
481 break;
482 address = (address + PGDIR_SIZE) & PGDIR_MASK;
483 dir++;
484 }
485 invalidate();
486 return error;
487 }
488
489
490
491
492
493
494 static inline void remap_pte_range(pte_t * pte, unsigned long address, unsigned long size,
495 unsigned long offset, pgprot_t prot)
496 {
497 unsigned long end;
498
499 address &= ~PMD_MASK;
500 end = address + size;
501 if (end > PMD_SIZE)
502 end = PMD_SIZE;
503 do {
504 pte_t oldpage = *pte;
505 *pte = mk_pte(offset, prot);
506 forget_pte(oldpage);
507 address += PAGE_SIZE;
508 offset += PAGE_SIZE;
509 pte++;
510 } while (address < end);
511 }
512
513 static inline int remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size,
514 unsigned long offset, pgprot_t prot)
515 {
516 unsigned long end;
517
518 address &= ~PGDIR_MASK;
519 end = address + size;
520 if (end > PGDIR_SIZE)
521 end = PGDIR_SIZE;
522 offset -= address;
523 do {
524 pte_t * pte = pte_alloc(pmd, address);
525 if (!pte)
526 return -ENOMEM;
527 remap_pte_range(pte, address, end - address, address + offset, prot);
528 address = (address + PMD_SIZE) & PMD_MASK;
529 pmd++;
530 } while (address < end);
531 return 0;
532 }
533
534 int remap_page_range(unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot)
535 {
536 int error = 0;
537 pgd_t * dir;
538 unsigned long end = from + size;
539
540 offset -= from;
541 dir = pgd_offset(current, from);
542 while (from < end) {
543 pmd_t *pmd = pmd_alloc(dir, from);
544 error = -ENOMEM;
545 if (!pmd)
546 break;
547 error = remap_pmd_range(pmd, from, end - from, offset + from, prot);
548 if (error)
549 break;
550 from = (from + PGDIR_SIZE) & PGDIR_MASK;
551 dir++;
552 }
553 invalidate();
554 return error;
555 }
556
557
558
559
560 static void put_page(pte_t * page_table, pte_t pte)
561 {
562 if (!pte_none(*page_table)) {
563 printk("put_page: page already exists\n");
564 free_page(pte_page(pte));
565 return;
566 }
567
568 *page_table = pte;
569 }
570
571
572
573
574
575 unsigned long put_dirty_page(struct task_struct * tsk, unsigned long page, unsigned long address)
576 {
577 pgd_t * pgd;
578 pmd_t * pmd;
579 pte_t * pte;
580
581 if (page >= high_memory)
582 printk("put_dirty_page: trying to put page %08lx at %08lx\n",page,address);
583 if (mem_map[MAP_NR(page)] != 1)
584 printk("mem_map disagrees with %08lx at %08lx\n",page,address);
585 pgd = pgd_offset(tsk,address);
586 pmd = pmd_alloc(pgd, address);
587 if (!pmd)
588 return 0;
589 pte = pte_alloc(pmd, address);
590 if (!pte)
591 return 0;
592 if (!pte_none(*pte)) {
593 printk("put_dirty_page: page already exists\n");
594 pte_clear(pte);
595 invalidate();
596 }
597 *pte = pte_mkwrite(pte_mkdirty(mk_pte(page, PAGE_COPY)));
598
599 return page;
600 }
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619 void do_wp_page(struct vm_area_struct * vma, unsigned long address,
620 int write_access)
621 {
622 pgd_t *page_dir;
623 pmd_t *page_middle;
624 pte_t *page_table, pte;
625 unsigned long old_page, new_page;
626
627 new_page = __get_free_page(GFP_KERNEL);
628 page_dir = pgd_offset(vma->vm_task,address);
629 if (pgd_none(*page_dir))
630 goto end_wp_page;
631 if (pgd_bad(*page_dir))
632 goto bad_wp_pagedir;
633 page_middle = pmd_offset(page_dir, address);
634 if (pmd_none(*page_middle))
635 goto end_wp_page;
636 if (pmd_bad(*page_middle))
637 goto bad_wp_pagemiddle;
638 page_table = pte_offset(page_middle, address);
639 pte = *page_table;
640 if (!pte_present(pte))
641 goto end_wp_page;
642 if (pte_write(pte))
643 goto end_wp_page;
644 old_page = pte_page(pte);
645 if (old_page >= high_memory)
646 goto bad_wp_page;
647 vma->vm_task->mm->min_flt++;
648
649
650
651 if (mem_map[MAP_NR(old_page)] != 1) {
652 if (new_page) {
653 if (mem_map[MAP_NR(old_page)] & MAP_PAGE_RESERVED)
654 ++vma->vm_task->mm->rss;
655 copy_page(old_page,new_page);
656 *page_table = pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)));
657 free_page(old_page);
658 invalidate();
659 return;
660 }
661 free_page(old_page);
662 oom(vma->vm_task);
663 *page_table = BAD_PAGE;
664 invalidate();
665 return;
666 }
667 *page_table = pte_mkdirty(pte_mkwrite(pte));
668 invalidate();
669 if (new_page)
670 free_page(new_page);
671 return;
672 bad_wp_page:
673 printk("do_wp_page: bogus page at address %08lx (%08lx)\n",address,old_page);
674 send_sig(SIGKILL, vma->vm_task, 1);
675 goto end_wp_page;
676 bad_wp_pagemiddle:
677 printk("do_wp_page: bogus page-middle at address %08lx (%08lx)\n", address, pmd_val(*page_middle));
678 send_sig(SIGKILL, vma->vm_task, 1);
679 goto end_wp_page;
680 bad_wp_pagedir:
681 printk("do_wp_page: bogus page-dir entry at address %08lx (%08lx)\n", address, pgd_val(*page_dir));
682 send_sig(SIGKILL, vma->vm_task, 1);
683 end_wp_page:
684 if (new_page)
685 free_page(new_page);
686 return;
687 }
688
689
690
691
692 int verify_area(int type, const void * addr, unsigned long size)
693 {
694 struct vm_area_struct * vma;
695 unsigned long start = (unsigned long) addr;
696
697
698
699
700
701 if (get_fs() == get_ds())
702 return 0;
703
704 vma = find_vma(current, start);
705 if (!vma)
706 goto bad_area;
707 if (vma->vm_start <= start)
708 goto good_area;
709 if (!(vma->vm_flags & VM_GROWSDOWN))
710 goto bad_area;
711 if (vma->vm_end - start > current->rlim[RLIMIT_STACK].rlim_cur)
712 goto bad_area;
713
714 good_area:
715 if (type == VERIFY_WRITE)
716 goto check_write;
717 for (;;) {
718 struct vm_area_struct * next;
719 if (!(vma->vm_flags & VM_READ))
720 goto bad_area;
721 if (vma->vm_end - start >= size)
722 return 0;
723 next = vma->vm_next;
724 if (!next || vma->vm_end != next->vm_start)
725 goto bad_area;
726 vma = next;
727 }
728
729 check_write:
730 if (!(vma->vm_flags & VM_WRITE))
731 goto bad_area;
732 if (!wp_works_ok)
733 goto check_wp_fault_by_hand;
734 for (;;) {
735 if (vma->vm_end - start >= size)
736 break;
737 if (!vma->vm_next || vma->vm_end != vma->vm_next->vm_start)
738 goto bad_area;
739 vma = vma->vm_next;
740 if (!(vma->vm_flags & VM_WRITE))
741 goto bad_area;
742 }
743 return 0;
744
745 check_wp_fault_by_hand:
746 size--;
747 size += start & ~PAGE_MASK;
748 size >>= PAGE_SHIFT;
749 start &= PAGE_MASK;
750
751 for (;;) {
752 do_wp_page(vma, start, 1);
753 if (!size)
754 break;
755 size--;
756 start += PAGE_SIZE;
757 if (start < vma->vm_end)
758 continue;
759 vma = vma->vm_next;
760 if (!vma || vma->vm_start != start)
761 goto bad_area;
762 if (!(vma->vm_flags & VM_WRITE))
763 goto bad_area;;
764 }
765 return 0;
766
767 bad_area:
768 return -EFAULT;
769 }
770
771 static inline void get_empty_page(struct vm_area_struct * vma, pte_t * page_table)
772 {
773 unsigned long tmp;
774
775 if (!(tmp = get_free_page(GFP_KERNEL))) {
776 oom(vma->vm_task);
777 put_page(page_table, BAD_PAGE);
778 return;
779 }
780 put_page(page_table, pte_mkwrite(mk_pte(tmp, vma->vm_page_prot)));
781 }
782
783
784
785
786
787
788
789
790
791 static int try_to_share(unsigned long to_address, struct vm_area_struct * to_area,
792 unsigned long from_address, struct vm_area_struct * from_area,
793 unsigned long newpage)
794 {
795 pgd_t * from_dir, * to_dir;
796 pmd_t * from_middle, * to_middle;
797 pte_t * from_table, * to_table;
798 pte_t from, to;
799
800 from_dir = pgd_offset(from_area->vm_task,from_address);
801
802 if (pgd_none(*from_dir))
803 return 0;
804 if (pgd_bad(*from_dir)) {
805 printk("try_to_share: bad page directory %08lx\n", pgd_val(*from_dir));
806 return 0;
807 }
808 from_middle = pmd_offset(from_dir, from_address);
809
810 if (pmd_none(*from_middle))
811 return 0;
812 if (pmd_bad(*from_middle)) {
813 printk("try_to_share: bad mid directory %08lx\n", pmd_val(*from_middle));
814 return 0;
815 }
816 from_table = pte_offset(from_middle, from_address);
817 from = *from_table;
818
819 if (!pte_present(from))
820 return 0;
821
822 if (pte_dirty(from)) {
823 if (!(from_area->vm_flags & VM_SHARED))
824 return 0;
825 if (pte_write(from)) {
826 printk("nonwritable, but dirty, shared page\n");
827 return 0;
828 }
829 }
830
831 if (pte_page(from) >= high_memory)
832 return 0;
833 if (mem_map[MAP_NR(pte_page(from))] & MAP_PAGE_RESERVED)
834 return 0;
835
836 to_dir = pgd_offset(to_area->vm_task,to_address);
837
838 if (pgd_none(*to_dir))
839 return 0;
840 if (pgd_bad(*to_dir)) {
841 printk("try_to_share: bad page directory %08lx\n", pgd_val(*to_dir));
842 return 0;
843 }
844 to_middle = pmd_offset(to_dir, to_address);
845
846 if (pmd_none(*to_middle))
847 return 0;
848 if (pmd_bad(*to_middle)) {
849 printk("try_to_share: bad mid directory %08lx\n", pmd_val(*to_middle));
850 return 0;
851 }
852 to_table = pte_offset(to_middle, to_address);
853 to = *to_table;
854 if (!pte_none(to))
855 return 0;
856
857 if (newpage) {
858
859
860 if (in_swap_cache(pte_page(from))) {
861 if (!(from_area->vm_flags & VM_SHARED))
862 return 0;
863 if (!pte_write(from)) {
864 printk("nonwritable, but dirty, shared page\n");
865 return 0;
866 }
867 }
868 copy_page(pte_page(from), newpage);
869 *to_table = mk_pte(newpage, to_area->vm_page_prot);
870 return 1;
871 }
872
873
874
875
876
877
878
879
880
881 if (in_swap_cache(pte_page(from))) {
882 if (!(from_area->vm_flags & VM_SHARED))
883 return 0;
884 *from_table = pte_mkdirty(from);
885 delete_from_swap_cache(pte_page(from));
886 }
887 mem_map[MAP_NR(pte_page(from))]++;
888 *to_table = mk_pte(pte_page(from), to_area->vm_page_prot);
889
890 if (!pte_write(from))
891 return 1;
892 if (from_area->vm_flags & VM_SHARED)
893 return 1;
894
895 *from_table = pte_wrprotect(from);
896 invalidate();
897 return 1;
898 }
899
900
901
902
903
904
905
906
907 static int share_page(struct vm_area_struct * area, unsigned long address,
908 int write_access, unsigned long newpage)
909 {
910 struct inode * inode;
911 unsigned long offset;
912 unsigned long from_address;
913 unsigned long give_page;
914 struct vm_area_struct * mpnt;
915
916 if (!area || !(inode = area->vm_inode) || inode->i_count < 2)
917 return 0;
918
919 give_page = 0;
920 if (write_access && !(area->vm_flags & VM_SHARED)) {
921 if (!newpage)
922 return 0;
923 give_page = newpage;
924 }
925 offset = address - area->vm_start + area->vm_offset;
926
927
928 for (mpnt = area->vm_next_share; mpnt != area; mpnt = mpnt->vm_next_share) {
929
930 if (mpnt->vm_inode != inode) {
931 printk("Aiee! Corrupt vm_area_struct i_mmap ring\n");
932 break;
933 }
934
935 if ((mpnt->vm_offset ^ area->vm_offset) & ~PAGE_MASK)
936 continue;
937
938 from_address = offset + mpnt->vm_start - mpnt->vm_offset;
939 if (from_address < mpnt->vm_start || from_address >= mpnt->vm_end)
940 continue;
941
942 if (!try_to_share(address, area, from_address, mpnt, give_page))
943 continue;
944
945 if (give_page || !newpage)
946 return 1;
947 free_page(newpage);
948 return 1;
949 }
950 return 0;
951 }
952
953
954
955
956 static inline pte_t * get_empty_pgtable(struct task_struct * tsk,unsigned long address)
957 {
958 pgd_t *pgd;
959 pmd_t *pmd;
960 pte_t *pte;
961
962 pgd = pgd_offset(tsk, address);
963 pmd = pmd_alloc(pgd, address);
964 if (!pmd)
965 return NULL;
966 pte = pte_alloc(pmd, address);
967 return pte;
968 }
969
970 static inline void do_swap_page(struct vm_area_struct * vma, unsigned long address,
971 pte_t * page_table, pte_t entry, int write_access)
972 {
973 pte_t page;
974
975 if (!vma->vm_ops || !vma->vm_ops->swapin) {
976 swap_in(vma, page_table, pte_val(entry), write_access);
977 return;
978 }
979 page = vma->vm_ops->swapin(vma, address - vma->vm_start + vma->vm_offset, pte_val(entry));
980 if (pte_val(*page_table) != pte_val(entry)) {
981 free_page(pte_page(page));
982 return;
983 }
984 if (mem_map[MAP_NR(pte_page(page))] > 1 && !(vma->vm_flags & VM_SHARED))
985 page = pte_wrprotect(page);
986 ++vma->vm_task->mm->rss;
987 ++vma->vm_task->mm->maj_flt;
988 *page_table = page;
989 return;
990 }
991
992
993
994
995
996
997
998 void do_no_page(struct vm_area_struct * vma, unsigned long address,
999 int write_access)
1000 {
1001 pte_t * page_table;
1002 pte_t entry;
1003 unsigned long page;
1004
1005 page_table = get_empty_pgtable(vma->vm_task,address);
1006 if (!page_table)
1007 return;
1008 entry = *page_table;
1009 if (pte_present(entry))
1010 return;
1011 if (!pte_none(entry)) {
1012 do_swap_page(vma, address, page_table, entry, write_access);
1013 return;
1014 }
1015 address &= PAGE_MASK;
1016 if (!vma->vm_ops || !vma->vm_ops->nopage) {
1017 ++vma->vm_task->mm->rss;
1018 ++vma->vm_task->mm->min_flt;
1019 get_empty_page(vma, page_table);
1020 return;
1021 }
1022 page = get_free_page(GFP_KERNEL);
1023 if (share_page(vma, address, write_access, page)) {
1024 ++vma->vm_task->mm->min_flt;
1025 ++vma->vm_task->mm->rss;
1026 return;
1027 }
1028 if (!page) {
1029 oom(current);
1030 put_page(page_table, BAD_PAGE);
1031 return;
1032 }
1033 ++vma->vm_task->mm->maj_flt;
1034 ++vma->vm_task->mm->rss;
1035
1036
1037
1038
1039
1040 page = vma->vm_ops->nopage(vma, address, page,
1041 write_access && !(vma->vm_flags & VM_SHARED));
1042 if (share_page(vma, address, write_access, 0)) {
1043 free_page(page);
1044 return;
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 entry = mk_pte(page, vma->vm_page_prot);
1057 if (write_access) {
1058 entry = pte_mkwrite(pte_mkdirty(entry));
1059 } else if (mem_map[MAP_NR(page)] > 1 && !(vma->vm_flags & VM_SHARED))
1060 entry = pte_wrprotect(entry);
1061 put_page(page_table, entry);
1062 }