This source file includes following definitions.
- oom
- free_one_pte
- free_one_pmd
- free_one_pgd
- clear_page_tables
- free_page_tables
- clone_page_tables
- copy_one_pte
- copy_one_pmd
- copy_one_pgd
- copy_page_tables
- forget_pte
- unmap_pte_range
- unmap_pmd_range
- unmap_page_range
- zeromap_pte_range
- zeromap_pmd_range
- zeromap_page_range
- remap_pte_range
- remap_pmd_range
- remap_page_range
- put_page
- put_dirty_page
- do_wp_page
- verify_area
- get_empty_page
- try_to_share
- share_page
- get_empty_pgtable
- do_swap_page
- do_no_page
- handle_pte_fault
- handle_mm_fault
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36 #include <linux/config.h>
37 #include <linux/signal.h>
38 #include <linux/sched.h>
39 #include <linux/head.h>
40 #include <linux/kernel.h>
41 #include <linux/errno.h>
42 #include <linux/string.h>
43 #include <linux/types.h>
44 #include <linux/ptrace.h>
45 #include <linux/mman.h>
46 #include <linux/mm.h>
47
48 #include <asm/system.h>
49 #include <asm/segment.h>
50 #include <asm/pgtable.h>
51
52 unsigned long high_memory = 0;
53
54
55
56
57
58 int nr_swap_pages = 0;
59 int nr_free_pages = 0;
60 struct mem_list free_area_list[NR_MEM_LISTS];
61 unsigned char * free_area_map[NR_MEM_LISTS];
62
63 #define copy_page(from,to) memcpy((void *) to, (void *) from, PAGE_SIZE)
64
65 #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE)
66
67 mem_map_t * mem_map = NULL;
68
69
70
71
72
73 void oom(struct task_struct * task)
74 {
75 printk("\nOut of memory for %s.\n", current->comm);
76 task->sigaction[SIGKILL-1].sa_handler = NULL;
77 task->blocked &= ~(1<<(SIGKILL-1));
78 send_sig(SIGKILL,task,1);
79 }
80
81 static inline void free_one_pte(pte_t * page_table)
82 {
83 pte_t page = *page_table;
84
85 if (pte_none(page))
86 return;
87 pte_clear(page_table);
88 if (!pte_present(page)) {
89 swap_free(pte_val(page));
90 return;
91 }
92 free_page(pte_page(page));
93 return;
94 }
95
96 static inline void free_one_pmd(pmd_t * dir)
97 {
98 int j;
99 pte_t * pte;
100
101 if (pmd_none(*dir))
102 return;
103 if (pmd_bad(*dir)) {
104 printk("free_one_pmd: bad directory entry %08lx\n", pmd_val(*dir));
105 pmd_clear(dir);
106 return;
107 }
108 pte = pte_offset(dir, 0);
109 pmd_clear(dir);
110 if (pte_inuse(pte)) {
111 pte_free(pte);
112 return;
113 }
114 for (j = 0; j < PTRS_PER_PTE ; j++)
115 free_one_pte(pte+j);
116 pte_free(pte);
117 }
118
119 static inline void free_one_pgd(pgd_t * dir)
120 {
121 int j;
122 pmd_t * pmd;
123
124 if (pgd_none(*dir))
125 return;
126 if (pgd_bad(*dir)) {
127 printk("free_one_pgd: bad directory entry %08lx\n", pgd_val(*dir));
128 pgd_clear(dir);
129 return;
130 }
131 pmd = pmd_offset(dir, 0);
132 pgd_clear(dir);
133 if (pmd_inuse(pmd)) {
134 pmd_free(pmd);
135 return;
136 }
137 for (j = 0; j < PTRS_PER_PMD ; j++)
138 free_one_pmd(pmd+j);
139 pmd_free(pmd);
140 }
141
142
143
144
145
146
147
148
149
150 void clear_page_tables(struct task_struct * tsk)
151 {
152 int i;
153 pgd_t * page_dir;
154
155 if (!tsk)
156 return;
157 if (tsk == task[0])
158 panic("task[0] (swapper) doesn't support exec()\n");
159 page_dir = pgd_offset(tsk, 0);
160 if (!page_dir || page_dir == swapper_pg_dir) {
161 printk("%s trying to clear kernel page-directory: not good\n", tsk->comm);
162 return;
163 }
164 if (pgd_inuse(page_dir)) {
165 pgd_t * new_pg;
166
167 if (!(new_pg = pgd_alloc())) {
168 oom(tsk);
169 return;
170 }
171 for (i = USER_PTRS_PER_PGD ; i < PTRS_PER_PGD ; i++)
172 new_pg[i] = page_dir[i];
173 SET_PAGE_DIR(tsk, new_pg);
174 pgd_free(page_dir);
175 return;
176 }
177 for (i = 0 ; i < USER_PTRS_PER_PGD ; i++)
178 free_one_pgd(page_dir + i);
179 invalidate();
180 return;
181 }
182
183
184
185
186 void free_page_tables(struct task_struct * tsk)
187 {
188 int i;
189 pgd_t * page_dir;
190
191 if (!tsk)
192 return;
193 if (tsk == task[0]) {
194 printk("task[0] (swapper) killed: unable to recover\n");
195 panic("Trying to free up swapper memory space");
196 }
197 page_dir = pgd_offset(tsk, 0);
198 if (!page_dir || page_dir == swapper_pg_dir) {
199 printk("%s trying to free kernel page-directory: not good\n", tsk->comm);
200 return;
201 }
202 SET_PAGE_DIR(tsk, swapper_pg_dir);
203 if (pgd_inuse(page_dir)) {
204 pgd_free(page_dir);
205 return;
206 }
207 for (i = 0 ; i < PTRS_PER_PGD ; i++)
208 free_one_pgd(page_dir + i);
209 pgd_free(page_dir);
210 invalidate();
211 }
212
213
214
215
216
217
218
219 int clone_page_tables(struct task_struct * tsk)
220 {
221 pgd_t * pg_dir;
222
223 pg_dir = pgd_offset(current, 0);
224 pgd_reuse(pg_dir);
225 SET_PAGE_DIR(tsk, pg_dir);
226 return 0;
227 }
228
229 static inline void copy_one_pte(pte_t * old_pte, pte_t * new_pte)
230 {
231 pte_t pte = *old_pte;
232
233 if (pte_none(pte))
234 return;
235 if (!pte_present(pte)) {
236 swap_duplicate(pte_val(pte));
237 *new_pte = pte;
238 return;
239 }
240 if (pte_page(pte) > high_memory || (mem_map[MAP_NR(pte_page(pte))] & MAP_PAGE_RESERVED)) {
241 *new_pte = pte;
242 return;
243 }
244 if (pte_cow(pte))
245 pte = pte_wrprotect(pte);
246 if (delete_from_swap_cache(pte_page(pte)))
247 pte = pte_mkdirty(pte);
248 *new_pte = pte_mkold(pte);
249 *old_pte = pte;
250 mem_map[MAP_NR(pte_page(pte))]++;
251 }
252
253 static inline int copy_one_pmd(pmd_t * old_pmd, pmd_t * new_pmd)
254 {
255 int j;
256 pte_t *old_pte, *new_pte;
257
258 if (pmd_none(*old_pmd))
259 return 0;
260 if (pmd_bad(*old_pmd)) {
261 printk("copy_one_pmd: bad page table: probable memory corruption\n");
262 pmd_clear(old_pmd);
263 return 0;
264 }
265 old_pte = pte_offset(old_pmd, 0);
266 if (pte_inuse(old_pte)) {
267 pte_reuse(old_pte);
268 *new_pmd = *old_pmd;
269 return 0;
270 }
271 new_pte = pte_alloc(new_pmd, 0);
272 if (!new_pte)
273 return -ENOMEM;
274 for (j = 0 ; j < PTRS_PER_PTE ; j++) {
275 copy_one_pte(old_pte, new_pte);
276 old_pte++;
277 new_pte++;
278 }
279 return 0;
280 }
281
282 static inline int copy_one_pgd(pgd_t * old_pgd, pgd_t * new_pgd)
283 {
284 int j;
285 pmd_t *old_pmd, *new_pmd;
286
287 if (pgd_none(*old_pgd))
288 return 0;
289 if (pgd_bad(*old_pgd)) {
290 printk("copy_one_pgd: bad page table (%p: %08lx): probable memory corruption\n", old_pgd, pgd_val(*old_pgd));
291 pgd_clear(old_pgd);
292 return 0;
293 }
294 old_pmd = pmd_offset(old_pgd, 0);
295 if (pmd_inuse(old_pmd)) {
296 pmd_reuse(old_pmd);
297 *new_pgd = *old_pgd;
298 return 0;
299 }
300 new_pmd = pmd_alloc(new_pgd, 0);
301 if (!new_pmd)
302 return -ENOMEM;
303 for (j = 0 ; j < PTRS_PER_PMD ; j++) {
304 int error = copy_one_pmd(old_pmd, new_pmd);
305 if (error)
306 return error;
307 old_pmd++;
308 new_pmd++;
309 }
310 return 0;
311 }
312
313
314
315
316
317
318 int copy_page_tables(struct task_struct * tsk)
319 {
320 int i;
321 pgd_t *old_pgd;
322 pgd_t *new_pgd;
323
324 new_pgd = pgd_alloc();
325 if (!new_pgd)
326 return -ENOMEM;
327 SET_PAGE_DIR(tsk, new_pgd);
328 old_pgd = pgd_offset(current, 0);
329 for (i = 0 ; i < PTRS_PER_PGD ; i++) {
330 int errno = copy_one_pgd(old_pgd, new_pgd);
331 if (errno) {
332 free_page_tables(tsk);
333 invalidate();
334 return errno;
335 }
336 old_pgd++;
337 new_pgd++;
338 }
339 invalidate();
340 return 0;
341 }
342
343 static inline void forget_pte(pte_t page)
344 {
345 if (pte_none(page))
346 return;
347 if (pte_present(page)) {
348 free_page(pte_page(page));
349 if (mem_map[MAP_NR(pte_page(page))] & MAP_PAGE_RESERVED)
350 return;
351 if (current->mm->rss <= 0)
352 return;
353 current->mm->rss--;
354 return;
355 }
356 swap_free(pte_val(page));
357 }
358
359 static inline void unmap_pte_range(pmd_t * pmd, unsigned long address, unsigned long size)
360 {
361 pte_t * pte;
362 unsigned long end;
363
364 if (pmd_none(*pmd))
365 return;
366 if (pmd_bad(*pmd)) {
367 printk("unmap_pte_range: bad pmd (%08lx)\n", pmd_val(*pmd));
368 pmd_clear(pmd);
369 return;
370 }
371 pte = pte_offset(pmd, address);
372 address &= ~PMD_MASK;
373 end = address + size;
374 if (end >= PMD_SIZE)
375 end = PMD_SIZE;
376 do {
377 pte_t page = *pte;
378 pte_clear(pte);
379 forget_pte(page);
380 address += PAGE_SIZE;
381 pte++;
382 } while (address < end);
383 }
384
385 static inline void unmap_pmd_range(pgd_t * dir, unsigned long address, unsigned long size)
386 {
387 pmd_t * pmd;
388 unsigned long end;
389
390 if (pgd_none(*dir))
391 return;
392 if (pgd_bad(*dir)) {
393 printk("unmap_pmd_range: bad pgd (%08lx)\n", pgd_val(*dir));
394 pgd_clear(dir);
395 return;
396 }
397 pmd = pmd_offset(dir, address);
398 address &= ~PGDIR_MASK;
399 end = address + size;
400 if (end > PGDIR_SIZE)
401 end = PGDIR_SIZE;
402 do {
403 unmap_pte_range(pmd, address, end - address);
404 address = (address + PMD_SIZE) & PMD_MASK;
405 pmd++;
406 } while (address < end);
407 }
408
409
410
411
412
413 int unmap_page_range(unsigned long address, unsigned long size)
414 {
415 pgd_t * dir;
416 unsigned long end = address + size;
417
418 dir = pgd_offset(current, address);
419 while (address < end) {
420 unmap_pmd_range(dir, address, end - address);
421 address = (address + PGDIR_SIZE) & PGDIR_MASK;
422 dir++;
423 }
424 invalidate();
425 return 0;
426 }
427
428 static inline void zeromap_pte_range(pte_t * pte, unsigned long address, unsigned long size, pte_t zero_pte)
429 {
430 unsigned long end;
431
432 address &= ~PMD_MASK;
433 end = address + size;
434 if (end > PMD_SIZE)
435 end = PMD_SIZE;
436 do {
437 pte_t oldpage = *pte;
438 *pte = zero_pte;
439 forget_pte(oldpage);
440 address += PAGE_SIZE;
441 pte++;
442 } while (address < end);
443 }
444
445 static inline int zeromap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, pte_t zero_pte)
446 {
447 unsigned long end;
448
449 address &= ~PGDIR_MASK;
450 end = address + size;
451 if (end > PGDIR_SIZE)
452 end = PGDIR_SIZE;
453 do {
454 pte_t * pte = pte_alloc(pmd, address);
455 if (!pte)
456 return -ENOMEM;
457 zeromap_pte_range(pte, address, end - address, zero_pte);
458 address = (address + PMD_SIZE) & PMD_MASK;
459 pmd++;
460 } while (address < end);
461 return 0;
462 }
463
464 int zeromap_page_range(unsigned long address, unsigned long size, pgprot_t prot)
465 {
466 int error = 0;
467 pgd_t * dir;
468 unsigned long end = address + size;
469 pte_t zero_pte;
470
471 zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE, prot));
472 dir = pgd_offset(current, address);
473 while (address < end) {
474 pmd_t *pmd = pmd_alloc(dir, address);
475 error = -ENOMEM;
476 if (!pmd)
477 break;
478 error = zeromap_pmd_range(pmd, address, end - address, zero_pte);
479 if (error)
480 break;
481 address = (address + PGDIR_SIZE) & PGDIR_MASK;
482 dir++;
483 }
484 invalidate();
485 return error;
486 }
487
488
489
490
491
492
493 static inline void remap_pte_range(pte_t * pte, unsigned long address, unsigned long size,
494 unsigned long offset, pgprot_t prot)
495 {
496 unsigned long end;
497
498 address &= ~PMD_MASK;
499 end = address + size;
500 if (end > PMD_SIZE)
501 end = PMD_SIZE;
502 do {
503 pte_t oldpage = *pte;
504 pte_clear(pte);
505 if (offset >= high_memory || (mem_map[MAP_NR(offset)] & MAP_PAGE_RESERVED))
506 *pte = mk_pte(offset, prot);
507 forget_pte(oldpage);
508 address += PAGE_SIZE;
509 offset += PAGE_SIZE;
510 pte++;
511 } while (address < end);
512 }
513
514 static inline int remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size,
515 unsigned long offset, pgprot_t prot)
516 {
517 unsigned long end;
518
519 address &= ~PGDIR_MASK;
520 end = address + size;
521 if (end > PGDIR_SIZE)
522 end = PGDIR_SIZE;
523 offset -= address;
524 do {
525 pte_t * pte = pte_alloc(pmd, address);
526 if (!pte)
527 return -ENOMEM;
528 remap_pte_range(pte, address, end - address, address + offset, prot);
529 address = (address + PMD_SIZE) & PMD_MASK;
530 pmd++;
531 } while (address < end);
532 return 0;
533 }
534
535 int remap_page_range(unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot)
536 {
537 int error = 0;
538 pgd_t * dir;
539 unsigned long end = from + size;
540
541 offset -= from;
542 dir = pgd_offset(current, from);
543 while (from < end) {
544 pmd_t *pmd = pmd_alloc(dir, from);
545 error = -ENOMEM;
546 if (!pmd)
547 break;
548 error = remap_pmd_range(pmd, from, end - from, offset + from, prot);
549 if (error)
550 break;
551 from = (from + PGDIR_SIZE) & PGDIR_MASK;
552 dir++;
553 }
554 invalidate();
555 return error;
556 }
557
558
559
560
561 static void put_page(pte_t * page_table, pte_t pte)
562 {
563 if (!pte_none(*page_table)) {
564 printk("put_page: page already exists %08lx\n", pte_val(*page_table));
565 free_page(pte_page(pte));
566 return;
567 }
568
569 *page_table = pte;
570 }
571
572
573
574
575
576 unsigned long put_dirty_page(struct task_struct * tsk, unsigned long page, unsigned long address)
577 {
578 pgd_t * pgd;
579 pmd_t * pmd;
580 pte_t * pte;
581
582 if (page >= high_memory)
583 printk("put_dirty_page: trying to put page %08lx at %08lx\n",page,address);
584 if (mem_map[MAP_NR(page)] != 1)
585 printk("mem_map disagrees with %08lx at %08lx\n",page,address);
586 pgd = pgd_offset(tsk,address);
587 pmd = pmd_alloc(pgd, address);
588 if (!pmd) {
589 free_page(page);
590 oom(tsk);
591 return 0;
592 }
593 pte = pte_alloc(pmd, address);
594 if (!pte) {
595 free_page(page);
596 oom(tsk);
597 return 0;
598 }
599 if (!pte_none(*pte)) {
600 printk("put_dirty_page: page already exists\n");
601 pte_clear(pte);
602 invalidate();
603 }
604 *pte = pte_mkwrite(pte_mkdirty(mk_pte(page, PAGE_COPY)));
605
606 return page;
607 }
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626 void do_wp_page(struct vm_area_struct * vma, unsigned long address,
627 int write_access)
628 {
629 pgd_t *page_dir;
630 pmd_t *page_middle;
631 pte_t *page_table, pte;
632 unsigned long old_page, new_page;
633
634 new_page = __get_free_page(GFP_KERNEL);
635 page_dir = pgd_offset(vma->vm_task,address);
636 if (pgd_none(*page_dir))
637 goto end_wp_page;
638 if (pgd_bad(*page_dir))
639 goto bad_wp_pagedir;
640 page_middle = pmd_offset(page_dir, address);
641 if (pmd_none(*page_middle))
642 goto end_wp_page;
643 if (pmd_bad(*page_middle))
644 goto bad_wp_pagemiddle;
645 page_table = pte_offset(page_middle, address);
646 pte = *page_table;
647 if (!pte_present(pte))
648 goto end_wp_page;
649 if (pte_write(pte))
650 goto end_wp_page;
651 old_page = pte_page(pte);
652 if (old_page >= high_memory)
653 goto bad_wp_page;
654 vma->vm_task->mm->min_flt++;
655
656
657
658 if (mem_map[MAP_NR(old_page)] != 1) {
659 if (new_page) {
660 if (mem_map[MAP_NR(old_page)] & MAP_PAGE_RESERVED)
661 ++vma->vm_task->mm->rss;
662 copy_page(old_page,new_page);
663 *page_table = pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)));
664 free_page(old_page);
665 invalidate();
666 return;
667 }
668 *page_table = BAD_PAGE;
669 free_page(old_page);
670 oom(vma->vm_task);
671 invalidate();
672 return;
673 }
674 *page_table = pte_mkdirty(pte_mkwrite(pte));
675 invalidate();
676 if (new_page)
677 free_page(new_page);
678 return;
679 bad_wp_page:
680 printk("do_wp_page: bogus page at address %08lx (%08lx)\n",address,old_page);
681 send_sig(SIGKILL, vma->vm_task, 1);
682 goto end_wp_page;
683 bad_wp_pagemiddle:
684 printk("do_wp_page: bogus page-middle at address %08lx (%08lx)\n", address, pmd_val(*page_middle));
685 send_sig(SIGKILL, vma->vm_task, 1);
686 goto end_wp_page;
687 bad_wp_pagedir:
688 printk("do_wp_page: bogus page-dir entry at address %08lx (%08lx)\n", address, pgd_val(*page_dir));
689 send_sig(SIGKILL, vma->vm_task, 1);
690 end_wp_page:
691 if (new_page)
692 free_page(new_page);
693 return;
694 }
695
696
697
698
699 int verify_area(int type, const void * addr, unsigned long size)
700 {
701 struct vm_area_struct * vma;
702 unsigned long start = (unsigned long) addr;
703
704
705
706
707
708 if (get_fs() == get_ds())
709 return 0;
710
711 vma = find_vma(current, start);
712 if (!vma)
713 goto bad_area;
714 if (vma->vm_start <= start)
715 goto good_area;
716 if (!(vma->vm_flags & VM_GROWSDOWN))
717 goto bad_area;
718 if (vma->vm_end - start > current->rlim[RLIMIT_STACK].rlim_cur)
719 goto bad_area;
720
721 good_area:
722 if (type == VERIFY_WRITE)
723 goto check_write;
724 for (;;) {
725 struct vm_area_struct * next;
726 if (!(vma->vm_flags & VM_READ))
727 goto bad_area;
728 if (vma->vm_end - start >= size)
729 return 0;
730 next = vma->vm_next;
731 if (!next || vma->vm_end != next->vm_start)
732 goto bad_area;
733 vma = next;
734 }
735
736 check_write:
737 if (!(vma->vm_flags & VM_WRITE))
738 goto bad_area;
739 if (!wp_works_ok)
740 goto check_wp_fault_by_hand;
741 for (;;) {
742 if (vma->vm_end - start >= size)
743 break;
744 if (!vma->vm_next || vma->vm_end != vma->vm_next->vm_start)
745 goto bad_area;
746 vma = vma->vm_next;
747 if (!(vma->vm_flags & VM_WRITE))
748 goto bad_area;
749 }
750 return 0;
751
752 check_wp_fault_by_hand:
753 size--;
754 size += start & ~PAGE_MASK;
755 size >>= PAGE_SHIFT;
756 start &= PAGE_MASK;
757
758 for (;;) {
759 do_wp_page(vma, start, 1);
760 if (!size)
761 break;
762 size--;
763 start += PAGE_SIZE;
764 if (start < vma->vm_end)
765 continue;
766 vma = vma->vm_next;
767 if (!vma || vma->vm_start != start)
768 goto bad_area;
769 if (!(vma->vm_flags & VM_WRITE))
770 goto bad_area;;
771 }
772 return 0;
773
774 bad_area:
775 return -EFAULT;
776 }
777
778 static inline void get_empty_page(struct vm_area_struct * vma, pte_t * page_table)
779 {
780 unsigned long tmp;
781
782 if (!(tmp = get_free_page(GFP_KERNEL))) {
783 oom(vma->vm_task);
784 put_page(page_table, BAD_PAGE);
785 return;
786 }
787 put_page(page_table, pte_mkwrite(mk_pte(tmp, vma->vm_page_prot)));
788 }
789
790
791
792
793
794
795
796
797
798 static int try_to_share(unsigned long to_address, struct vm_area_struct * to_area,
799 unsigned long from_address, struct vm_area_struct * from_area,
800 unsigned long newpage)
801 {
802 pgd_t * from_dir, * to_dir;
803 pmd_t * from_middle, * to_middle;
804 pte_t * from_table, * to_table;
805 pte_t from, to;
806
807 from_dir = pgd_offset(from_area->vm_task,from_address);
808
809 if (pgd_none(*from_dir))
810 return 0;
811 if (pgd_bad(*from_dir)) {
812 printk("try_to_share: bad page directory %08lx\n", pgd_val(*from_dir));
813 pgd_clear(from_dir);
814 return 0;
815 }
816 from_middle = pmd_offset(from_dir, from_address);
817
818 if (pmd_none(*from_middle))
819 return 0;
820 if (pmd_bad(*from_middle)) {
821 printk("try_to_share: bad mid directory %08lx\n", pmd_val(*from_middle));
822 pmd_clear(from_middle);
823 return 0;
824 }
825 from_table = pte_offset(from_middle, from_address);
826 from = *from_table;
827
828 if (!pte_present(from))
829 return 0;
830
831 if (pte_dirty(from)) {
832 if (!(from_area->vm_flags & VM_SHARED))
833 return 0;
834 if (pte_write(from)) {
835 printk("nonwritable, but dirty, shared page\n");
836 return 0;
837 }
838 }
839
840 if (pte_page(from) >= high_memory)
841 return 0;
842 if (mem_map[MAP_NR(pte_page(from))] & MAP_PAGE_RESERVED)
843 return 0;
844
845 to_dir = pgd_offset(to_area->vm_task,to_address);
846
847 if (pgd_none(*to_dir))
848 return 0;
849 if (pgd_bad(*to_dir)) {
850 printk("try_to_share: bad page directory %08lx\n", pgd_val(*to_dir));
851 return 0;
852 }
853 to_middle = pmd_offset(to_dir, to_address);
854
855 if (pmd_none(*to_middle))
856 return 0;
857 if (pmd_bad(*to_middle)) {
858 printk("try_to_share: bad mid directory %08lx\n", pmd_val(*to_middle));
859 return 0;
860 }
861 to_table = pte_offset(to_middle, to_address);
862 to = *to_table;
863 if (!pte_none(to))
864 return 0;
865
866 if (newpage) {
867
868
869 if (in_swap_cache(pte_page(from))) {
870 if (!(from_area->vm_flags & VM_SHARED))
871 return 0;
872 if (!pte_write(from)) {
873 printk("nonwritable, but dirty, shared page\n");
874 return 0;
875 }
876 }
877 copy_page(pte_page(from), newpage);
878 *to_table = mk_pte(newpage, to_area->vm_page_prot);
879 return 1;
880 }
881
882
883
884
885
886
887
888
889
890 if (in_swap_cache(pte_page(from))) {
891 if (!(from_area->vm_flags & VM_SHARED))
892 return 0;
893 *from_table = pte_mkdirty(from);
894 delete_from_swap_cache(pte_page(from));
895 }
896 mem_map[MAP_NR(pte_page(from))]++;
897 *to_table = mk_pte(pte_page(from), to_area->vm_page_prot);
898
899 if (!pte_write(from))
900 return 1;
901 if (from_area->vm_flags & VM_SHARED)
902 return 1;
903
904 *from_table = pte_wrprotect(from);
905 invalidate();
906 return 1;
907 }
908
909
910
911
912
913
914
915
916 static int share_page(struct vm_area_struct * area, unsigned long address,
917 int write_access, unsigned long newpage)
918 {
919 struct inode * inode;
920 unsigned long offset;
921 unsigned long from_address;
922 unsigned long give_page;
923 struct vm_area_struct * mpnt;
924
925 if (!area || !(inode = area->vm_inode) || inode->i_count < 2)
926 return 0;
927
928 give_page = 0;
929 if (write_access && !(area->vm_flags & VM_SHARED)) {
930 if (!newpage)
931 return 0;
932 give_page = newpage;
933 }
934 offset = address - area->vm_start + area->vm_offset;
935
936
937 for (mpnt = area->vm_next_share; mpnt != area; mpnt = mpnt->vm_next_share) {
938
939 if (mpnt->vm_inode != inode) {
940 printk("Aiee! Corrupt vm_area_struct i_mmap ring\n");
941 break;
942 }
943
944 if ((mpnt->vm_offset ^ area->vm_offset) & ~PAGE_MASK)
945 continue;
946
947 from_address = offset + mpnt->vm_start - mpnt->vm_offset;
948 if (from_address < mpnt->vm_start || from_address >= mpnt->vm_end)
949 continue;
950
951 if (!try_to_share(address, area, from_address, mpnt, give_page))
952 continue;
953
954 if (give_page || !newpage)
955 return 1;
956 free_page(newpage);
957 return 1;
958 }
959 return 0;
960 }
961
962
963
964
965 static inline pte_t * get_empty_pgtable(struct task_struct * tsk,unsigned long address)
966 {
967 pgd_t *pgd;
968 pmd_t *pmd;
969 pte_t *pte;
970
971 pgd = pgd_offset(tsk, address);
972 pmd = pmd_alloc(pgd, address);
973 if (!pmd) {
974 oom(tsk);
975 return NULL;
976 }
977 pte = pte_alloc(pmd, address);
978 if (!pte) {
979 oom(tsk);
980 return NULL;
981 }
982 return pte;
983 }
984
985 static inline void do_swap_page(struct vm_area_struct * vma, unsigned long address,
986 pte_t * page_table, pte_t entry, int write_access)
987 {
988 pte_t page;
989
990 if (!vma->vm_ops || !vma->vm_ops->swapin) {
991 swap_in(vma, page_table, pte_val(entry), write_access);
992 return;
993 }
994 page = vma->vm_ops->swapin(vma, address - vma->vm_start + vma->vm_offset, pte_val(entry));
995 if (pte_val(*page_table) != pte_val(entry)) {
996 free_page(pte_page(page));
997 return;
998 }
999 if (mem_map[MAP_NR(pte_page(page))] > 1 && !(vma->vm_flags & VM_SHARED))
1000 page = pte_wrprotect(page);
1001 ++vma->vm_task->mm->rss;
1002 ++vma->vm_task->mm->maj_flt;
1003 *page_table = page;
1004 return;
1005 }
1006
1007
1008
1009
1010
1011
1012
1013 void do_no_page(struct vm_area_struct * vma, unsigned long address,
1014 int write_access)
1015 {
1016 pte_t * page_table;
1017 pte_t entry;
1018 unsigned long page;
1019
1020 page_table = get_empty_pgtable(vma->vm_task,address);
1021 if (!page_table)
1022 return;
1023 entry = *page_table;
1024 if (pte_present(entry))
1025 return;
1026 if (!pte_none(entry)) {
1027 do_swap_page(vma, address, page_table, entry, write_access);
1028 return;
1029 }
1030 address &= PAGE_MASK;
1031 if (!vma->vm_ops || !vma->vm_ops->nopage) {
1032 ++vma->vm_task->mm->rss;
1033 ++vma->vm_task->mm->min_flt;
1034 get_empty_page(vma, page_table);
1035 return;
1036 }
1037 page = __get_free_page(GFP_KERNEL);
1038 if (share_page(vma, address, write_access, page)) {
1039 ++vma->vm_task->mm->min_flt;
1040 ++vma->vm_task->mm->rss;
1041 return;
1042 }
1043 if (!page) {
1044 oom(current);
1045 put_page(page_table, BAD_PAGE);
1046 return;
1047 }
1048 ++vma->vm_task->mm->maj_flt;
1049 ++vma->vm_task->mm->rss;
1050
1051
1052
1053
1054
1055 page = vma->vm_ops->nopage(vma, address, page,
1056 write_access && !(vma->vm_flags & VM_SHARED));
1057 if (share_page(vma, address, write_access, 0)) {
1058 free_page(page);
1059 return;
1060 }
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071 entry = mk_pte(page, vma->vm_page_prot);
1072 if (write_access) {
1073 entry = pte_mkwrite(pte_mkdirty(entry));
1074 } else if (mem_map[MAP_NR(page)] > 1 && !(vma->vm_flags & VM_SHARED))
1075 entry = pte_wrprotect(entry);
1076 put_page(page_table, entry);
1077 }
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092 static inline void handle_pte_fault(struct vm_area_struct * vma, unsigned long address,
1093 int write_access, pte_t * pte)
1094 {
1095 if (!pte_present(*pte)) {
1096 do_no_page(vma, address, write_access);
1097 return;
1098 }
1099 *pte = pte_mkyoung(*pte);
1100 if (!write_access)
1101 return;
1102 if (pte_write(*pte)) {
1103 *pte = pte_mkdirty(*pte);
1104 return;
1105 }
1106 do_wp_page(vma, address, write_access);
1107 }
1108
1109 void handle_mm_fault(struct vm_area_struct * vma, unsigned long address,
1110 int write_access)
1111 {
1112 pgd_t *pgd;
1113 pmd_t *pmd;
1114 pte_t *pte;
1115
1116 pgd = pgd_offset(vma->vm_task, address);
1117 pmd = pmd_alloc(pgd, address);
1118 if (!pmd)
1119 goto no_memory;
1120 pte = pte_alloc(pmd, address);
1121 if (!pte)
1122 goto no_memory;
1123 handle_pte_fault(vma, address, write_access, pte);
1124 update_mmu_cache(vma, address, *pte);
1125 return;
1126 no_memory:
1127 oom(vma->vm_task);
1128 }