mm/swap.c

/* */
This source file includes following definitions.
show_swap_cache_info
add_to_swap_cache
init_swap_cache
rw_swap_page
get_swap_page
swap_duplicate
swap_free
swap_in
try_to_swap_out
swap_out_pmd
swap_out_pgd
swap_out_vma
swap_out_process
swap_out
try_to_free_page
add_mem_queue
remove_mem_queue
free_pages_ok
check_free_buffers
free_pages
mark_used
__get_free_pages
show_free_areas
unuse_pte
unuse_pmd
unuse_pgd
unuse_vma
unuse_process
try_to_unuse
sys_swapoff
sys_swapon
si_swapinfo
free_area_init
   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6 
   7 /*
   8  * This file should contain most things doing the swapping from/to disk.
   9  * Started 18.12.91
  10  */
  11 
  12 #include <linux/mm.h>
  13 #include <linux/sched.h>
  14 #include <linux/head.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kernel_stat.h>
  17 #include <linux/errno.h>
  18 #include <linux/string.h>
  19 #include <linux/stat.h>
  20 #include <linux/swap.h>
  21 #include <linux/fs.h>
  22 
  23 #include <asm/dma.h>
  24 #include <asm/system.h> /* for cli()/sti() */
  25 #include <asm/bitops.h>
  26 #include <asm/pgtable.h>
  27 
  28 #define MAX_SWAPFILES 8
  29 
  30 #define SWP_USED        1
  31 #define SWP_WRITEOK     3
  32 
  33 int min_free_pages = 20;
  34 
  35 static int nr_swapfiles = 0;
  36 static struct wait_queue * lock_queue = NULL;
  37 static struct {
  38         int head;       /* head of priority-ordered swapfile list */
  39         int next;       /* swapfile to be used next */
  40 } swap_list = {-1, -1};
  41 
  42 static struct swap_info_struct {
  43         unsigned int flags;
  44         unsigned int swap_device;
  45         struct inode * swap_file;
  46         unsigned char * swap_map;
  47         unsigned char * swap_lockmap;
  48         int lowest_bit;
  49         int highest_bit;
  50         int prio;                       /* swap priority */
  51         int pages;
  52         unsigned long max;
  53         int next;                       /* next entry on swap list */
  54 } swap_info[MAX_SWAPFILES];
  55 
  56 extern int shm_swap (int, unsigned long);
  57 
  58 /*
  59  * To save us from swapping out pages which have just been swapped in and
  60  * have not been modified since then, we keep in swap_cache[page>>PAGE_SHIFT]
  61  * the swap entry which was last used to fill the page, or zero if the
  62  * page does not currently correspond to a page in swap. PAGE_DIRTY makes
  63  * this info useless.
  64  */
  65 unsigned long *swap_cache;
  66 
  67 #ifdef SWAP_CACHE_INFO
  68 unsigned long swap_cache_add_total = 0;
  69 unsigned long swap_cache_add_success = 0;
  70 unsigned long swap_cache_del_total = 0;
  71 unsigned long swap_cache_del_success = 0;
  72 unsigned long swap_cache_find_total = 0;
  73 unsigned long swap_cache_find_success = 0;
  74 
  75 extern inline void show_swap_cache_info(void)
     /*  */
  76 {
  77         printk("Swap cache: add %ld/%ld, delete %ld/%ld, find %ld/%ld\n",
  78                 swap_cache_add_total, swap_cache_add_success, 
  79                 swap_cache_del_total, swap_cache_del_success,
  80                 swap_cache_find_total, swap_cache_find_success);
  81 }
  82 #endif
  83 
  84 static int add_to_swap_cache(unsigned long addr, unsigned long entry)
     /*  */
  85 {
  86         struct swap_info_struct * p = &swap_info[SWP_TYPE(entry)];
  87 
  88 #ifdef SWAP_CACHE_INFO
  89         swap_cache_add_total++;
  90 #endif
  91         if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
  92                 entry = xchg(swap_cache + MAP_NR(addr), entry);
  93                 if (entry)  {
  94                         printk("swap_cache: replacing non-NULL entry\n");
  95                 }
  96 #ifdef SWAP_CACHE_INFO
  97                 swap_cache_add_success++;
  98 #endif
  99                 return 1;
 100         }
 101         return 0;
 102 }
 103 
 104 static unsigned long init_swap_cache(unsigned long mem_start,
     /*  */
 105         unsigned long mem_end)
 106 {
 107         unsigned long swap_cache_size;
 108 
 109         mem_start = (mem_start + 15) & ~15;
 110         swap_cache = (unsigned long *) mem_start;
 111         swap_cache_size = MAP_NR(mem_end);
 112         memset(swap_cache, 0, swap_cache_size * sizeof (unsigned long));
 113         return (unsigned long) (swap_cache + swap_cache_size);
 114 }
 115 
 116 void rw_swap_page(int rw, unsigned long entry, char * buf)
     /*  */
 117 {
 118         unsigned long type, offset;
 119         struct swap_info_struct * p;
 120 
 121         type = SWP_TYPE(entry);
 122         if (type >= nr_swapfiles) {
 123                 printk("Internal error: bad swap-device\n");
 124                 return;
 125         }
 126         p = &swap_info[type];
 127         offset = SWP_OFFSET(entry);
 128         if (offset >= p->max) {
 129                 printk("rw_swap_page: weirdness\n");
 130                 return;
 131         }
 132         if (p->swap_map && !p->swap_map[offset]) {
 133                 printk("Hmm.. Trying to use unallocated swap (%08lx)\n", entry);
 134                 return;
 135         }
 136         if (!(p->flags & SWP_USED)) {
 137                 printk("Trying to swap to unused swap-device\n");
 138                 return;
 139         }
 140         while (set_bit(offset,p->swap_lockmap))
 141                 sleep_on(&lock_queue);
 142         if (rw == READ)
 143                 kstat.pswpin++;
 144         else
 145                 kstat.pswpout++;
 146         if (p->swap_device) {
 147                 ll_rw_page(rw,p->swap_device,offset,buf);
 148         } else if (p->swap_file) {
 149                 struct inode *swapf = p->swap_file;
 150                 unsigned int zones[PAGE_SIZE/512];
 151                 int i;
 152                 if (swapf->i_op->bmap == NULL
 153                         && swapf->i_op->smap != NULL){
 154                         /*
 155                                 With MsDOS, we use msdos_smap which return
 156                                 a sector number (not a cluster or block number).
 157                                 It is a patch to enable the UMSDOS project.
 158                                 Other people are working on better solution.
 159 
 160                                 It sounds like ll_rw_swap_file defined
 161                                 it operation size (sector size) based on
 162                                 PAGE_SIZE and the number of block to read.
 163                                 So using bmap or smap should work even if
 164                                 smap will require more blocks.
 165                         */
 166                         int j;
 167                         unsigned int block = offset << 3;
 168 
 169                         for (i=0, j=0; j< PAGE_SIZE ; i++, j += 512){
 170                                 if (!(zones[i] = swapf->i_op->smap(swapf,block++))) {
 171                                         printk("rw_swap_page: bad swap file\n");
 172                                         return;
 173                                 }
 174                         }
 175                 }else{
 176                         int j;
 177                         unsigned int block = offset
 178                                 << (PAGE_SHIFT - swapf->i_sb->s_blocksize_bits);
 179 
 180                         for (i=0, j=0; j< PAGE_SIZE ; i++, j +=swapf->i_sb->s_blocksize)
 181                                 if (!(zones[i] = bmap(swapf,block++))) {
 182                                         printk("rw_swap_page: bad swap file\n");
 183                                         return;
 184                                 }
 185                 }
 186                 ll_rw_swap_file(rw,swapf->i_dev, zones, i,buf);
 187         } else
 188                 printk("re_swap_page: no swap file or device\n");
 189         if (offset && !clear_bit(offset,p->swap_lockmap))
 190                 printk("rw_swap_page: lock already cleared\n");
 191         wake_up(&lock_queue);
 192 }
 193 
 194 unsigned long get_swap_page(void)
     /*  */
 195 {
 196         struct swap_info_struct * p;
 197         unsigned long offset, entry;
 198         int type, wrapped = 0;
 199 
 200         type = swap_list.next;
 201         if (type < 0)
 202           return 0;
 203 
 204         while (1) {
 205                 p = &swap_info[type];
 206                 if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
 207                         for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
 208                                 if (p->swap_map[offset])
 209                                   continue;
 210                                 if (test_bit(offset, p->swap_lockmap))
 211                                   continue;
 212                                 p->swap_map[offset] = 1;
 213                                 nr_swap_pages--;
 214                                 if (offset == p->highest_bit)
 215                                   p->highest_bit--;
 216                                 p->lowest_bit = offset;
 217                                 entry = SWP_ENTRY(type,offset);
 218 
 219                                 type = swap_info[type].next;
 220                                 if (type < 0 || p->prio != swap_info[type].prio) {
 221                                     swap_list.next = swap_list.head;
 222                                 } else {
 223                                     swap_list.next = type;
 224                                 }
 225                                 return entry;
 226                         }
 227                 }
 228                 type = p->next;
 229                 if (!wrapped) {
 230                         if (type < 0 || p->prio != swap_info[type].prio) {
 231                                 type = swap_list.head;
 232                                 wrapped = 1;
 233                         }
 234                 } else if (type < 0) {
 235                         return 0;       /* out of swap space */
 236                 }
 237         }
 238 }
 239 
 240 void swap_duplicate(unsigned long entry)
     /*  */
 241 {
 242         struct swap_info_struct * p;
 243         unsigned long offset, type;
 244 
 245         if (!entry)
 246                 return;
 247         offset = SWP_OFFSET(entry);
 248         type = SWP_TYPE(entry);
 249         if (type & SHM_SWP_TYPE)
 250                 return;
 251         if (type >= nr_swapfiles) {
 252                 printk("Trying to duplicate nonexistent swap-page\n");
 253                 return;
 254         }
 255         p = type + swap_info;
 256         if (offset >= p->max) {
 257                 printk("swap_duplicate: weirdness\n");
 258                 return;
 259         }
 260         if (!p->swap_map[offset]) {
 261                 printk("swap_duplicate: trying to duplicate unused page\n");
 262                 return;
 263         }
 264         p->swap_map[offset]++;
 265         return;
 266 }
 267 
 268 void swap_free(unsigned long entry)
     /*  */
 269 {
 270         struct swap_info_struct * p;
 271         unsigned long offset, type;
 272 
 273         if (!entry)
 274                 return;
 275         type = SWP_TYPE(entry);
 276         if (type & SHM_SWP_TYPE)
 277                 return;
 278         if (type >= nr_swapfiles) {
 279                 printk("Trying to free nonexistent swap-page\n");
 280                 return;
 281         }
 282         p = & swap_info[type];
 283         offset = SWP_OFFSET(entry);
 284         if (offset >= p->max) {
 285                 printk("swap_free: weirdness\n");
 286                 return;
 287         }
 288         if (!(p->flags & SWP_USED)) {
 289                 printk("Trying to free swap from unused swap-device\n");
 290                 return;
 291         }
 292         if (offset < p->lowest_bit)
 293                 p->lowest_bit = offset;
 294         if (offset > p->highest_bit)
 295                 p->highest_bit = offset;
 296         if (!p->swap_map[offset])
 297                 printk("swap_free: swap-space map bad (entry %08lx)\n",entry);
 298         else
 299                 if (!--p->swap_map[offset])
 300                         nr_swap_pages++;
 301         if (p->prio > swap_info[swap_list.next].prio) {
 302             swap_list.next = swap_list.head;
 303         }
 304 }
 305 
 306 /*
 307  * The tests may look silly, but it essentially makes sure that
 308  * no other process did a swap-in on us just as we were waiting.
 309  *
 310  * Also, don't bother to add to the swap cache if this page-in
 311  * was due to a write access.
 312  */
 313 void swap_in(struct vm_area_struct * vma, pte_t * page_table,
     /*  */
 314         unsigned long entry, int write_access)
 315 {
 316         unsigned long page = __get_free_page(GFP_KERNEL);
 317 
 318         if (pte_val(*page_table) != entry) {
 319                 free_page(page);
 320                 return;
 321         }
 322         if (!page) {
 323                 set_pte(page_table, BAD_PAGE);
 324                 swap_free(entry);
 325                 oom(current);
 326                 return;
 327         }
 328         read_swap_page(entry, (char *) page);
 329         if (pte_val(*page_table) != entry) {
 330                 free_page(page);
 331                 return;
 332         }
 333         vma->vm_task->mm->rss++;
 334         vma->vm_task->mm->maj_flt++;
 335         if (!write_access && add_to_swap_cache(page, entry)) {
 336                 set_pte(page_table, mk_pte(page, vma->vm_page_prot));
 337                 return;
 338         }
 339         set_pte(page_table, pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))));
 340         swap_free(entry);
 341         return;
 342 }
 343 
 344 /*
 345  * The swap-out functions return 1 if they successfully
 346  * threw something out, and we got a free page. It returns
 347  * zero if it couldn't do anything, and any other value
 348  * indicates it decreased rss, but the page was shared.
 349  *
 350  * NOTE! If it sleeps, it *must* return 1 to make sure we
 351  * don't continue with the swap-out. Otherwise we may be
 352  * using a process that no longer actually exists (it might
 353  * have died while we slept).
 354  */
 355 static inline int try_to_swap_out(struct vm_area_struct* vma, unsigned long address, pte_t * page_table, unsigned long limit)
     /*  */
 356 {
 357         pte_t pte;
 358         unsigned long entry;
 359         unsigned long page;
 360 
 361         pte = *page_table;
 362         if (!pte_present(pte))
 363                 return 0;
 364         page = pte_page(pte);
 365         if (page >= high_memory)
 366                 return 0;
 367         if (page >= limit)
 368                 return 0;
 369         if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
 370                 return 0;
 371         if ((pte_dirty(pte) && delete_from_swap_cache(page)) || pte_young(pte))  {
 372                 set_pte(page_table, pte_mkold(pte));
 373                 return 0;
 374         }       
 375         if (pte_dirty(pte)) {
 376                 if (vma->vm_ops && vma->vm_ops->swapout) {
 377                         pid_t pid = vma->vm_task->pid;
 378                         vma->vm_task->mm->rss--;
 379                         if (vma->vm_ops->swapout(vma, address - vma->vm_start + vma->vm_offset, page_table))
 380                                 kill_proc(pid, SIGBUS, 1);
 381                 } else {
 382                         if (mem_map[MAP_NR(page)] != 1)
 383                                 return 0;
 384                         if (!(entry = get_swap_page()))
 385                                 return 0;
 386                         vma->vm_task->mm->rss--;
 387                         set_pte(page_table, __pte(entry));
 388                         invalidate();
 389                         write_swap_page(entry, (char *) page);
 390                 }
 391                 free_page(page);
 392                 return 1;       /* we slept: the process may not exist any more */
 393         }
 394         if ((entry = find_in_swap_cache(page)))  {
 395                 if (mem_map[MAP_NR(page)] != 1) {
 396                         set_pte(page_table, pte_mkdirty(pte));
 397                         printk("Aiee.. duplicated cached swap-cache entry\n");
 398                         return 0;
 399                 }
 400                 vma->vm_task->mm->rss--;
 401                 set_pte(page_table, __pte(entry));
 402                 invalidate();
 403                 free_page(page);
 404                 return 1;
 405         } 
 406         vma->vm_task->mm->rss--;
 407         pte_clear(page_table);
 408         invalidate();
 409         entry = mem_map[MAP_NR(page)];
 410         free_page(page);
 411         return entry;
 412 }
 413 
 414 /*
 415  * A new implementation of swap_out().  We do not swap complete processes,
 416  * but only a small number of blocks, before we continue with the next
 417  * process.  The number of blocks actually swapped is determined on the
 418  * number of page faults, that this process actually had in the last time,
 419  * so we won't swap heavily used processes all the time ...
 420  *
 421  * Note: the priority argument is a hint on much CPU to waste with the
 422  *       swap block search, not a hint, of how much blocks to swap with
 423  *       each process.
 424  *
 425  * (C) 1993 Kai Petzke, wpp@marie.physik.tu-berlin.de
 426  */
 427 
 428 /*
 429  * These are the minimum and maximum number of pages to swap from one process,
 430  * before proceeding to the next:
 431  */
 432 #define SWAP_MIN        4
 433 #define SWAP_MAX        32
 434 
 435 /*
 436  * The actual number of pages to swap is determined as:
 437  * SWAP_RATIO / (number of recent major page faults)
 438  */
 439 #define SWAP_RATIO      128
 440 
 441 static inline int swap_out_pmd(struct vm_area_struct * vma, pmd_t *dir,
     /*  */
 442         unsigned long address, unsigned long end, unsigned long limit)
 443 {
 444         pte_t * pte;
 445         unsigned long pmd_end;
 446 
 447         if (pmd_none(*dir))
 448                 return 0;
 449         if (pmd_bad(*dir)) {
 450                 printk("swap_out_pmd: bad pmd (%08lx)\n", pmd_val(*dir));
 451                 pmd_clear(dir);
 452                 return 0;
 453         }
 454         
 455         pte = pte_offset(dir, address);
 456         
 457         pmd_end = (address + PMD_SIZE) & PMD_MASK;
 458         if (end > pmd_end)
 459                 end = pmd_end;
 460 
 461         do {
 462                 int result;
 463                 vma->vm_task->mm->swap_address = address + PAGE_SIZE;
 464                 result = try_to_swap_out(vma, address, pte, limit);
 465                 if (result)
 466                         return result;
 467                 address += PAGE_SIZE;
 468                 pte++;
 469         } while (address < end);
 470         return 0;
 471 }
 472 
 473 static inline int swap_out_pgd(struct vm_area_struct * vma, pgd_t *dir,
     /*  */
 474         unsigned long address, unsigned long end, unsigned long limit)
 475 {
 476         pmd_t * pmd;
 477         unsigned long pgd_end;
 478 
 479         if (pgd_none(*dir))
 480                 return 0;
 481         if (pgd_bad(*dir)) {
 482                 printk("swap_out_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
 483                 pgd_clear(dir);
 484                 return 0;
 485         }
 486 
 487         pmd = pmd_offset(dir, address);
 488 
 489         pgd_end = (address + PGDIR_SIZE) & PGDIR_MASK;  
 490         if (end > pgd_end)
 491                 end = pgd_end;
 492         
 493         do {
 494                 int result = swap_out_pmd(vma, pmd, address, end, limit);
 495                 if (result)
 496                         return result;
 497                 address = (address + PMD_SIZE) & PMD_MASK;
 498                 pmd++;
 499         } while (address < end);
 500         return 0;
 501 }
 502 
 503 static int swap_out_vma(struct vm_area_struct * vma, pgd_t *pgdir,
     /*  */
 504         unsigned long start, unsigned long limit)
 505 {
 506         unsigned long end;
 507 
 508         /* Don't swap out areas like shared memory which have their
 509             own separate swapping mechanism. */
 510         if (vma->vm_flags & VM_SHM)
 511                 return 0;
 512 
 513         end = vma->vm_end;
 514         while (start < end) {
 515                 int result = swap_out_pgd(vma, pgdir, start, end, limit);
 516                 if (result)
 517                         return result;
 518                 start = (start + PGDIR_SIZE) & PGDIR_MASK;
 519                 pgdir++;
 520         }
 521         return 0;
 522 }
 523 
 524 static int swap_out_process(struct task_struct * p, unsigned long limit)
     /*  */
 525 {
 526         unsigned long address;
 527         struct vm_area_struct* vma;
 528 
 529         /*
 530          * Go through process' page directory.
 531          */
 532         address = p->mm->swap_address;
 533         p->mm->swap_address = 0;
 534 
 535         /*
 536          * Find the proper vm-area
 537          */
 538         vma = find_vma(p, address);
 539         if (!vma)
 540                 return 0;
 541         if (address < vma->vm_start)
 542                 address = vma->vm_start;
 543 
 544         for (;;) {
 545                 int result = swap_out_vma(vma, pgd_offset(p, address), address, limit);
 546                 if (result)
 547                         return result;
 548                 vma = vma->vm_next;
 549                 if (!vma)
 550                         break;
 551                 address = vma->vm_start;
 552         }
 553         p->mm->swap_address = 0;
 554         return 0;
 555 }
 556 
 557 static int swap_out(unsigned int priority, unsigned long limit)
     /*  */
 558 {
 559         static int swap_task;
 560         int loop, counter;
 561         struct task_struct *p;
 562 
 563         counter = 6*nr_tasks >> priority;
 564         for(; counter >= 0; counter--) {
 565                 /*
 566                  * Check that swap_task is suitable for swapping.  If not, look for
 567                  * the next suitable process.
 568                  */
 569                 loop = 0;
 570                 while(1) {
 571                         if (swap_task >= NR_TASKS) {
 572                                 swap_task = 1;
 573                                 if (loop)
 574                                         /* all processes are unswappable or already swapped out */
 575                                         return 0;
 576                                 loop = 1;
 577                         }
 578 
 579                         p = task[swap_task];
 580                         if (p && p->mm->swappable && p->mm->rss)
 581                                 break;
 582 
 583                         swap_task++;
 584                 }
 585 
 586                 /*
 587                  * Determine the number of pages to swap from this process.
 588                  */
 589                 if (!p->mm->swap_cnt) {
 590                         p->mm->dec_flt = (p->mm->dec_flt * 3) / 4 + p->mm->maj_flt - p->mm->old_maj_flt;
 591                         p->mm->old_maj_flt = p->mm->maj_flt;
 592 
 593                         if (p->mm->dec_flt >= SWAP_RATIO / SWAP_MIN) {
 594                                 p->mm->dec_flt = SWAP_RATIO / SWAP_MIN;
 595                                 p->mm->swap_cnt = SWAP_MIN;
 596                         } else if (p->mm->dec_flt <= SWAP_RATIO / SWAP_MAX)
 597                                 p->mm->swap_cnt = SWAP_MAX;
 598                         else
 599                                 p->mm->swap_cnt = SWAP_RATIO / p->mm->dec_flt;
 600                 }
 601                 if (!--p->mm->swap_cnt)
 602                         swap_task++;
 603                 switch (swap_out_process(p, limit)) {
 604                         case 0:
 605                                 if (p->mm->swap_cnt)
 606                                         swap_task++;
 607                                 break;
 608                         case 1:
 609                                 return 1;
 610                         default:
 611                                 break;
 612                 }
 613         }
 614         return 0;
 615 }
 616 
 617 /*
 618  * we keep on shrinking one resource until it's considered "too hard",
 619  * and then switch to the next one (priority being an indication on how
 620  * hard we should try with the resource).
 621  *
 622  * This should automatically find the resource that can most easily be
 623  * free'd, so hopefully we'll get reasonable behaviour even under very
 624  * different circumstances.
 625  */
 626 static int try_to_free_page(int priority, unsigned long limit)
     /*  */
 627 {
 628         static int state = 0;
 629         int i=6;
 630 
 631         switch (state) {
 632                 do {
 633                 case 0:
 634                         if (priority != GFP_NOBUFFER && shrink_buffers(i, limit))
 635                                 return 1;
 636                         state = 1;
 637                 case 1:
 638                         if (shm_swap(i, limit))
 639                                 return 1;
 640                         state = 2;
 641                 default:
 642                         if (swap_out(i, limit))
 643                                 return 1;
 644                         state = 0;
 645                 } while(i--);
 646         }
 647         return 0;
 648 }
 649 
 650 static inline void add_mem_queue(struct mem_list * head, struct mem_list * entry)
     /*  */
 651 {
 652         entry->prev = head;
 653         (entry->next = head->next)->prev = entry;
 654         head->next = entry;
 655 }
 656 
 657 static inline void remove_mem_queue(struct mem_list * head, struct mem_list * entry)
     /*  */
 658 {
 659         entry->next->prev = entry->prev;
 660         entry->prev->next = entry->next;
 661 }
 662 
 663 /*
 664  * Free_page() adds the page to the free lists. This is optimized for
 665  * fast normal cases (no error jumps taken normally).
 666  *
 667  * The way to optimize jumps for gcc-2.2.2 is to:
 668  *  - select the "normal" case and put it inside the if () { XXX }
 669  *  - no else-statements if you can avoid them
 670  *
 671  * With the above two rules, you get a straight-line execution path
 672  * for the normal case, giving better asm-code.
 673  *
 674  * free_page() may sleep since the page being freed may be a buffer
 675  * page or present in the swap cache. It will not sleep, however,
 676  * for a freshly allocated page (get_free_page()).
 677  */
 678 
 679 /*
 680  * Buddy system. Hairy. You really aren't expected to understand this
 681  */
 682 static inline void free_pages_ok(unsigned long addr, unsigned long order)
     /*  */
 683 {
 684         unsigned long index = MAP_NR(addr) >> (1 + order);
 685         unsigned long mask = PAGE_MASK << order;
 686 
 687         addr &= mask;
 688         nr_free_pages += 1 << order;
 689         while (order < NR_MEM_LISTS-1) {
 690                 if (!change_bit(index, free_area_map[order]))
 691                         break;
 692                 remove_mem_queue(free_area_list+order, (struct mem_list *) (addr ^ (1+~mask)));
 693                 order++;
 694                 index >>= 1;
 695                 mask <<= 1;
 696                 addr &= mask;
 697         }
 698         add_mem_queue(free_area_list+order, (struct mem_list *) addr);
 699 }
 700 
 701 static inline void check_free_buffers(unsigned long addr)
     /*  */
 702 {
 703         struct buffer_head * bh;
 704 
 705         bh = buffer_pages[MAP_NR(addr)];
 706         if (bh) {
 707                 struct buffer_head *tmp = bh;
 708                 do {
 709                         if (tmp->b_list == BUF_SHARED && tmp->b_dev != 0xffff)
 710                                 refile_buffer(tmp);
 711                         tmp = tmp->b_this_page;
 712                 } while (tmp != bh);
 713         }
 714 }
 715 
 716 void free_pages(unsigned long addr, unsigned long order)
     /*  */
 717 {
 718         if (addr < high_memory) {
 719                 unsigned long flag;
 720                 mem_map_t * map = mem_map + MAP_NR(addr);
 721                 if (*map) {
 722                         if (!(*map & MAP_PAGE_RESERVED)) {
 723                                 save_flags(flag);
 724                                 cli();
 725                                 if (!--*map)  {
 726                                         free_pages_ok(addr, order);
 727                                         delete_from_swap_cache(addr);
 728                                 }
 729                                 restore_flags(flag);
 730                                 if (*map == 1)
 731                                         check_free_buffers(addr);
 732                         }
 733                         return;
 734                 }
 735                 printk("Trying to free free memory (%08lx): memory probably corrupted\n",addr);
 736                 printk("PC = %p\n", __builtin_return_address(0));
 737                 return;
 738         }
 739 }
 740 
 741 /*
 742  * Some ugly macros to speed up __get_free_pages()..
 743  */
 744 #define RMQUEUE(order, limit) \
 745 do { struct mem_list * queue = free_area_list+order; \
 746      unsigned long new_order = order; \
 747         do { struct mem_list *prev = queue, *ret; \
 748                 while (queue != (ret = prev->next)) { \
 749                         if ((unsigned long) ret < (limit)) { \
 750                                 (prev->next = ret->next)->prev = prev; \
 751                                 mark_used((unsigned long) ret, new_order); \
 752                                 nr_free_pages -= 1 << order; \
 753                                 restore_flags(flags); \
 754                                 EXPAND(ret, order, new_order); \
 755                                 return (unsigned long) ret; \
 756                         } \
 757                         prev = ret; \
 758                 } \
 759                 new_order++; queue++; \
 760         } while (new_order < NR_MEM_LISTS); \
 761 } while (0)
 762 
 763 static inline int mark_used(unsigned long addr, unsigned long order)
     /*  */
 764 {
 765         return change_bit(MAP_NR(addr) >> (1+order), free_area_map[order]);
 766 }
 767 
 768 #define EXPAND(addr,low,high) \
 769 do { unsigned long size = PAGE_SIZE << high; \
 770         while (high > low) { \
 771                 high--; size >>= 1; cli(); \
 772                 add_mem_queue(free_area_list+high, addr); \
 773                 mark_used((unsigned long) addr, high); \
 774                 restore_flags(flags); \
 775                 addr = (struct mem_list *) (size + (unsigned long) addr); \
 776         } mem_map[MAP_NR((unsigned long) addr)] = 1; \
 777 } while (0)
 778 
 779 unsigned long __get_free_pages(int priority, unsigned long order, unsigned long limit)
     /*  */
 780 {
 781         unsigned long flags;
 782         int reserved_pages;
 783 
 784         if (intr_count && priority != GFP_ATOMIC) {
 785                 static int count = 0;
 786                 if (++count < 5) {
 787                         printk("gfp called nonatomically from interrupt %p\n",
 788                                 __builtin_return_address(0));
 789                         priority = GFP_ATOMIC;
 790                 }
 791         }
 792         reserved_pages = 5;
 793         if (priority != GFP_NFS)
 794                 reserved_pages = min_free_pages;
 795         save_flags(flags);
 796 repeat:
 797         cli();
 798         if ((priority==GFP_ATOMIC) || nr_free_pages > reserved_pages) {
 799                 RMQUEUE(order, limit);
 800                 restore_flags(flags);
 801                 return 0;
 802         }
 803         restore_flags(flags);
 804         if (priority != GFP_BUFFER && try_to_free_page(priority, limit))
 805                 goto repeat;
 806         return 0;
 807 }
 808 
 809 /*
 810  * Show free area list (used inside shift_scroll-lock stuff)
 811  * We also calculate the percentage fragmentation. We do this by counting the
 812  * memory on each free list with the exception of the first item on the list.
 813  */
 814 void show_free_areas(void)
     /*  */
 815 {
 816         unsigned long order, flags;
 817         unsigned long total = 0;
 818 
 819         printk("Free pages:      %6dkB\n ( ",nr_free_pages<<(PAGE_SHIFT-10));
 820         save_flags(flags);
 821         cli();
 822         for (order=0 ; order < NR_MEM_LISTS; order++) {
 823                 struct mem_list * tmp;
 824                 unsigned long nr = 0;
 825                 for (tmp = free_area_list[order].next ; tmp != free_area_list + order ; tmp = tmp->next) {
 826                         nr ++;
 827                 }
 828                 total += nr * ((PAGE_SIZE>>10) << order);
 829                 printk("%lu*%lukB ", nr, (PAGE_SIZE>>10) << order);
 830         }
 831         restore_flags(flags);
 832         printk("= %lukB)\n", total);
 833 #ifdef SWAP_CACHE_INFO
 834         show_swap_cache_info();
 835 #endif  
 836 }
 837 
 838 /*
 839  * Trying to stop swapping from a file is fraught with races, so
 840  * we repeat quite a bit here when we have to pause. swapoff()
 841  * isn't exactly timing-critical, so who cares (but this is /really/
 842  * inefficient, ugh).
 843  *
 844  * We return 1 after having slept, which makes the process start over
 845  * from the beginning for this process..
 846  */
 847 static inline int unuse_pte(struct vm_area_struct * vma, unsigned long address,
     /*  */
 848         pte_t *dir, unsigned int type, unsigned long page)
 849 {
 850         pte_t pte = *dir;
 851 
 852         if (pte_none(pte))
 853                 return 0;
 854         if (pte_present(pte)) {
 855                 unsigned long page = pte_page(pte);
 856                 if (page >= high_memory)
 857                         return 0;
 858                 if (!in_swap_cache(page))
 859                         return 0;
 860                 if (SWP_TYPE(in_swap_cache(page)) != type)
 861                         return 0;
 862                 delete_from_swap_cache(page);
 863                 set_pte(dir, pte_mkdirty(pte));
 864                 return 0;
 865         }
 866         if (SWP_TYPE(pte_val(pte)) != type)
 867                 return 0;
 868         read_swap_page(pte_val(pte), (char *) page);
 869         if (pte_val(*dir) != pte_val(pte)) {
 870                 free_page(page);
 871                 return 1;
 872         }
 873         set_pte(dir, pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))));
 874         ++vma->vm_task->mm->rss;
 875         swap_free(pte_val(pte));
 876         return 1;
 877 }
 878 
 879 static inline int unuse_pmd(struct vm_area_struct * vma, pmd_t *dir,
     /*  */
 880         unsigned long address, unsigned long size, unsigned long offset,
 881         unsigned int type, unsigned long page)
 882 {
 883         pte_t * pte;
 884         unsigned long end;
 885 
 886         if (pmd_none(*dir))
 887                 return 0;
 888         if (pmd_bad(*dir)) {
 889                 printk("unuse_pmd: bad pmd (%08lx)\n", pmd_val(*dir));
 890                 pmd_clear(dir);
 891                 return 0;
 892         }
 893         pte = pte_offset(dir, address);
 894         offset += address & PMD_MASK;
 895         address &= ~PMD_MASK;
 896         end = address + size;
 897         if (end > PMD_SIZE)
 898                 end = PMD_SIZE;
 899         do {
 900                 if (unuse_pte(vma, offset+address-vma->vm_start, pte, type, page))
 901                         return 1;
 902                 address += PAGE_SIZE;
 903                 pte++;
 904         } while (address < end);
 905         return 0;
 906 }
 907 
 908 static inline int unuse_pgd(struct vm_area_struct * vma, pgd_t *dir,
     /*  */
 909         unsigned long address, unsigned long size,
 910         unsigned int type, unsigned long page)
 911 {
 912         pmd_t * pmd;
 913         unsigned long offset, end;
 914 
 915         if (pgd_none(*dir))
 916                 return 0;
 917         if (pgd_bad(*dir)) {
 918                 printk("unuse_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
 919                 pgd_clear(dir);
 920                 return 0;
 921         }
 922         pmd = pmd_offset(dir, address);
 923         offset = address & PGDIR_MASK;
 924         address &= ~PGDIR_MASK;
 925         end = address + size;
 926         if (end > PGDIR_SIZE)
 927                 end = PGDIR_SIZE;
 928         do {
 929                 if (unuse_pmd(vma, pmd, address, end - address, offset, type, page))
 930                         return 1;
 931                 address = (address + PMD_SIZE) & PMD_MASK;
 932                 pmd++;
 933         } while (address < end);
 934         return 0;
 935 }
 936 
 937 static int unuse_vma(struct vm_area_struct * vma, pgd_t *pgdir,
     /*  */
 938         unsigned long start, unsigned long end,
 939         unsigned int type, unsigned long page)
 940 {
 941         while (start < end) {
 942                 if (unuse_pgd(vma, pgdir, start, end - start, type, page))
 943                         return 1;
 944                 start = (start + PGDIR_SIZE) & PGDIR_MASK;
 945                 pgdir++;
 946         }
 947         return 0;
 948 }
 949 
 950 static int unuse_process(struct task_struct * p, unsigned int type, unsigned long page)
     /*  */
 951 {
 952         struct vm_area_struct* vma;
 953 
 954         /*
 955          * Go through process' page directory.
 956          */
 957         vma = p->mm->mmap;
 958         while (vma) {
 959                 pgd_t * pgd = pgd_offset(p, vma->vm_start);
 960                 if (unuse_vma(vma, pgd, vma->vm_start, vma->vm_end, type, page))
 961                         return 1;
 962                 vma = vma->vm_next;
 963         }
 964         return 0;
 965 }
 966 
 967 /*
 968  * To avoid races, we repeat for each process after having
 969  * swapped something in. That gets rid of a few pesky races,
 970  * and "swapoff" isn't exactly timing critical.
 971  */
 972 static int try_to_unuse(unsigned int type)
     /*  */
 973 {
 974         int nr;
 975         unsigned long page = get_free_page(GFP_KERNEL);
 976 
 977         if (!page)
 978                 return -ENOMEM;
 979         nr = 0;
 980         while (nr < NR_TASKS) {
 981                 if (task[nr]) {
 982                         if (unuse_process(task[nr], type, page)) {
 983                                 page = get_free_page(GFP_KERNEL);
 984                                 if (!page)
 985                                         return -ENOMEM;
 986                                 continue;
 987                         }
 988                 }
 989                 nr++;
 990         }
 991         free_page(page);
 992         return 0;
 993 }
 994 
 995 asmlinkage int sys_swapoff(const char * specialfile)
     /*  */
 996 {
 997         struct swap_info_struct * p;
 998         struct inode * inode;
 999         struct file filp;
1000         int i, type, prev;
1001 
1002         if (!suser())
1003                 return -EPERM;
1004         i = namei(specialfile,&inode);
1005         if (i)
1006                 return i;
1007         prev = -1;
1008         for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
1009                 p = swap_info + type;
1010                 if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) {
1011                         if (p->swap_file) {
1012                                 if (p->swap_file == inode)
1013                                   break;
1014                         } else {
1015                                 if (S_ISBLK(inode->i_mode)
1016                                     && (p->swap_device == inode->i_rdev))
1017                                   break;
1018                         }
1019                 }
1020                 prev = type;
1021         }
1022         if (type < 0){
1023                 iput(inode);
1024                 return -EINVAL;
1025         }
1026         if (prev < 0) {
1027                 swap_list.head = p->next;
1028         } else {
1029                 swap_info[prev].next = p->next;
1030         }
1031         if (type == swap_list.next) {
1032                 /* just pick something that's safe... */
1033                 swap_list.next = swap_list.head;
1034         }
1035         p->flags = SWP_USED;
1036         i = try_to_unuse(type);
1037         if (i) {
1038                 iput(inode);
1039                 p->flags = SWP_WRITEOK;
1040                 return i;
1041         }
1042 
1043         if(p->swap_device){
1044                 memset(&filp, 0, sizeof(filp));         
1045                 filp.f_inode = inode;
1046                 filp.f_mode = 3; /* read write */
1047                 /* open it again to get fops */
1048                 if( !blkdev_open(inode, &filp) &&
1049                    filp.f_op && filp.f_op->release){
1050                         filp.f_op->release(inode,&filp);
1051                         filp.f_op->release(inode,&filp);
1052                 }
1053         }
1054         iput(inode);
1055 
1056         nr_swap_pages -= p->pages;
1057         iput(p->swap_file);
1058         p->swap_file = NULL;
1059         p->swap_device = 0;
1060         vfree(p->swap_map);
1061         p->swap_map = NULL;
1062         free_page((long) p->swap_lockmap);
1063         p->swap_lockmap = NULL;
1064         p->flags = 0;
1065         return 0;
1066 }
1067 
1068 /*
1069  * Written 01/25/92 by Simmule Turner, heavily changed by Linus.
1070  *
1071  * The swapon system call
1072  */
1073 asmlinkage int sys_swapon(const char * specialfile, int swap_flags)
     /*  */
1074 {
1075         struct swap_info_struct * p;
1076         struct inode * swap_inode;
1077         unsigned int type;
1078         int i, j, prev;
1079         int error;
1080         struct file filp;
1081         static int least_priority = 0;
1082 
1083         memset(&filp, 0, sizeof(filp));
1084         if (!suser())
1085                 return -EPERM;
1086         p = swap_info;
1087         for (type = 0 ; type < nr_swapfiles ; type++,p++)
1088                 if (!(p->flags & SWP_USED))
1089                         break;
1090         if (type >= MAX_SWAPFILES)
1091                 return -EPERM;
1092         if (type >= nr_swapfiles)
1093                 nr_swapfiles = type+1;
1094         p->flags = SWP_USED;
1095         p->swap_file = NULL;
1096         p->swap_device = 0;
1097         p->swap_map = NULL;
1098         p->swap_lockmap = NULL;
1099         p->lowest_bit = 0;
1100         p->highest_bit = 0;
1101         p->max = 1;
1102         p->next = -1;
1103         if (swap_flags & SWAP_FLAG_PREFER) {
1104                 p->prio =
1105                   (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
1106         } else {
1107                 p->prio = --least_priority;
1108         }
1109         error = namei(specialfile,&swap_inode);
1110         if (error)
1111                 goto bad_swap_2;
1112         p->swap_file = swap_inode;
1113         error = -EBUSY;
1114         if (swap_inode->i_count != 1)
1115                 goto bad_swap_2;
1116         error = -EINVAL;
1117 
1118         if (S_ISBLK(swap_inode->i_mode)) {
1119                 p->swap_device = swap_inode->i_rdev;
1120 
1121                 filp.f_inode = swap_inode;
1122                 filp.f_mode = 3; /* read write */
1123                 error = blkdev_open(swap_inode, &filp);
1124                 p->swap_file = NULL;
1125                 iput(swap_inode);
1126                 if(error)
1127                         goto bad_swap_2;
1128                 error = -ENODEV;
1129                 if (!p->swap_device)
1130                         goto bad_swap;
1131                 error = -EBUSY;
1132                 for (i = 0 ; i < nr_swapfiles ; i++) {
1133                         if (i == type)
1134                                 continue;
1135                         if (p->swap_device == swap_info[i].swap_device)
1136                                 goto bad_swap;
1137                 }
1138         } else if (!S_ISREG(swap_inode->i_mode))
1139                 goto bad_swap;
1140         p->swap_lockmap = (unsigned char *) get_free_page(GFP_USER);
1141         if (!p->swap_lockmap) {
1142                 printk("Unable to start swapping: out of memory :-)\n");
1143                 error = -ENOMEM;
1144                 goto bad_swap;
1145         }
1146         read_swap_page(SWP_ENTRY(type,0), (char *) p->swap_lockmap);
1147         if (memcmp("SWAP-SPACE",p->swap_lockmap+PAGE_SIZE-10,10)) {
1148                 printk("Unable to find swap-space signature\n");
1149                 error = -EINVAL;
1150                 goto bad_swap;
1151         }
1152         memset(p->swap_lockmap+PAGE_SIZE-10,0,10);
1153         j = 0;
1154         p->lowest_bit = 0;
1155         p->highest_bit = 0;
1156         for (i = 1 ; i < 8*PAGE_SIZE ; i++) {
1157                 if (test_bit(i,p->swap_lockmap)) {
1158                         if (!p->lowest_bit)
1159                                 p->lowest_bit = i;
1160                         p->highest_bit = i;
1161                         p->max = i+1;
1162                         j++;
1163                 }
1164         }
1165         if (!j) {
1166                 printk("Empty swap-file\n");
1167                 error = -EINVAL;
1168                 goto bad_swap;
1169         }
1170         p->swap_map = (unsigned char *) vmalloc(p->max);
1171         if (!p->swap_map) {
1172                 error = -ENOMEM;
1173                 goto bad_swap;
1174         }
1175         for (i = 1 ; i < p->max ; i++) {
1176                 if (test_bit(i,p->swap_lockmap))
1177                         p->swap_map[i] = 0;
1178                 else
1179                         p->swap_map[i] = 0x80;
1180         }
1181         p->swap_map[0] = 0x80;
1182         memset(p->swap_lockmap,0,PAGE_SIZE);
1183         p->flags = SWP_WRITEOK;
1184         p->pages = j;
1185         nr_swap_pages += j;
1186         printk("Adding Swap: %dk swap-space\n",j<<(PAGE_SHIFT-10));
1187 
1188         /* insert swap space into swap_list: */
1189         prev = -1;
1190         for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
1191                 if (p->prio >= swap_info[i].prio) {
1192                         break;
1193                 }
1194                 prev = i;
1195         }
1196         p->next = i;
1197         if (prev < 0) {
1198                 swap_list.head = swap_list.next = p - swap_info;
1199         } else {
1200                 swap_info[prev].next = p - swap_info;
1201         }
1202         return 0;
1203 bad_swap:
1204         if(filp.f_op && filp.f_op->release)
1205                 filp.f_op->release(filp.f_inode,&filp);
1206 bad_swap_2:
1207         free_page((long) p->swap_lockmap);
1208         vfree(p->swap_map);
1209         iput(p->swap_file);
1210         p->swap_device = 0;
1211         p->swap_file = NULL;
1212         p->swap_map = NULL;
1213         p->swap_lockmap = NULL;
1214         p->flags = 0;
1215         return error;
1216 }
1217 
1218 void si_swapinfo(struct sysinfo *val)
     /*  */
1219 {
1220         unsigned int i, j;
1221 
1222         val->freeswap = val->totalswap = 0;
1223         for (i = 0; i < nr_swapfiles; i++) {
1224                 if ((swap_info[i].flags & SWP_WRITEOK) != SWP_WRITEOK)
1225                         continue;
1226                 for (j = 0; j < swap_info[i].max; ++j)
1227                         switch (swap_info[i].swap_map[j]) {
1228                                 case 128:
1229                                         continue;
1230                                 case 0:
1231                                         ++val->freeswap;
1232                                 default:
1233                                         ++val->totalswap;
1234                         }
1235         }
1236         val->freeswap <<= PAGE_SHIFT;
1237         val->totalswap <<= PAGE_SHIFT;
1238         return;
1239 }
1240 
1241 #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
1242 
1243 /*
1244  * set up the free-area data structures:
1245  *   - mark all pages MAP_PAGE_RESERVED
1246  *   - mark all memory queues empty
1247  *   - clear the memory bitmaps
1248  */
1249 unsigned long free_area_init(unsigned long start_mem, unsigned long end_mem)
     /*  */
1250 {
1251         mem_map_t * p;
1252         unsigned long mask = PAGE_MASK;
1253         int i;
1254 
1255         /*
1256          * select nr of pages we try to keep free for important stuff
1257          * with a minimum of 16 pages. This is totally arbitrary
1258          */
1259         i = (end_mem - PAGE_OFFSET) >> (PAGE_SHIFT+6);
1260         if (i < 16)
1261                 i = 16;
1262         min_free_pages = i;
1263         start_mem = init_swap_cache(start_mem, end_mem);
1264         mem_map = (mem_map_t *) start_mem;
1265         p = mem_map + MAP_NR(end_mem);
1266         start_mem = LONG_ALIGN((unsigned long) p);
1267         while (p > mem_map)
1268                 *--p = MAP_PAGE_RESERVED;
1269 
1270         for (i = 0 ; i < NR_MEM_LISTS ; i++) {
1271                 unsigned long bitmap_size;
1272                 free_area_list[i].prev = free_area_list[i].next = &free_area_list[i];
1273                 mask += mask;
1274                 end_mem = (end_mem + ~mask) & mask;
1275                 bitmap_size = (end_mem - PAGE_OFFSET) >> (PAGE_SHIFT + i);
1276                 bitmap_size = (bitmap_size + 7) >> 3;
1277                 bitmap_size = LONG_ALIGN(bitmap_size);
1278                 free_area_map[i] = (unsigned char *) start_mem;
1279                 memset((void *) start_mem, 0, bitmap_size);
1280                 start_mem += bitmap_size;
1281         }
1282         return start_mem;
1283 }
/* */
root/mm/swap.c

DEFINITIONS
