mm/swap.c

/* */
This source file includes following definitions.
show_swap_cache_info
add_to_swap_cache
init_swap_cache
rw_swap_page
get_swap_page
swap_duplicate
swap_free
swap_in
try_to_swap_out
swap_out_process
swap_out
try_to_free_page
add_mem_queue
remove_mem_queue
free_pages_ok
check_free_buffers
free_pages
mark_used
__get_free_pages
__get_dma_pages
show_free_areas
try_to_unuse
sys_swapoff
sys_swapon
si_swapinfo
free_area_init
   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6 
   7 /*
   8  * This file should contain most things doing the swapping from/to disk.
   9  * Started 18.12.91
  10  */
  11 
  12 #include <linux/mm.h>
  13 #include <linux/sched.h>
  14 #include <linux/head.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kernel_stat.h>
  17 #include <linux/errno.h>
  18 #include <linux/string.h>
  19 #include <linux/stat.h>
  20 #include <linux/fs.h>
  21 
  22 #include <asm/dma.h>
  23 #include <asm/system.h> /* for cli()/sti() */
  24 #include <asm/bitops.h>
  25 
  26 #define MAX_SWAPFILES 8
  27 
  28 #define SWP_USED        1
  29 #define SWP_WRITEOK     3
  30 
  31 #define SWP_TYPE(entry) (((entry) & 0xfe) >> 1)
  32 #define SWP_OFFSET(entry) ((entry) >> PAGE_SHIFT)
  33 #define SWP_ENTRY(type,offset) (((type) << 1) | ((offset) << PAGE_SHIFT))
  34 
  35 int min_free_pages = 20;
  36 
  37 static int nr_swapfiles = 0;
  38 static struct wait_queue * lock_queue = NULL;
  39 
  40 static struct swap_info_struct {
  41         unsigned long flags;
  42         struct inode * swap_file;
  43         unsigned int swap_device;
  44         unsigned char * swap_map;
  45         unsigned char * swap_lockmap;
  46         int pages;
  47         int lowest_bit;
  48         int highest_bit;
  49         unsigned long max;
  50 } swap_info[MAX_SWAPFILES];
  51 
  52 extern int shm_swap (int);
  53 
  54 unsigned long *swap_cache;
  55 
  56 #ifdef SWAP_CACHE_INFO
  57 unsigned long swap_cache_add_total = 0;
  58 unsigned long swap_cache_add_success = 0;
  59 unsigned long swap_cache_del_total = 0;
  60 unsigned long swap_cache_del_success = 0;
  61 unsigned long swap_cache_find_total = 0;
  62 unsigned long swap_cache_find_success = 0;
  63 
  64 extern inline void show_swap_cache_info(void)
     /*  */
  65 {
  66         printk("Swap cache: add %ld/%ld, delete %ld/%ld, find %ld/%ld\n",
  67                 swap_cache_add_total, swap_cache_add_success, 
  68                 swap_cache_del_total, swap_cache_del_success,
  69                 swap_cache_find_total, swap_cache_find_success);
  70 }
  71 #endif
  72 
  73 extern inline int add_to_swap_cache(unsigned long addr, unsigned long entry)
     /*  */
  74 {
  75         struct swap_info_struct * p = &swap_info[SWP_TYPE(entry)];
  76         
  77 #ifdef SWAP_CACHE_INFO
  78         swap_cache_add_total++;
  79 #endif
  80         if ((p->flags & SWP_WRITEOK) == SWP_WRITEOK) { 
  81                 __asm__ __volatile__ (
  82                                       "xchgl %0,%1\n"
  83                                       : "=m" (swap_cache[addr >> PAGE_SHIFT]),
  84                                        "=r" (entry)
  85                                       : "0" (swap_cache[addr >> PAGE_SHIFT]),
  86                                        "1" (entry)
  87                                       );
  88                 if (entry)  {
  89                         printk("swap_cache: replacing non-NULL entry\n");
  90                 }
  91 #ifdef SWAP_CACHE_INFO
  92                 swap_cache_add_success++;
  93 #endif
  94                 return 1;
  95         }
  96         return 0;
  97 }
  98 
  99 static unsigned long init_swap_cache(unsigned long mem_start,
     /*  */
 100         unsigned long mem_end)
 101 {
 102         unsigned long swap_cache_size;
 103 
 104         mem_start = (mem_start + 15) & ~15;
 105         swap_cache = (unsigned long *) mem_start;
 106         swap_cache_size = mem_end >> PAGE_SHIFT;
 107         memset(swap_cache, 0, swap_cache_size * sizeof (unsigned long));
 108         return (unsigned long) (swap_cache + swap_cache_size);
 109 }
 110 
 111 void rw_swap_page(int rw, unsigned long entry, char * buf)
     /*  */
 112 {
 113         unsigned long type, offset;
 114         struct swap_info_struct * p;
 115 
 116         type = SWP_TYPE(entry);
 117         if (type >= nr_swapfiles) {
 118                 printk("Internal error: bad swap-device\n");
 119                 return;
 120         }
 121         p = &swap_info[type];
 122         offset = SWP_OFFSET(entry);
 123         if (offset >= p->max) {
 124                 printk("rw_swap_page: weirdness\n");
 125                 return;
 126         }
 127         if (!(p->flags & SWP_USED)) {
 128                 printk("Trying to swap to unused swap-device\n");
 129                 return;
 130         }
 131         while (set_bit(offset,p->swap_lockmap))
 132                 sleep_on(&lock_queue);
 133         if (rw == READ)
 134                 kstat.pswpin++;
 135         else
 136                 kstat.pswpout++;
 137         if (p->swap_device) {
 138                 ll_rw_page(rw,p->swap_device,offset,buf);
 139         } else if (p->swap_file) {
 140                 struct inode *swapf = p->swap_file;
 141                 unsigned int zones[8];
 142                 int i;
 143                 if (swapf->i_op->bmap == NULL
 144                         && swapf->i_op->smap != NULL){
 145                         /*
 146                                 With MsDOS, we use msdos_smap which return
 147                                 a sector number (not a cluster or block number).
 148                                 It is a patch to enable the UMSDOS project.
 149                                 Other people are working on better solution.
 150 
 151                                 It sounds like ll_rw_swap_file defined
 152                                 it operation size (sector size) based on
 153                                 PAGE_SIZE and the number of block to read.
 154                                 So using bmap or smap should work even if
 155                                 smap will require more blocks.
 156                         */
 157                         int j;
 158                         unsigned int block = offset << 3;
 159 
 160                         for (i=0, j=0; j< PAGE_SIZE ; i++, j += 512){
 161                                 if (!(zones[i] = swapf->i_op->smap(swapf,block++))) {
 162                                         printk("rw_swap_page: bad swap file\n");
 163                                         return;
 164                                 }
 165                         }
 166                 }else{
 167                         int j;
 168                         unsigned int block = offset
 169                                 << (12 - swapf->i_sb->s_blocksize_bits);
 170 
 171                         for (i=0, j=0; j< PAGE_SIZE ; i++, j +=swapf->i_sb->s_blocksize)
 172                                 if (!(zones[i] = bmap(swapf,block++))) {
 173                                         printk("rw_swap_page: bad swap file\n");
 174                                         return;
 175                                 }
 176                 }
 177                 ll_rw_swap_file(rw,swapf->i_dev, zones, i,buf);
 178         } else
 179                 printk("re_swap_page: no swap file or device\n");
 180         if (offset && !clear_bit(offset,p->swap_lockmap))
 181                 printk("rw_swap_page: lock already cleared\n");
 182         wake_up(&lock_queue);
 183 }
 184 
 185 unsigned int get_swap_page(void)
     /*  */
 186 {
 187         struct swap_info_struct * p;
 188         unsigned int offset, type;
 189 
 190         p = swap_info;
 191         for (type = 0 ; type < nr_swapfiles ; type++,p++) {
 192                 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
 193                         continue;
 194                 for (offset = p->lowest_bit; offset <= p->highest_bit ; offset++) {
 195                         if (p->swap_map[offset])
 196                                 continue;
 197                         p->swap_map[offset] = 1;
 198                         nr_swap_pages--;
 199                         if (offset == p->highest_bit)
 200                                 p->highest_bit--;
 201                         p->lowest_bit = offset;
 202                         return SWP_ENTRY(type,offset);
 203                 }
 204         }
 205         return 0;
 206 }
 207 
 208 unsigned long swap_duplicate(unsigned long entry)
     /*  */
 209 {
 210         struct swap_info_struct * p;
 211         unsigned long offset, type;
 212 
 213         if (!entry)
 214                 return 0;
 215         offset = SWP_OFFSET(entry);
 216         type = SWP_TYPE(entry);
 217         if (type == SHM_SWP_TYPE)
 218                 return entry;
 219         if (type >= nr_swapfiles) {
 220                 printk("Trying to duplicate nonexistent swap-page\n");
 221                 return 0;
 222         }
 223         p = type + swap_info;
 224         if (offset >= p->max) {
 225                 printk("swap_duplicate: weirdness\n");
 226                 return 0;
 227         }
 228         if (!p->swap_map[offset]) {
 229                 printk("swap_duplicate: trying to duplicate unused page\n");
 230                 return 0;
 231         }
 232         p->swap_map[offset]++;
 233         return entry;
 234 }
 235 
 236 void swap_free(unsigned long entry)
     /*  */
 237 {
 238         struct swap_info_struct * p;
 239         unsigned long offset, type;
 240 
 241         if (!entry)
 242                 return;
 243         type = SWP_TYPE(entry);
 244         if (type == SHM_SWP_TYPE)
 245                 return;
 246         if (type >= nr_swapfiles) {
 247                 printk("Trying to free nonexistent swap-page\n");
 248                 return;
 249         }
 250         p = & swap_info[type];
 251         offset = SWP_OFFSET(entry);
 252         if (offset >= p->max) {
 253                 printk("swap_free: weirdness\n");
 254                 return;
 255         }
 256         if (!(p->flags & SWP_USED)) {
 257                 printk("Trying to free swap from unused swap-device\n");
 258                 return;
 259         }
 260         while (set_bit(offset,p->swap_lockmap))
 261                 sleep_on(&lock_queue);
 262         if (offset < p->lowest_bit)
 263                 p->lowest_bit = offset;
 264         if (offset > p->highest_bit)
 265                 p->highest_bit = offset;
 266         if (!p->swap_map[offset])
 267                 printk("swap_free: swap-space map bad (entry %08lx)\n",entry);
 268         else
 269                 if (!--p->swap_map[offset])
 270                         nr_swap_pages++;
 271         if (!clear_bit(offset,p->swap_lockmap))
 272                 printk("swap_free: lock already cleared\n");
 273         wake_up(&lock_queue);
 274 }
 275 
 276 unsigned long swap_in(unsigned long entry)
     /*  */
 277 {
 278         unsigned long page;
 279 
 280         if (!(page = get_free_page(GFP_KERNEL))) {
 281                 oom(current);
 282                 return BAD_PAGE;
 283         }
 284         read_swap_page(entry, (char *) page);
 285         if (add_to_swap_cache(page, entry))
 286                 return page | PAGE_PRESENT;
 287         swap_free(entry);
 288         return page | PAGE_DIRTY | PAGE_PRESENT;
 289 }
 290 
 291 static inline int try_to_swap_out(struct vm_area_struct* vma, unsigned offset, unsigned long * table_ptr)
     /*  */
 292 {
 293         unsigned long page, entry;
 294 
 295         page = *table_ptr;
 296         if (!(PAGE_PRESENT & page))
 297                 return 0;
 298         if (page >= high_memory)
 299                 return 0;
 300         if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
 301                 return 0;
 302         
 303         if ((PAGE_DIRTY & page) && delete_from_swap_cache(page))  {
 304                 *table_ptr &= ~PAGE_ACCESSED;
 305                 return 0;
 306         }
 307         if (PAGE_ACCESSED & page) {
 308                 *table_ptr &= ~PAGE_ACCESSED;
 309                 return 0;
 310         }
 311         if (PAGE_DIRTY & page) {
 312                 page &= PAGE_MASK;
 313                 if (mem_map[MAP_NR(page)] != 1)
 314                         return 0;
 315                 if (vma->vm_ops && vma->vm_ops->swapout)
 316                         vma->vm_ops->swapout(vma, offset, table_ptr);
 317                 else
 318                 {
 319                         if (!(entry = get_swap_page()))
 320                                 return 0;
 321                         *table_ptr = entry;
 322                         invalidate();
 323                         write_swap_page(entry, (char *) page);
 324                 }
 325                 free_page(page);
 326                 return 1 + mem_map[MAP_NR(page)];
 327         }
 328         if ((entry = find_in_swap_cache(page)))  {
 329                 if (mem_map[MAP_NR(page)] != 1) {
 330                         *table_ptr |= PAGE_DIRTY;
 331                         printk("Aiee.. duplicated cached swap-cache entry\n");
 332                         return 0;
 333                 }
 334                 *table_ptr = entry;
 335                 invalidate();
 336                 free_page(page & PAGE_MASK);
 337                 return 1;
 338         } 
 339         page &= PAGE_MASK;
 340         *table_ptr = 0;
 341         invalidate();
 342         free_page(page);
 343         return 1 + mem_map[MAP_NR(page)];
 344 }
 345 
 346 /*
 347  * A new implementation of swap_out().  We do not swap complete processes,
 348  * but only a small number of blocks, before we continue with the next
 349  * process.  The number of blocks actually swapped is determined on the
 350  * number of page faults, that this process actually had in the last time,
 351  * so we won't swap heavily used processes all the time ...
 352  *
 353  * Note: the priority argument is a hint on much CPU to waste with the
 354  *       swap block search, not a hint, of how much blocks to swap with
 355  *       each process.
 356  *
 357  * (C) 1993 Kai Petzke, wpp@marie.physik.tu-berlin.de
 358  */
 359 
 360 /*
 361  * These are the minimum and maximum number of pages to swap from one process,
 362  * before proceeding to the next:
 363  */
 364 #define SWAP_MIN        4
 365 #define SWAP_MAX        32
 366 
 367 /*
 368  * The actual number of pages to swap is determined as:
 369  * SWAP_RATIO / (number of recent major page faults)
 370  */
 371 #define SWAP_RATIO      128
 372 
 373 static int swap_out_process(struct task_struct * p)
     /*  */
 374 {
 375         unsigned long address;
 376         unsigned long offset;
 377         unsigned long *pgdir;
 378         unsigned long pg_table;
 379         struct vm_area_struct* vma;
 380 
 381         /*
 382          * Go through process' page directory.
 383          */
 384         address = p->mm->swap_address;
 385         p->mm->swap_address = 0;
 386 
 387         /*
 388          * Find the proper vm-area
 389          */
 390         vma = p->mm->mmap;
 391         for (;;) {
 392                 if (!vma)
 393                         return 0;
 394                 if (address <= vma->vm_end)
 395                         break;
 396                 vma = vma->vm_next;
 397         }
 398         if (address < vma->vm_start)
 399                 address = vma->vm_start;
 400 
 401         pgdir = (address >> PGDIR_SHIFT) + (unsigned long *) p->tss.cr3;
 402         offset = address & ~PGDIR_MASK;
 403         address &= PGDIR_MASK;
 404         for ( ; address < TASK_SIZE ;
 405         pgdir++, address = address + PGDIR_SIZE, offset = 0) {
 406                 pg_table = *pgdir;
 407                 if (pg_table >= high_memory)
 408                         continue;
 409                 if (mem_map[MAP_NR(pg_table)] & MAP_PAGE_RESERVED)
 410                         continue;
 411                 if (!(PAGE_PRESENT & pg_table)) {
 412                         printk("swap_out_process (%s): bad page-table at vm %08lx: %08lx\n",
 413                                         p->comm, address + offset, pg_table);
 414                         *pgdir = 0;
 415                         continue;
 416                 }
 417                 pg_table &= 0xfffff000;
 418 
 419                 /*
 420                  * Go through this page table.
 421                  */
 422                 for( ; offset < ~PGDIR_MASK ; offset += PAGE_SIZE) {
 423                         /*
 424                          * Update vma again..
 425                          */
 426                         for (;;) {
 427                                 if (address+offset < vma->vm_end)
 428                                         break;
 429                                 vma = vma->vm_next;
 430                                 if (!vma)
 431                                         return 0;
 432                         }
 433 
 434                         switch(try_to_swap_out(vma, offset+address-vma->vm_start, (unsigned long *) (pg_table + (offset >> 10)))) {
 435                                 case 0:
 436                                         break;
 437 
 438                                 case 1:
 439                                         p->mm->rss--;
 440                                         /* continue with the following page the next time */
 441                                         p->mm->swap_address = address + offset + PAGE_SIZE;
 442                                         return 1;
 443 
 444                                 default:
 445                                         p->mm->rss--;
 446                                         break;
 447                         }
 448                 }
 449         }
 450         /*
 451          * Finish work with this process, if we reached the end of the page
 452          * directory.
 453          */
 454         return 0;
 455 }
 456 
 457 static int swap_out(unsigned int priority)
     /*  */
 458 {
 459         static int swap_task;
 460         int loop;
 461         int counter = NR_TASKS * 2 >> priority;
 462         struct task_struct *p;
 463 
 464         counter = NR_TASKS * 2 >> priority;
 465         for(; counter >= 0; counter--, swap_task++) {
 466                 /*
 467                  * Check that swap_task is suitable for swapping.  If not, look for
 468                  * the next suitable process.
 469                  */
 470                 loop = 0;
 471                 while(1) {
 472                         if (swap_task >= NR_TASKS) {
 473                                 swap_task = 1;
 474                                 if (loop)
 475                                         /* all processes are unswappable or already swapped out */
 476                                         return 0;
 477                                 loop = 1;
 478                         }
 479 
 480                         p = task[swap_task];
 481                         if (p && p->mm->swappable && p->mm->rss)
 482                                 break;
 483 
 484                         swap_task++;
 485                 }
 486 
 487                 /*
 488                  * Determine the number of pages to swap from this process.
 489                  */
 490                 if (!p->mm->swap_cnt) {
 491                         p->mm->dec_flt = (p->mm->dec_flt * 3) / 4 + p->mm->maj_flt - p->mm->old_maj_flt;
 492                         p->mm->old_maj_flt = p->mm->maj_flt;
 493 
 494                         if (p->mm->dec_flt >= SWAP_RATIO / SWAP_MIN) {
 495                                 p->mm->dec_flt = SWAP_RATIO / SWAP_MIN;
 496                                 p->mm->swap_cnt = SWAP_MIN;
 497                         } else if (p->mm->dec_flt <= SWAP_RATIO / SWAP_MAX)
 498                                 p->mm->swap_cnt = SWAP_MAX;
 499                         else
 500                                 p->mm->swap_cnt = SWAP_RATIO / p->mm->dec_flt;
 501                 }
 502                 if (swap_out_process(p)) {
 503                         if ((--p->mm->swap_cnt) == 0)
 504                                 swap_task++;
 505                         return 1;
 506                 }
 507         }
 508         return 0;
 509 }
 510 
 511 static int try_to_free_page(int priority)
     /*  */
 512 {
 513         int i=6;
 514 
 515         while (i--) {
 516                 if (priority != GFP_NOBUFFER && shrink_buffers(i))
 517                         return 1;
 518                 if (shm_swap(i))
 519                         return 1;
 520                 if (swap_out(i))
 521                         return 1;
 522         }
 523         return 0;
 524 }
 525 
 526 static inline void add_mem_queue(struct mem_list * head, struct mem_list * entry)
     /*  */
 527 {
 528         entry->prev = head;
 529         (entry->next = head->next)->prev = entry;
 530         head->next = entry;
 531 }
 532 
 533 static inline void remove_mem_queue(struct mem_list * head, struct mem_list * entry)
     /*  */
 534 {
 535         entry->next->prev = entry->prev;
 536         entry->prev->next = entry->next;
 537 }
 538 
 539 /*
 540  * Free_page() adds the page to the free lists. This is optimized for
 541  * fast normal cases (no error jumps taken normally).
 542  *
 543  * The way to optimize jumps for gcc-2.2.2 is to:
 544  *  - select the "normal" case and put it inside the if () { XXX }
 545  *  - no else-statements if you can avoid them
 546  *
 547  * With the above two rules, you get a straight-line execution path
 548  * for the normal case, giving better asm-code.
 549  */
 550 
 551 /*
 552  * Buddy system. Hairy. You really aren't expected to understand this
 553  */
 554 static inline void free_pages_ok(unsigned long addr, unsigned long order)
     /*  */
 555 {
 556         unsigned long index = addr >> (PAGE_SHIFT + 1 + order);
 557         unsigned long mask = PAGE_MASK << order;
 558 
 559         addr &= mask;
 560         nr_free_pages += 1 << order;
 561         while (order < NR_MEM_LISTS-1) {
 562                 if (!change_bit(index, free_area_map[order]))
 563                         break;
 564                 remove_mem_queue(free_area_list+order, (struct mem_list *) (addr ^ (1+~mask)));
 565                 order++;
 566                 index >>= 1;
 567                 mask <<= 1;
 568                 addr &= mask;
 569         }
 570         add_mem_queue(free_area_list+order, (struct mem_list *) addr);
 571 }
 572 
 573 static inline void check_free_buffers(unsigned long addr)
     /*  */
 574 {
 575         struct buffer_head * bh;
 576 
 577         bh = buffer_pages[MAP_NR(addr)];
 578         if (bh) {
 579                 struct buffer_head *tmp = bh;
 580                 do {
 581                         if (tmp->b_list == BUF_SHARED && tmp->b_dev != 0xffff)
 582                                 refile_buffer(tmp);
 583                         tmp = tmp->b_this_page;
 584                 } while (tmp != bh);
 585         }
 586 }
 587 
 588 void free_pages(unsigned long addr, unsigned long order)
     /*  */
 589 {
 590         if (addr < high_memory) {
 591                 unsigned long flag;
 592                 unsigned short * map = mem_map + MAP_NR(addr);
 593                 if (*map) {
 594                         if (!(*map & MAP_PAGE_RESERVED)) {
 595                                 save_flags(flag);
 596                                 cli();
 597                                 if (!--*map)  {
 598                                         free_pages_ok(addr, order);
 599                                         delete_from_swap_cache(addr);
 600                                 }
 601                                 restore_flags(flag);
 602                                 if (*map == 1)
 603                                         check_free_buffers(addr);
 604                         }
 605                         return;
 606                 }
 607                 printk("Trying to free free memory (%08lx): memory probably corrupted\n",addr);
 608                 printk("PC = %08lx\n",*(((unsigned long *)&addr)-1));
 609                 return;
 610         }
 611 }
 612 
 613 /*
 614  * Some ugly macros to speed up __get_free_pages()..
 615  */
 616 #define RMQUEUE(order) \
 617 do { struct mem_list * queue = free_area_list+order; \
 618      unsigned long new_order = order; \
 619         do { struct mem_list *next = queue->next; \
 620                 if (queue != next) { \
 621                         (queue->next = next->next)->prev = queue; \
 622                         mark_used((unsigned long) next, new_order); \
 623                         nr_free_pages -= 1 << order; \
 624                         restore_flags(flags); \
 625                         EXPAND(next, order, new_order); \
 626                         return (unsigned long) next; \
 627                 } new_order++; queue++; \
 628         } while (new_order < NR_MEM_LISTS); \
 629 } while (0)
 630 
 631 static inline int mark_used(unsigned long addr, unsigned long order)
     /*  */
 632 {
 633         return change_bit(addr >> (PAGE_SHIFT+1+order), free_area_map[order]);
 634 }
 635 
 636 #define EXPAND(addr,low,high) \
 637 do { unsigned long size = PAGE_SIZE << high; \
 638         while (high > low) { \
 639                 high--; size >>= 1; cli(); \
 640                 add_mem_queue(free_area_list+high, addr); \
 641                 mark_used((unsigned long) addr, high); \
 642                 restore_flags(flags); \
 643                 addr = (struct mem_list *) (size + (unsigned long) addr); \
 644         } mem_map[MAP_NR((unsigned long) addr)] = 1; \
 645 } while (0)
 646 
 647 unsigned long __get_free_pages(int priority, unsigned long order)
     /*  */
 648 {
 649         unsigned long flags;
 650         int reserved_pages;
 651 
 652         if (intr_count && priority != GFP_ATOMIC) {
 653                 static int count = 0;
 654                 if (++count < 5) {
 655                         printk("gfp called nonatomically from interrupt %p\n",
 656                                 __builtin_return_address(0));
 657                         priority = GFP_ATOMIC;
 658                 }
 659         }
 660         reserved_pages = 5;
 661         if (priority != GFP_NFS)
 662                 reserved_pages = min_free_pages;
 663         save_flags(flags);
 664 repeat:
 665         cli();
 666         if ((priority==GFP_ATOMIC) || nr_free_pages > reserved_pages) {
 667                 RMQUEUE(order);
 668                 restore_flags(flags);
 669                 return 0;
 670         }
 671         restore_flags(flags);
 672         if (priority != GFP_BUFFER && try_to_free_page(priority))
 673                 goto repeat;
 674         return 0;
 675 }
 676 
 677 /*
 678  * Yes, I know this is ugly. Don't tell me.
 679  */
 680 unsigned long __get_dma_pages(int priority, unsigned long order)
     /*  */
 681 {
 682         unsigned long list = 0;
 683         unsigned long result;
 684         unsigned long limit = MAX_DMA_ADDRESS;
 685 
 686         /* if (EISA_bus) limit = ~0UL; */
 687         if (priority != GFP_ATOMIC)
 688                 priority = GFP_BUFFER;
 689         for (;;) {
 690                 result = __get_free_pages(priority, order);
 691                 if (result < limit) /* covers failure as well */
 692                         break;
 693                 *(unsigned long *) result = list;
 694                 list = result;
 695         }
 696         while (list) {
 697                 unsigned long tmp = list;
 698                 list = *(unsigned long *) list;
 699                 free_pages(tmp, order);
 700         }
 701         return result;
 702 }
 703 
 704 /*
 705  * Show free area list (used inside shift_scroll-lock stuff)
 706  * We also calculate the percentage fragmentation. We do this by counting the
 707  * memory on each free list with the exception of the first item on the list.
 708  */
 709 void show_free_areas(void)
     /*  */
 710 {
 711         unsigned long order, flags;
 712         unsigned long total = 0;
 713 
 714         printk("Free pages:      %6dkB\n ( ",nr_free_pages<<(PAGE_SHIFT-10));
 715         save_flags(flags);
 716         cli();
 717         for (order=0 ; order < NR_MEM_LISTS; order++) {
 718                 struct mem_list * tmp;
 719                 unsigned long nr = 0;
 720                 for (tmp = free_area_list[order].next ; tmp != free_area_list + order ; tmp = tmp->next) {
 721                         nr ++;
 722                 }
 723                 total += nr * (4 << order);
 724                 printk("%lu*%ukB ", nr, 4 << order);
 725         }
 726         restore_flags(flags);
 727         printk("= %lukB)\n", total);
 728 #ifdef SWAP_CACHE_INFO
 729         show_swap_cache_info();
 730 #endif  
 731 }
 732 
 733 /*
 734  * Trying to stop swapping from a file is fraught with races, so
 735  * we repeat quite a bit here when we have to pause. swapoff()
 736  * isn't exactly timing-critical, so who cares?
 737  */
 738 static int try_to_unuse(unsigned int type)
     /*  */
 739 {
 740         int nr, pgt, pg;
 741         unsigned long page, *ppage;
 742         unsigned long tmp = 0;
 743         struct task_struct *p;
 744 
 745         nr = 0;
 746         
 747 /*
 748  * When we have to sleep, we restart the whole algorithm from the same
 749  * task we stopped in. That at least rids us of all races.
 750  */
 751 repeat:
 752         for (; nr < NR_TASKS ; nr++) {
 753                 p = task[nr];
 754                 if (!p)
 755                         continue;
 756                 for (pgt = 0 ; pgt < PTRS_PER_PAGE ; pgt++) {
 757                         ppage = pgt + ((unsigned long *) p->tss.cr3);
 758                         page = *ppage;
 759                         if (!page)
 760                                 continue;
 761                         if (!(page & PAGE_PRESENT) || (page >= high_memory))
 762                                 continue;
 763                         if (mem_map[MAP_NR(page)] & MAP_PAGE_RESERVED)
 764                                 continue;
 765                         ppage = (unsigned long *) (page & PAGE_MASK);   
 766                         for (pg = 0 ; pg < PTRS_PER_PAGE ; pg++,ppage++) {
 767                                 page = *ppage;
 768                                 if (!page)
 769                                         continue;
 770                                 if (page & PAGE_PRESENT) {
 771                                         if (page >= high_memory)
 772                                                 continue;
 773                                         if (!(page = in_swap_cache(page)))
 774                                                 continue;
 775                                         if (SWP_TYPE(page) != type)
 776                                                 continue;
 777                                         *ppage |= PAGE_DIRTY;
 778                                         delete_from_swap_cache(*ppage);
 779                                         continue;
 780                                 }
 781                                 if (SWP_TYPE(page) != type)
 782                                         continue;
 783                                 if (!tmp) {
 784                                         if (!(tmp = __get_free_page(GFP_KERNEL)))
 785                                                 return -ENOMEM;
 786                                         goto repeat;
 787                                 }
 788                                 read_swap_page(page, (char *) tmp);
 789                                 if (*ppage == page) {
 790                                         *ppage = tmp | (PAGE_DIRTY | PAGE_PRIVATE);
 791                                         ++p->mm->rss;
 792                                         swap_free(page);
 793                                         tmp = 0;
 794                                 }
 795                                 goto repeat;
 796                         }
 797                 }
 798         }
 799         free_page(tmp);
 800         return 0;
 801 }
 802 
 803 asmlinkage int sys_swapoff(const char * specialfile)
     /*  */
 804 {
 805         struct swap_info_struct * p;
 806         struct inode * inode;
 807         unsigned int type;
 808         struct file filp;
 809         int i;
 810 
 811         if (!suser())
 812                 return -EPERM;
 813         i = namei(specialfile,&inode);
 814         if (i)
 815                 return i;
 816         p = swap_info;
 817         for (type = 0 ; type < nr_swapfiles ; type++,p++) {
 818                 if ((p->flags & SWP_WRITEOK) != SWP_WRITEOK)
 819                         continue;
 820                 if (p->swap_file) {
 821                         if (p->swap_file == inode)
 822                                 break;
 823                 } else {
 824                         if (!S_ISBLK(inode->i_mode))
 825                                 continue;
 826                         if (p->swap_device == inode->i_rdev)
 827                                 break;
 828                 }
 829         }
 830 
 831         if (type >= nr_swapfiles){
 832                 iput(inode);
 833                 return -EINVAL;
 834         }
 835         p->flags = SWP_USED;
 836         i = try_to_unuse(type);
 837         if (i) {
 838                 iput(inode);
 839                 p->flags = SWP_WRITEOK;
 840                 return i;
 841         }
 842 
 843         if(p->swap_device){
 844                 memset(&filp, 0, sizeof(filp));         
 845                 filp.f_inode = inode;
 846                 filp.f_mode = 3; /* read write */
 847                 /* open it again to get fops */
 848                 if( !blkdev_open(inode, &filp) &&
 849                    filp.f_op && filp.f_op->release){
 850                         filp.f_op->release(inode,&filp);
 851                         filp.f_op->release(inode,&filp);
 852                 }
 853         }
 854         iput(inode);
 855 
 856         nr_swap_pages -= p->pages;
 857         iput(p->swap_file);
 858         p->swap_file = NULL;
 859         p->swap_device = 0;
 860         vfree(p->swap_map);
 861         p->swap_map = NULL;
 862         free_page((long) p->swap_lockmap);
 863         p->swap_lockmap = NULL;
 864         p->flags = 0;
 865         return 0;
 866 }
 867 
 868 /*
 869  * Written 01/25/92 by Simmule Turner, heavily changed by Linus.
 870  *
 871  * The swapon system call
 872  */
 873 asmlinkage int sys_swapon(const char * specialfile)
     /*  */
 874 {
 875         struct swap_info_struct * p;
 876         struct inode * swap_inode;
 877         unsigned int type;
 878         int i,j;
 879         int error;
 880         struct file filp;
 881 
 882         memset(&filp, 0, sizeof(filp));
 883         if (!suser())
 884                 return -EPERM;
 885         p = swap_info;
 886         for (type = 0 ; type < nr_swapfiles ; type++,p++)
 887                 if (!(p->flags & SWP_USED))
 888                         break;
 889         if (type >= MAX_SWAPFILES)
 890                 return -EPERM;
 891         if (type >= nr_swapfiles)
 892                 nr_swapfiles = type+1;
 893         p->flags = SWP_USED;
 894         p->swap_file = NULL;
 895         p->swap_device = 0;
 896         p->swap_map = NULL;
 897         p->swap_lockmap = NULL;
 898         p->lowest_bit = 0;
 899         p->highest_bit = 0;
 900         p->max = 1;
 901         error = namei(specialfile,&swap_inode);
 902         if (error)
 903                 goto bad_swap_2;
 904         p->swap_file = swap_inode;
 905         error = -EBUSY;
 906         if (swap_inode->i_count != 1)
 907                 goto bad_swap_2;
 908         error = -EINVAL;
 909 
 910         if (S_ISBLK(swap_inode->i_mode)) {
 911                 p->swap_device = swap_inode->i_rdev;
 912 
 913                 filp.f_inode = swap_inode;
 914                 filp.f_mode = 3; /* read write */
 915                 error = blkdev_open(swap_inode, &filp);
 916                 p->swap_file = NULL;
 917                 iput(swap_inode);
 918                 if(error)
 919                         goto bad_swap_2;
 920                 error = -ENODEV;
 921                 if (!p->swap_device)
 922                         goto bad_swap;
 923                 error = -EBUSY;
 924                 for (i = 0 ; i < nr_swapfiles ; i++) {
 925                         if (i == type)
 926                                 continue;
 927                         if (p->swap_device == swap_info[i].swap_device)
 928                                 goto bad_swap;
 929                 }
 930         } else if (!S_ISREG(swap_inode->i_mode))
 931                 goto bad_swap;
 932         p->swap_lockmap = (unsigned char *) get_free_page(GFP_USER);
 933         if (!p->swap_lockmap) {
 934                 printk("Unable to start swapping: out of memory :-)\n");
 935                 error = -ENOMEM;
 936                 goto bad_swap;
 937         }
 938         read_swap_page(SWP_ENTRY(type,0), (char *) p->swap_lockmap);
 939         if (memcmp("SWAP-SPACE",p->swap_lockmap+4086,10)) {
 940                 printk("Unable to find swap-space signature\n");
 941                 error = -EINVAL;
 942                 goto bad_swap;
 943         }
 944         memset(p->swap_lockmap+PAGE_SIZE-10,0,10);
 945         j = 0;
 946         p->lowest_bit = 0;
 947         p->highest_bit = 0;
 948         for (i = 1 ; i < 8*PAGE_SIZE ; i++) {
 949                 if (test_bit(i,p->swap_lockmap)) {
 950                         if (!p->lowest_bit)
 951                                 p->lowest_bit = i;
 952                         p->highest_bit = i;
 953                         p->max = i+1;
 954                         j++;
 955                 }
 956         }
 957         if (!j) {
 958                 printk("Empty swap-file\n");
 959                 error = -EINVAL;
 960                 goto bad_swap;
 961         }
 962         p->swap_map = (unsigned char *) vmalloc(p->max);
 963         if (!p->swap_map) {
 964                 error = -ENOMEM;
 965                 goto bad_swap;
 966         }
 967         for (i = 1 ; i < p->max ; i++) {
 968                 if (test_bit(i,p->swap_lockmap))
 969                         p->swap_map[i] = 0;
 970                 else
 971                         p->swap_map[i] = 0x80;
 972         }
 973         p->swap_map[0] = 0x80;
 974         memset(p->swap_lockmap,0,PAGE_SIZE);
 975         p->flags = SWP_WRITEOK;
 976         p->pages = j;
 977         nr_swap_pages += j;
 978         printk("Adding Swap: %dk swap-space\n",j<<2);
 979         return 0;
 980 bad_swap:
 981         if(filp.f_op && filp.f_op->release)
 982                 filp.f_op->release(filp.f_inode,&filp);
 983 bad_swap_2:
 984         free_page((long) p->swap_lockmap);
 985         vfree(p->swap_map);
 986         iput(p->swap_file);
 987         p->swap_device = 0;
 988         p->swap_file = NULL;
 989         p->swap_map = NULL;
 990         p->swap_lockmap = NULL;
 991         p->flags = 0;
 992         return error;
 993 }
 994 
 995 void si_swapinfo(struct sysinfo *val)
     /*  */
 996 {
 997         unsigned int i, j;
 998 
 999         val->freeswap = val->totalswap = 0;
1000         for (i = 0; i < nr_swapfiles; i++) {
1001                 if ((swap_info[i].flags & SWP_WRITEOK) != SWP_WRITEOK)
1002                         continue;
1003                 for (j = 0; j < swap_info[i].max; ++j)
1004                         switch (swap_info[i].swap_map[j]) {
1005                                 case 128:
1006                                         continue;
1007                                 case 0:
1008                                         ++val->freeswap;
1009                                 default:
1010                                         ++val->totalswap;
1011                         }
1012         }
1013         val->freeswap <<= PAGE_SHIFT;
1014         val->totalswap <<= PAGE_SHIFT;
1015         return;
1016 }
1017 
1018 /*
1019  * set up the free-area data structures:
1020  *   - mark all pages MAP_PAGE_RESERVED
1021  *   - mark all memory queues empty
1022  *   - clear the memory bitmaps
1023  */
1024 unsigned long free_area_init(unsigned long start_mem, unsigned long end_mem)
     /*  */
1025 {
1026         unsigned short * p;
1027         unsigned long mask = PAGE_MASK;
1028         int i;
1029 
1030         /*
1031          * select nr of pages we try to keep free for important stuff
1032          * with a minimum of 16 pages. This is totally arbitrary
1033          */
1034         i = end_mem >> (PAGE_SHIFT+6);
1035         if (i < 16)
1036                 i = 16;
1037         min_free_pages = i;
1038         start_mem = init_swap_cache(start_mem, end_mem);
1039         mem_map = (unsigned short *) start_mem;
1040         p = mem_map + MAP_NR(end_mem);
1041         start_mem = (unsigned long) p;
1042         while (p > mem_map)
1043                 *--p = MAP_PAGE_RESERVED;
1044 
1045         for (i = 0 ; i < NR_MEM_LISTS ; i++) {
1046                 unsigned long bitmap_size;
1047                 free_area_list[i].prev = free_area_list[i].next = &free_area_list[i];
1048                 mask += mask;
1049                 end_mem = (end_mem + ~mask) & mask;
1050                 bitmap_size = end_mem >> (PAGE_SHIFT + i);
1051                 bitmap_size = (bitmap_size + 7) >> 3;
1052                 free_area_map[i] = (unsigned char *) start_mem;
1053                 memset((void *) start_mem, 0, bitmap_size);
1054                 start_mem += bitmap_size;
1055         }
1056         return start_mem;
1057 }
/* */
root/mm/swap.c

DEFINITIONS
