I've attached 3 optimization for the 2.4 kernel:
* optimize fixed size kmallocs.
It's a bit difficult to benchmark this one, but it seems that
kmalloc(PAGE_SIZE) is speed-up by 20-40 cpu ticks (K6/200)
* use kmalloc instead of __get_free_page() for getname()
kmalloc+kfree is ~ 500 cpu ticks faster. This means that stat("/bin") is
now 30% faster.
* the memory allocations in select.
I hope I didn't introduce any new bugs, but the double allocations in
sys_poll were too slow. I rewrote it with a linked list+kmalloc. The
result is 40% faster for a single fd poll.
All numbers are from special micro-benchmarks, apache on localhost is
~1.5 % faster with all 3 patches applied.
-- ManfredP.S.: I disabled SLAB_DEBUGGING, and all patches are tested with 2.3.99-pre8 on i386.
// $Header$ // Kernel Version: // VERSION = 2 // PATCHLEVEL = 3 // SUBLEVEL = 99 // EXTRAVERSION = -pre6 --- 2.3/kernel/ksyms.c Thu Apr 27 11:27:26 2000 +++ build-2.3/kernel/ksyms.c Wed May 3 21:08:04 2000 @@ -112,7 +112,8 @@ EXPORT_SYMBOL(kmem_cache_shrink); EXPORT_SYMBOL(kmem_cache_alloc); EXPORT_SYMBOL(kmem_cache_free); -EXPORT_SYMBOL(kmalloc); +EXPORT_SYMBOL(cache_sizes); +EXPORT_SYMBOL(__kmalloc); EXPORT_SYMBOL(kfree); EXPORT_SYMBOL(kfree_s); EXPORT_SYMBOL(vmalloc); --- 2.3/mm/slab.c Thu Apr 27 11:27:26 2000 +++ build-2.3/mm/slab.c Wed May 3 21:08:04 2000 @@ -324,13 +324,8 @@ #define SLAB_SET_PAGE_SLAB(pg,x) ((pg)->list.prev = (struct list_head *)(x)) #define SLAB_GET_PAGE_SLAB(pg) ((kmem_slab_t *)(pg)->list.prev) -/* Size description struct for general caches. */ -typedef struct cache_sizes { - size_t cs_size; - kmem_cache_t *cs_cachep; -} cache_sizes_t; - -static cache_sizes_t cache_sizes[] = { +/* these values are hardcoded in <linux/kmalloc.h> */ +cache_sizes_t cache_sizes[] = { #if PAGE_SIZE == 4096 { 32, NULL}, #endif @@ -1680,7 +1675,7 @@ } void * -kmalloc(size_t size, int flags) +__kmalloc(size_t size, int flags) { cache_sizes_t *csizep = cache_sizes; --- 2.3/include/linux/slab.h Sat Feb 12 20:42:24 2000 +++ build-2.3/include/linux/slab.h Wed May 3 21:08:04 2000 @@ -56,10 +56,6 @@ extern void *kmem_cache_alloc(kmem_cache_t *, int); extern void kmem_cache_free(kmem_cache_t *, void *); -extern void *kmalloc(size_t, int); -extern void kfree(const void *); -extern void kfree_s(const void *, size_t); - extern void kmem_cache_reap(int); extern int get_slabinfo(char *); @@ -67,6 +63,56 @@ extern kmem_cache_t *vm_area_cachep; extern kmem_cache_t *mm_cachep; -#endif /* __KERNEL__ */ +/* generic kmalloc */ +extern void *__kmalloc(size_t, int); +extern void kfree(const void *); +extern void kfree_s(const void *, size_t); +/* Size description struct for general caches. */ +typedef struct cache_sizes { + size_t cs_size; + kmem_cache_t *cs_cachep; +} cache_sizes_t; + +extern cache_sizes_t cache_sizes[]; +extern void __you_cannot_kmalloc_more_than_128_kilo_bytes(void); + +static inline void* __constant_kmalloc(size_t size, int flags) +{ +#if PAGE_SIZE == 4096 + if(size < 32) + return kmem_cache_alloc(cache_sizes[0].cs_cachep, flags); +#define KSHIFT 0 +#else +#define KSHIFT 1 +#endif +#define FIXED_ALLOC(len,off) \ + if(size <= len) \ + return kmem_cache_alloc(cache_sizes[off-KSHIFT].cs_cachep, flags) + FIXED_ALLOC(64,1); + FIXED_ALLOC(128,2); + FIXED_ALLOC(256,3); + FIXED_ALLOC(512,4); + FIXED_ALLOC(1024,5); + FIXED_ALLOC(2048,6); + FIXED_ALLOC(4096,7); + FIXED_ALLOC(8192,8); + FIXED_ALLOC(16384,9); + FIXED_ALLOC(32768,10); + FIXED_ALLOC(65536,11); + FIXED_ALLOC(131072,12); +#undef FIXED_ALLOC +#undef KSHIFT + __you_cannot_kmalloc_more_than_128_kilo_bytes(); + return NULL; +} + +extern void *kmem_cache_alloc(kmem_cache_t *, int); + +#define kmalloc(size, flags) \ + (__builtin_constant_p(size) ? \ + __constant_kmalloc((size),(flags)) : \ + __kmalloc(size,flags)) + +#endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */
// $Header$ // Kernel Version: // VERSION = 2 // PATCHLEVEL = 3 // SUBLEVEL = 99 // EXTRAVERSION = -pre6 --- 2.3/include/linux/fs.h Thu Apr 27 11:27:24 2000 +++ build-2.3/include/linux/fs.h Wed May 3 21:15:53 2000 @@ -858,8 +858,8 @@ extern struct file * dentry_open(struct dentry *, struct vfsmount *, int); extern int filp_close(struct file *, fl_owner_t id); extern char * getname(const char *); -#define __getname() ((char *) __get_free_page(GFP_KERNEL)) -#define putname(name) free_page((unsigned long)(name)) +#define __getname() ((char *) kmalloc(PAGE_SIZE, GFP_KERNEL)) +#define putname(name) kfree((name)) enum {BDEV_FILE, BDEV_SWAP, BDEV_FS, BDEV_RAW}; extern void kill_fasync(struct fasync_struct *, int, int); --- 2.3/fs/open.c Thu Apr 27 11:27:14 2000 +++ build-2.3/fs/open.c Wed May 3 21:15:53 2000 @@ -10,6 +10,7 @@ #include <linux/file.h> #include <linux/smp_lock.h> #include <linux/quotaops.h> +#include <linux/slab.h> #include <asm/uaccess.h> --- 2.3/fs/select.c Thu Apr 27 11:27:14 2000 +++ build-2.3/fs/select.c Wed May 3 21:15:53 2000 @@ -48,7 +48,7 @@ poll_table* out; poll_table* walk; - out = (poll_table *) __get_free_page(GFP_KERNEL); + out = (poll_table *) kmalloc(PAGE_SIZE,GFP_KERNEL); if(out==NULL) return NULL; out->nr = 0; @@ -57,11 +57,11 @@ nfds -=__MAX_POLL_TABLE_ENTRIES; walk = out; while(nfds > 0) { - poll_table *tmp = (poll_table *) __get_free_page(GFP_KERNEL); + poll_table *tmp = (poll_table *) kmalloc(PAGE_SIZE,GFP_KERNEL); if (!tmp) { while(out != NULL) { tmp = out->next; - free_page((unsigned long)out); + kfree(out); out = tmp; } return NULL; @@ -91,7 +91,7 @@ } old = p; p = p->next; - free_page((unsigned long) old); + kfree(old); } }
// $Header$ // Kernel Version: // VERSION = 2 // PATCHLEVEL = 3 // SUBLEVEL = 99 // EXTRAVERSION = -pre8 --- 2.3/fs/select.c Thu Apr 27 11:27:14 2000 +++ build-2.3/fs/select.c Sat May 20 20:32:20 2000 @@ -344,7 +344,7 @@ return ret; } -#define POLLFD_PER_PAGE ((PAGE_SIZE) / sizeof(struct pollfd)) +#define POLLFD_PER_PAGE ((PAGE_SIZE-sizeof(void*)) / sizeof(struct pollfd)) static void do_pollfd(unsigned int num, struct pollfd * fdpage, poll_table ** pwait, int *count) @@ -377,20 +377,28 @@ fdp->revents = mask; } } +struct poll_list { + struct poll_list *next; +}; -static int do_poll(unsigned int nfds, unsigned int nchunks, unsigned int nleft, - struct pollfd *fds[], poll_table *wait, long timeout) +static int do_poll(int nfds, struct poll_list *list, + poll_table *wait, long timeout) { int count = 0; + struct poll_list* walk; for (;;) { unsigned int i; set_current_state(TASK_INTERRUPTIBLE); - for (i=0; i < nchunks; i++) - do_pollfd(POLLFD_PER_PAGE, fds[i], &wait, &count); - if (nleft) - do_pollfd(nleft, fds[nchunks], &wait, &count); + walk = list; + i = nfds; + while(walk != NULL) { + do_pollfd( i>POLLFD_PER_PAGE ? POLLFD_PER_PAGE : i, + (struct pollfd*)(walk+1), &wait, &count); + i -= POLLFD_PER_PAGE; + walk = walk->next; + } wait = NULL; if (count || !timeout || signal_pending(current)) break; @@ -402,10 +410,10 @@ asmlinkage long sys_poll(struct pollfd * ufds, unsigned int nfds, long timeout) { - int i, j, fdcount, err; - struct pollfd **fds; + int j, fdcount, err; + unsigned int i; + struct poll_list *polllist, *pollwalk, **pp; poll_table *wait = NULL; - int nchunks, nleft; /* Do a sanity check on nfds ... */ if (nfds > current->files->max_fds) @@ -424,65 +432,61 @@ if (!wait) return -ENOMEM; } - err = -ENOMEM; - - fds = NULL; - if (nfds != 0) { - fds = (struct pollfd **)kmalloc( - (1 + (nfds - 1) / POLLFD_PER_PAGE) * sizeof(struct pollfd *), - GFP_KERNEL); - if (fds == NULL) - goto out; - } - nchunks = 0; - nleft = nfds; - while (nleft > POLLFD_PER_PAGE) { /* allocate complete PAGE_SIZE chunks */ - fds[nchunks] = (struct pollfd *)__get_free_page(GFP_KERNEL); - if (fds[nchunks] == NULL) + polllist = NULL; + i = nfds; + pp = &polllist; + j = 0; + err = -ENOMEM; + for(;;) { + pollwalk = kmalloc(sizeof(struct poll_list)+ + sizeof(struct pollfd)* + (i>POLLFD_PER_PAGE?POLLFD_PER_PAGE:i), + GFP_KERNEL); + if(pollwalk==NULL) goto out_fds; - nchunks++; - nleft -= POLLFD_PER_PAGE; - } - if (nleft) { /* allocate last PAGE_SIZE chunk, only nleft elements used */ - fds[nchunks] = (struct pollfd *)__get_free_page(GFP_KERNEL); - if (fds[nchunks] == NULL) + pollwalk->next=NULL; + *pp = pollwalk; + pp = &pollwalk->next; + if (copy_from_user(pollwalk+1, ufds + j*POLLFD_PER_PAGE, + sizeof(struct pollfd)* + ( i>POLLFD_PER_PAGE ? POLLFD_PER_PAGE : i))) { + err = -EFAULT; goto out_fds; - } + } + j++; - err = -EFAULT; - for (i=0; i < nchunks; i++) - if (copy_from_user(fds[i], ufds + i*POLLFD_PER_PAGE, PAGE_SIZE)) - goto out_fds1; - if (nleft) { - if (copy_from_user(fds[nchunks], ufds + nchunks*POLLFD_PER_PAGE, - nleft * sizeof(struct pollfd))) - goto out_fds1; + if(i<=POLLFD_PER_PAGE) + break; + i -= POLLFD_PER_PAGE; } - - fdcount = do_poll(nfds, nchunks, nleft, fds, wait, timeout); + + fdcount = do_poll(nfds, polllist, wait, timeout); /* OK, now copy the revents fields back to user space. */ - for(i=0; i < nchunks; i++) - for (j=0; j < POLLFD_PER_PAGE; j++, ufds++) - __put_user((fds[i] + j)->revents, &ufds->revents); - if (nleft) - for (j=0; j < nleft; j++, ufds++) - __put_user((fds[nchunks] + j)->revents, &ufds->revents); + i = nfds; + pollwalk = polllist; + err = -EFAULT; + while(pollwalk != NULL) { + struct pollfd * fds = (struct pollfd*)(pollwalk+1); + for (j=0; j < ( i>POLLFD_PER_PAGE ? POLLFD_PER_PAGE : i); j++, ufds++) { + if(__put_user(fds[j].revents, &ufds->revents)) + goto out_fds; + } + i -= POLLFD_PER_PAGE; + pollwalk = pollwalk->next; + } err = fdcount; if (!fdcount && signal_pending(current)) err = -EINTR; -out_fds1: - if (nleft) - free_page((unsigned long)(fds[nchunks])); out_fds: - for (i=0; i < nchunks; i++) - free_page((unsigned long)(fds[i])); - if (nfds != 0) - kfree(fds); -out: + while(polllist!=NULL) { + pollwalk = polllist; + polllist = polllist->next; + kfree(pollwalk); + } free_wait(wait); return err; }
- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.rutgers.edu Please read the FAQ at http://www.tux.org/lkml/
This archive was generated by hypermail 2b29 : Tue May 23 2000 - 21:00:22 EST