Requires the last 4 patches.
Name: per-cpu support inside modules
Author: Rusty Russell
Status: Tested on 2.5.69-bk13
Depends: Misc/kmalloc_percpu-full.patch.gz
D: This adds percpu support for modules.
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/include/asm-generic/percpu.h .22478-2.5.69-bk13-module-percpu/include/asm-generic/percpu.h
--- .22478-2.5.69-bk13-module-percpu.pre/include/asm-generic/percpu.h 2003-05-20 14:58:41.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/include/asm-generic/percpu.h 2003-05-20 14:58:42.000000000 +1000
@@ -5,10 +5,8 @@
#ifdef CONFIG_SMP
/* Separate out the type, so (int[3], foo) works. */
-#ifndef MODULE
#define DEFINE_PER_CPU(type, name) \
__attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu
-#endif
#define __get_cpu_var(var) per_cpu(var, smp_processor_id())
#define __get_cpu_ptr(ptr) per_cpu_ptr(ptr, smp_processor_id())
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/include/asm-ia64/percpu.h .22478-2.5.69-bk13-module-percpu/include/asm-ia64/percpu.h
--- .22478-2.5.69-bk13-module-percpu.pre/include/asm-ia64/percpu.h 2003-05-20 14:58:41.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/include/asm-ia64/percpu.h 2003-05-20 14:58:42.000000000 +1000
@@ -18,10 +18,8 @@
#include <linux/threads.h>
#ifdef CONFIG_SMP
-#ifndef MODULE
#define DEFINE_PER_CPU(type, name) \
__attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu
-#endif
#define __get_cpu_var(var) (var##__per_cpu)
#define __get_cpu_ptr(ptr) (ptr)
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/include/linux/module.h .22478-2.5.69-bk13-module-percpu/include/linux/module.h
--- .22478-2.5.69-bk13-module-percpu.pre/include/linux/module.h 2003-05-05 12:37:12.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/include/linux/module.h 2003-05-20 14:58:42.000000000 +1000
@@ -247,6 +247,9 @@ struct module
char *strtab;
#endif
+ /* Per-cpu data. */
+ void *percpu;
+
/* The command line arguments (may be mangled). People like
keeping pointers to this stuff */
char *args;
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/include/linux/percpu.h .22478-2.5.69-bk13-module-percpu/include/linux/percpu.h
--- .22478-2.5.69-bk13-module-percpu.pre/include/linux/percpu.h 2003-05-20 14:58:41.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/include/linux/percpu.h 2003-05-20 14:58:42.000000000 +1000
@@ -44,11 +44,8 @@ static inline void free_percpu(const voi
kfree(ptr);
}
-/* Can't define per-cpu variables in modules. Sorry --RR */
-#ifndef MODULE
#define DEFINE_PER_CPU(type, name) \
__typeof__(type) name##__per_cpu
-#endif
#define per_cpu(var, cpu) ((void)cpu, var##__per_cpu)
#define __get_cpu_var(var) var##__per_cpu
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/kernel/module.c .22478-2.5.69-bk13-module-percpu/kernel/module.c
--- .22478-2.5.69-bk13-module-percpu.pre/kernel/module.c 2003-05-19 10:53:51.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/kernel/module.c 2003-05-20 14:58:42.000000000 +1000
@@ -913,6 +913,8 @@ static void free_module(struct module *m
/* This may be NULL, but that's OK */
module_free(mod, mod->module_init);
kfree(mod->args);
+ if (mod->percpu)
+ free_percpu(mod->percpu);
/* Finally, free the core (containing the module structure) */
module_free(mod, mod->module_core);
@@ -939,10 +941,11 @@ static int simplify_symbols(Elf_Shdr *se
unsigned int symindex,
const char *strtab,
unsigned int versindex,
+ unsigned int pcpuindex,
struct module *mod)
{
Elf_Sym *sym = (void *)sechdrs[symindex].sh_addr;
-
+ unsigned long secbase;
unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym);
int ret = 0;
@@ -979,10 +982,12 @@ static int simplify_symbols(Elf_Shdr *se
break;
default:
- sym[i].st_value
- = (unsigned long)
- (sechdrs[sym[i].st_shndx].sh_addr
- + sym[i].st_value);
+ /* Divert to percpu allocation if a percpu var. */
+ if (sym[i].st_shndx == pcpuindex)
+ secbase = (unsigned long)mod->percpu;
+ else
+ secbase = sechdrs[sym[i].st_shndx].sh_addr;
+ sym[i].st_value += secbase;
break;
}
}
@@ -1119,7 +1124,7 @@ static struct module *load_module(void _
char *secstrings, *args, *modmagic, *strtab = NULL;
unsigned int i, symindex = 0, strindex = 0, setupindex, exindex,
exportindex, modindex, obsparmindex, infoindex, gplindex,
- crcindex, gplcrcindex, versindex;
+ crcindex, gplcrcindex, versindex, pcpuindex;
long arglen;
struct module *mod;
long err = 0;
@@ -1194,6 +1199,7 @@ static struct module *load_module(void _
obsparmindex = find_sec(hdr, sechdrs, secstrings, "__obsparm");
versindex = find_sec(hdr, sechdrs, secstrings, "__versions");
infoindex = find_sec(hdr, sechdrs, secstrings, ".modinfo");
+ pcpuindex = find_sec(hdr, sechdrs, secstrings, ".data.percpu");
/* Don't keep modinfo section */
sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
@@ -1272,6 +1278,17 @@ static struct module *load_module(void _
memset(ptr, 0, mod->init_size);
mod->module_init = ptr;
+ if (pcpuindex) {
+ /* We have a special allocation for this section. */
+ mod->percpu = __alloc_percpu(sechdrs[pcpuindex].sh_size,
+ sechdrs[pcpuindex].sh_addralign);
+ if (!mod->percpu) {
+ err = -ENOMEM;
+ goto free_init;
+ }
+ sechdrs[i].sh_flags &= ~(unsigned long)SHF_ALLOC;
+ }
+
/* Transfer each section which specifies SHF_ALLOC */
DEBUGP("final section addresses:\n");
for (i = 0; i < hdr->e_shnum; i++) {
@@ -1303,7 +1320,8 @@ static struct module *load_module(void _
set_license(mod, get_modinfo(sechdrs, infoindex, "license"));
/* Fix up syms, so that st_value is a pointer to location. */
- err = simplify_symbols(sechdrs, symindex, strtab, versindex, mod);
+ err = simplify_symbols(sechdrs, symindex, strtab, versindex, pcpuindex,
+ mod);
if (err < 0)
goto cleanup;
@@ -1342,6 +1360,15 @@ static struct module *load_module(void _
goto cleanup;
}
+ /* Finally, copy percpu area over. */
+ if (pcpuindex) {
+ for (i = 0; i < NR_CPUS; i++)
+ if (cpu_possible(i))
+ memcpy(per_cpu_ptr(mod->percpu, i),
+ (void *)sechdrs[pcpuindex].sh_addr,
+ sechdrs[pcpuindex].sh_size);
+ }
+
#ifdef CONFIG_KALLSYMS
mod->symtab = (void *)sechdrs[symindex].sh_addr;
mod->num_symtab = sechdrs[symindex].sh_size / sizeof(Elf_Sym);
@@ -1380,6 +1407,9 @@ static struct module *load_module(void _
cleanup:
module_unload_free(mod);
+ if (mod->percpu)
+ free_percpu(mod->percpu);
+ free_init:
module_free(mod, mod->module_init);
free_core:
module_free(mod, mod->module_core);
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .22478-2.5.69-bk13-module-percpu.pre/net/ipv4/netfilter/ip_conntrack_core.c .22478-2.5.69-bk13-module-percpu/net/ipv4/netfilter/ip_conntrack_core.c
--- .22478-2.5.69-bk13-module-percpu.pre/net/ipv4/netfilter/ip_conntrack_core.c 2003-05-19 10:53:53.000000000 +1000
+++ .22478-2.5.69-bk13-module-percpu/net/ipv4/netfilter/ip_conntrack_core.c 2003-05-20 14:58:42.000000000 +1000
@@ -1402,11 +1402,23 @@ void ip_conntrack_cleanup(void)
static int hashsize = 0;
MODULE_PARM(hashsize, "i");
+#include <linux/percpu.h>
+DEFINE_PER_CPU(int, percpu_int) = 7;
+
int __init ip_conntrack_init(void)
{
unsigned int i;
int ret;
+ for (i = 0; i < NR_CPUS; i++) {
+ if (!cpu_possible(i))
+ continue;
+ printk("cpu %u: value %p = %i\n", i, &per_cpu(percpu_int, i),
+ per_cpu(percpu_int, i));
+ per_cpu(percpu_int, i) += i;
+ printk("cpu %u: value now %i\n", i, per_cpu(percpu_int, i));
+ }
+
/* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB
* machine has 256 buckets. >= 1GB machines have 8192 buckets. */
if (hashsize) {
-- Anyone who quotes me in their sig is an idiot. -- Rusty Russell. - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
This archive was generated by hypermail 2b29 : Fri May 23 2003 - 22:00:39 EST