Re: [PATCH] fs/proc: Move kfree outside pde_unload_lock

[Nathan Zimmer]

On Fri, Aug 24, 2012 at 11:45:45AM -0500, Nathan Zimmer wrote:
> On 08/24/2012 09:58 AM, Eric Dumazet wrote:
>> Le vendredi 24 août 2012 à 09:48 -0500, Nathan Zimmer a écrit :
>>> On Wed, Aug 22, 2012 at 11:42:58PM +0200, Eric Dumazet wrote:
>>>> On Wed, 2012-08-22 at 20:28 +0200, Eric Dumazet wrote:
>>>>
>>>>> Thats interesting, but if you really want this to fly, one RCU
>>>>> conversion would be much better ;)
>>>>>
>>>>> pde_users would be an atomic_t and you would avoid the spinlock
>>>>> contention.
>>>> Here is what I had in mind, I would be interested to know how it helps a 512 core machine ;)
>>>>
>>> Here are the results and they look great.
>>>
>>> cpuinfo	baseline	moved kfree	Rcu
>>> tasks	read-sec	read-sec	read-sec
>>> 1	0.0141		0.0141		0.0141
>>> 2	0.0140		0.0140		0.0142
>>> 4	0.0140		0.0141		0.0141
>>> 8	0.0145		0.0145		0.0140
>>> 16	0.0553		0.0548		0.0168
>>> 32	0.1688		0.1622		0.0549
>>> 64	0.5017		0.3856		0.1690
>>> 128	1.7005		0.9710		0.5038
>>> 256	5.2513		2.6519		2.0804
>>> 512	8.0529		6.2976		3.0162
>>>
>>>
>>>
>> Indeed...
>>
>> Could you explicit the test you are actually doing ?
>>
>> Thanks
>>
>>
>
>
> It is a dead simple test.
> The test starts by forking off X number of tasks
> assigning each their own cpu.
> Each task then allocs a bit of memory.
> All tasks wait on a memory cell for the go order.
> We measure the read time starting here.
> Once the go order is given they all read a chunk of the selected proc file.
> I was using /proc/cpuinfo to test.
> Once everyone has finished we take the end read time.
>

Here is the text for those who are curious.


/*------------------------------------------------------------------------------------*/
char *helpstr[] = {
	"This test program is a generic template.",
	0
};

#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <sys/mman.h>

#include <sched.h>
#include <sys/time.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <linux/unistd.h>

//#include "setup.h"

#define MAXCPUS                 4096
#define perrorx(s)              do { perror(s); exit(1);} while(0)
#define mb()                    asm volatile("mfence":::"memory")
#define barrier()               asm volatile("": : :"memory")
#define cpu_relax()             asm volatile ("rep;nop":::"memory");


extern int optind, opterr;
extern char *optarg;

static int verbose = 0;
static int header = 0;
static char *file = "/proc/stat";
static int numtasks = 1;
static int repeat = 1;
static int bufsize = 1024;

struct control_s {
	int ready;
	int done;
	int go;
	int exit;
} *cntl;


static cpu_set_t *defmask;
static int cpu_set_size;

static void runon_init(void)
{
        if (!defmask) {
                cpu_set_size = CPU_ALLOC_SIZE(MAXCPUS);
                defmask = CPU_ALLOC(MAXCPUS);
                if (sched_getaffinity(0, cpu_set_size, defmask) < 0)
                        perrorx("unexpected failure in runon_init");
        }
}


static double timeInSeconds(long time_in_microseconds)
{
        double temp;

        temp = time_in_microseconds;
        temp /= 1000000;

        return temp;
}

static int runon(int cpu)
{
        cpu_set_t *mask;

	runon_init();
        mask = CPU_ALLOC(MAXCPUS);
        if (cpu < 0 || cpu >= MAXCPUS)
                return -1;
        CPU_ZERO_S(cpu_set_size, mask);
        CPU_SET_S(cpu, cpu_set_size, mask);
        if (sched_setaffinity(0, cpu_set_size, mask) < 0)
                return -1;
        CPU_FREE(mask);
        return 0;
}

static long getCurrentTime()
{
        struct timeval tp;
        long usec;

        mb();
        gettimeofday(&tp, 0);
        usec = tp.tv_sec * 1000000 + tp.tv_usec;
        mb();
        return usec;
}


static void do_help(void)
{
	char **p;

	for (p = helpstr; *p; p++)
		printf("%s\n", *p);
	exit(0);
}

static void slave(int id)
{
	FILE *f;
	int i;
	char *buf;

	runon(id);
	buf = malloc(bufsize);
	memset(buf, 0, bufsize);

	if ((f = fopen(file, "r")) < 0)
		perrorx("open failed");
	while (fgets(buf, bufsize, f) != NULL) {
	}
	fclose(f);

	(void)__sync_fetch_and_add(&cntl->ready, 1);
	while (!cntl->go)
		cpu_relax();

	for (i = 0; i < repeat; i++) {
		if ((f = fopen(file, "r")) < 0)
			perrorx("open failed");
		while (fgets(buf, bufsize, f) != NULL) {
		}
		fclose(f);
		barrier();
	}

	(void)__sync_fetch_and_add(&cntl->done, 1);
	while (!cntl->exit)
		cpu_relax();
	exit(0);
}

int main(int argc, char **argv)
{
	int i, c, stat, er = 0;
	static char optstr[] = "b:f:hn:r:v";
	unsigned long t, tfork, tready, tread, texit;

	opterr = 1;
	while ((c = getopt(argc, argv, optstr)) != EOF)
		switch (c) {
		case 'b':
			bufsize = atoi(optarg);
			break;
		case 'f':
			file = optarg;
			break;
		case 'h':
			header++;
			break;
		case 'n':
			numtasks = atoi(optarg);
			break;
		case 'r':
			repeat = atoi(optarg);
			break;
		case 'v':
			verbose++;
			break;
		case '?':
			er = 1;
			break;
		}

	if (er)
		do_help();

	runon(0);
	cntl = mmap(NULL, getpagesize(), PROT_WRITE | PROT_READ, MAP_SHARED | MAP_ANONYMOUS, -1, 0);

	tfork = getCurrentTime();
	for (i = 0; i < numtasks; i++)
		if (fork() == 0)
			slave(i + 1);
	t = getCurrentTime();
	tfork = t - tfork;

	tready = t;
	while (cntl->ready != numtasks)
		usleep(1000);
	t = getCurrentTime();
	tready = t - tready;

	tread = t;
	cntl->go = 1;

	while (cntl->done != numtasks)
		cpu_relax();
	t = getCurrentTime();
	tread = t - tread;

	texit = t;
	cntl->exit = 1;
	while (wait(&stat) > 0)
		usleep(1000);
	texit = getCurrentTime() - texit;

	if (header) {
		printf("File: %s\n", file);
		printf("Bufsize: %d\n", bufsize);
		printf("Repeats: %d\n", repeat);
		printf("%6s%18s%18s%18s%18s%18s\n", "tasks", "fork-sec", "ready-sec", "read-sec", "read/repeat sec",
		       "texit");
	}
	printf("%6d%18.6f%18.6f%18.6f%18.6f%18.6f\n", numtasks, timeInSeconds(tfork), timeInSeconds(tready),
	       timeInSeconds(tread), timeInSeconds(tread) / repeat, timeInSeconds(texit));
	return 0;
}