Weird tcp performance differences with 2.0 and 2.2 kernels

Mark R. Boyns (boyns@contigo.com)
09 Feb 1999 17:35:56 -0800


We're trying to figure out some weird performance differences between
2.0.x / 2.2.x kernels and the interface being used. One of the most
noticeable differences is that 2.0 kernels seem to be able to
communicate much faster over the loopback interface. Another
interesting thing is that 2.2 kernels seem to communicate with each
other faster when the data being written is >= 724 bytes.

We came up with a small client and server (included below) to generate
some stats. The client connects to the server and asks the server to
send some number of bytes. In the stats below the client connects to
the server 100 times for each number of bytes and the resulting
connections/second is displayed.

server linux 2.2.1, 400mhz Pentium II, 3c905B Cyclone 100baseTx

local client (loopback)
10 bytes 50.18 c/s
723 bytes 49.59 c/s
724 bytes 49.87 c/s
1024 bytes 50.14 c/s

remote client linux 2.2.1, 400mhz Pentium II, 3c905B Cyclone 100baseTx
10 bytes 46.51 c/s
723 bytes 47.68 c/s
724 bytes 279.62 c/s
1024 bytes 300.92 c/s

server linux 2.0.36, 120mhz Pentium, 3c562 10baseT (laptop)

local client (loopback)
10 bytes 540.00 c/s
723 bytes 533.28 c/s
724 bytes 533.25 c/s
1024 bytes 521.59 c/s

remote client linux 2.2.1, 400mhz Pentium II, 3c905B Cyclone 100baseTx
10 bytes 467.46 c/s
723 bytes 188.48 c/s
724 bytes 182.81 c/s
1024 bytes 160.13 c/s

We would be very interested to know if other people are seeing the
same numbers and hopefully someone can explain the differences.

unshar the archive below and compile with:
$ make server
$ make client

to run the server:
$ ./server

to run the client:
$ ./client <server-host> <connections> <bytes>

the tests above used:
$ ./client <server-host> 100 10
$ ./client <server-host> 100 723
$ ./client <server-host> 100 724
$ ./client <server-host> 100 1024

#!/bin/sh
# This is a shell archive (produced by GNU sharutils 4.2).
# To extract the files from this archive, save it to some FILE, remove
# everything before the `!/bin/sh' line above, then type `sh FILE'.
#
# Made on 1999-02-09 14:31 PST by <boyns@doctor>.
# Source directory was `/localwork/home/boyns'.
#
# Existing files will *not* be overwritten unless `-c' is specified.
#
# This shar contains:
# length mode name
# ------ ---------- ------------------------------------------
# 1364 -rw-rw-r-- server.c
# 2156 -rw-rw-r-- client.c
#
save_IFS="${IFS}"
IFS="${IFS}:"
gettext_dir=FAILED
locale_dir=FAILED
first_param="$1"
for dir in $PATH
do
if test "$gettext_dir" = FAILED && test -f $dir/gettext \
&& ($dir/gettext --version >/dev/null 2>&1)
then
set `$dir/gettext --version 2>&1`
if test "$3" = GNU
then
gettext_dir=$dir
fi
fi
if test "$locale_dir" = FAILED && test -f $dir/shar \
&& ($dir/shar --print-text-domain-dir >/dev/null 2>&1)
then
locale_dir=`$dir/shar --print-text-domain-dir`
fi
done
IFS="$save_IFS"
if test "$locale_dir" = FAILED || test "$gettext_dir" = FAILED
then
echo=echo
else
TEXTDOMAINDIR=$locale_dir
export TEXTDOMAINDIR
TEXTDOMAIN=sharutils
export TEXTDOMAIN
echo="$gettext_dir/gettext -s"
fi
touch -am 1231235999 $$.touch >/dev/null 2>&1
if test ! -f 1231235999 && test -f $$.touch; then
shar_touch=touch
else
shar_touch=:
echo
$echo 'WARNING: not restoring timestamps. Consider getting and'
$echo "installing GNU \`touch', distributed in GNU File Utilities..."
echo
fi
rm -f 1231235999 $$.touch
#
if mkdir _sh17400; then
$echo 'x -' 'creating lock directory'
else
$echo 'failed to create lock directory'
exit 1
fi
# ============= server.c ==============
if test -f 'server.c' && test "$first_param" != -c; then
$echo 'x -' SKIPPING 'server.c' '(file already exists)'
else
$echo 'x -' extracting 'server.c' '(text)'
sed 's/^X//' << 'SHAR_EOF' > 'server.c' &&
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <netinet/in.h>
#include <netdb.h>
#include <sys/time.h>
X
X
int main(int argc, char** argv)
{
X struct sockaddr_in server;
X struct sockaddr_in sin;
X int on = 1;
X int sock;
X int port = 7890;
X char buf[4096];
X
X memset(buf, ' ', 4096);
X
X sock = socket(AF_INET, SOCK_STREAM, 0);
X if (sock < 0)
X {
X perror("socket");
X exit(1);
X }
X
X setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
X
X server.sin_addr.s_addr = 0;
X server.sin_port = htons(port);
X server.sin_family = AF_INET;
X
X if (bind(sock, (struct sockaddr *)&server, sizeof(server)) == -1)
X {
X perror("bind");
X exit(1);
X }
X
X listen(sock, 512);
X
X for (;;)
X {
X int size, n, client;
X
X n = sizeof(sin);
X client = accept(sock, (struct sockaddr *)&sin, &n);
X if (client < 0)
X {
X perror("accept");
X exit(1);
X }
X
X read(client, &size, 4);
X size = ntohl(size);
X
X n = write(client, buf, size);
X if (n != size)
X {
X printf("short write\n");
X exit(1);
X }
X
X close(client);
X }
X
}
SHAR_EOF
$shar_touch -am 020914201999 'server.c' &&
chmod 0664 'server.c' ||
$echo 'restore of' 'server.c' 'failed'
if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \
&& ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then
md5sum -c << SHAR_EOF >/dev/null 2>&1 \
|| $echo 'server.c:' 'MD5 check failed'
8bb7faf8871b49628beeaab4cf3531c2 server.c
SHAR_EOF
else
shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'server.c'`"
test 1364 -eq "$shar_count" ||
$echo 'server.c:' 'original size' '1364,' 'current size' "$shar_count!"
fi
fi
# ============= client.c ==============
if test -f 'client.c' && test "$first_param" != -c; then
$echo 'x -' SKIPPING 'client.c' '(file already exists)'
else
$echo 'x -' extracting 'client.c' '(text)'
sed 's/^X//' << 'SHAR_EOF' > 'client.c' &&
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <netinet/in.h>
#include <netdb.h>
#include <sys/time.h>
X
long tvsub(struct timeval stop, struct timeval start)
{
X long sec, usec;
X sec = (stop.tv_sec - start.tv_sec);
X usec = (stop.tv_usec - start.tv_usec);
X if (usec < 0)
X {
X sec--;
X usec += 1000000;
X }
X
X return sec * 1000000 + usec;
}
X
int main(int argc, char** argv)
{
X char *host = argv[1];
X int count = atoi(argv[2]);
X int len = atoi(argv[3]);
X int on = 1;
X int sock, i, n, bytes;
X int port = 7890;
X char buf[4096];
X struct hostent *hp;
X unsigned long addr;
X struct sockaddr_in server;
X struct timeval start, stop;
X long usec;
X
X addr = inet_addr(host);
X if (addr == ~0L)
X {
X hp = gethostbyname(host);
X if (hp == 0)
X return 0;
X memcpy((char *)&server.sin_addr, (char *)hp->h_addr, hp->h_length);
X }
X else
X memcpy((char *)&server.sin_addr, (char *)&addr, sizeof(addr));
X
X server.sin_port = htons(port);
X server.sin_family = AF_INET;
X
X gettimeofday(&start, 0);
X
X bytes = 0;
X for (i = 0; i < count; i++)
X {
X sock = socket(AF_INET, SOCK_STREAM, 0);
X setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
X
X if (connect(sock, (struct sockaddr *)&server, sizeof(server)) == -1)
X {
X return -1;
X }
X
X n = htonl(len);
X write(sock, &n, 4);
X
X while ((n = read(sock, buf, 4096)) > 0)
X {
X bytes += n;
X }
X
X close(sock);
X }
X
X gettimeofday(&stop, 0);
X
X usec = tvsub(stop, start);
X printf("%10.2fs", usec/1000000.0);
X if (bytes > 1024)
X printf(" %10.2fkb ", bytes / 1024.0);
X else
X printf(" %10db", bytes);
X printf(" %10.2fc/s", (count) / (usec / 1000000.0));
X printf(" %10.2fkb/s", (bytes / 1024.0) / (usec / 1000000.0));
X printf("\n");
}
SHAR_EOF
$shar_touch -am 020914191999 'client.c' &&
chmod 0664 'client.c' ||
$echo 'restore of' 'client.c' 'failed'
if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \
&& ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then
md5sum -c << SHAR_EOF >/dev/null 2>&1 \
|| $echo 'client.c:' 'MD5 check failed'
e36dd00d612ef704a109e1a387ea26b3 client.c
SHAR_EOF
else
shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'client.c'`"
test 2156 -eq "$shar_count" ||
$echo 'client.c:' 'original size' '2156,' 'current size' "$shar_count!"
fi
fi
rm -fr _sh17400
exit 0

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.rutgers.edu
Please read the FAQ at http://www.tux.org/lkml/