x86: Hardware breakpoints are not always triggered

[Andrey Wagin]

Hi,

We use hardware breakpoints in CRIU and we found that sometimes we set
a break-point, but a process doesn't stop on it.

I write a small reproducer for this bug. It create two processes,
where a parent process traces a child. The parent process sets a
break-point and each time when the child stop on it, the parent sets
the variable "xxx" to A in a child process. The child runs an infinite
loop, where it check the variable "xxx" and sets it to B. If a child
process finds that xxx is equal to B, it exits with a non-zero code,
what means that a break-point was not triggered. The source code is
attached.

The reproducer uses a different break-point address if it is executed
with arguments than when it executed without arguments.

Then I made a few experiments. The bug is triggered, if we execute
this program a few times in a KVM virtual machine.

[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint > /dev/null
|| { echo "FAIL - $?"; break; }; done ) &
[3] 4088
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint > /dev/null
|| { echo "FAIL - $?"; break; }; done ) &
[4] 4091
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint 1 2 >
/dev/null || { echo "FAIL - $?"; break; }; done ) &
[5] 4094
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint 1 2 >
/dev/null || { echo "FAIL - $?"; break; }; done ) &
[6] 4097
[8] 4103
[root@fc22-vm ptrace]# 0087: exit - 5
0131: exited, status=1
0126: wait: No child processes
FAIL - 3

I tried to execute the reproducer on the host (where kvm VM-s are
running), but the bug was not triggered during one hour.

When I executed the reproducer in VM without stopping processes on the
host, I found that a bug is triggered much faster in this case.

[root@fc22-vm ptrace]# ./ptrace_breakpoint 1
....
stop 24675
cont
child2 1
stop 24676
cont
child2 1
child2 5
0088: exit - 5
stop 24677
0132: exited, status=1
cont
0127: wait: No child processes

I know that this bug can be reproduced starting with the 4.2 kernel. I
haven't test older versions of the kernel.

I tried to print drX registers after a break-point. Looks like they
are set correctly.

Maybe someone has any ideas where a problem is or how it can be investigated.

Here is a criu issue for this problem:
https://github.com/xemul/criu/issues/107

Thanks,
Andrew
#define _GNU_SOURCE         /* See feature_test_macros(7) */
#include <sys/syscall.h>   /* For SYS_xxx definitions */

#include <unistd.h>
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/ptrace.h>
#include <errno.h>
#include <sys/user.h>
#include <asm/debugreg.h>
#include <string.h>

/* Copied from the gdb header gdb/nat/x86-dregs.h */

/* Debug registers' indices.  */
#define DR_FIRSTADDR 0
#define DR_LASTADDR  3
#define DR_NADDR     4  /* The number of debug address registers.  */
#define DR_STATUS    6  /* Index of debug status register (DR6).  */
#define DR_CONTROL   7  /* Index of debug control register (DR7).  */

#define DR_LOCAL_ENABLE_SHIFT   0 /* Extra shift to the local enable bit.  */
#define DR_GLOBAL_ENABLE_SHIFT  1 /* Extra shift to the global enable bit.  */
#define DR_ENABLE_SIZE          2 /* Two enable bits per debug register.  */

/* Locally enable the break/watchpoint in the I'th debug register.  */
#define X86_DR_LOCAL_ENABLE(i) (1 << (DR_LOCAL_ENABLE_SHIFT + DR_ENABLE_SIZE * (i)))


# define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#define pr_perror(fmt, ...)                                             \
		fprintf(stderr, "%04d: " fmt ": %s\n", __LINE__, ##__VA_ARGS__, strerror(errno))
#define pr_err(fmt, ...)                                             \
		fprintf(stderr, "%04d: " fmt "\n", __LINE__, ##__VA_ARGS__)

int ptrace_set_breakpoint(pid_t pid, void *addr)
{
        int ret;

        /* Set a breakpoint */
        if (ptrace(PTRACE_POKEUSER, pid,
                        offsetof(struct user, u_debugreg[DR_FIRSTADDR]),
                        addr)) {
                pr_perror("Unable to setup a breakpoint into %d", pid);
                return -1;
        }

        /* Enable the breakpoint */
        if (ptrace(PTRACE_POKEUSER, pid,
                        offsetof(struct user, u_debugreg[DR_CONTROL]),
                        X86_DR_LOCAL_ENABLE(DR_FIRSTADDR))) {
                pr_perror("Unable to enable the breakpoint for %d", pid);
                return -1;
        }

        ret = ptrace(PTRACE_CONT, pid, NULL, NULL);
        if (ret) {
                pr_perror("Unable to restart the  stopped tracee process %d", pid);
                return -1;
        }

        return 1;
}

static long xxx = -1;
int child()
{
	printf("child %d\n", xxx);
	syscall(__NR_getppid);
	if (xxx == 5) {
		pr_err("exit - %d", xxx);
		exit(1);
	}
	if (xxx > 0)
		xxx = 5;
	return 0;
}

int child2()
{
	printf("child2 %d\n", xxx);
	syscall(__NR_getppid);
	if (xxx == 5) {
		pr_err("exit - %d", xxx);
		exit(1);
	}
	if (xxx > 0)
		xxx = 5;
	return 0;
}

int main(int argc, char **argv)
{
	pid_t pid;
	int status, i = 0;
	int (*addr)();

	if (argc > 1)
		addr = child2;
	else
		addr = child;

	pid = fork();
	if (pid == 0) {
		while (1)
			addr();
	}

	if (ptrace(PTRACE_ATTACH, pid, NULL, NULL)) {
		pr_perror("ptrace");
		return 1;
	}
	if (waitpid(pid, &status, 0) == -1) {
		pr_perror("waitpid");
		return 1;
	}

	if (ptrace_set_breakpoint(pid, addr) != 1)
		return 2;

	while (1) {
		if (waitpid(pid, &status, 0) != pid) {
			pr_perror("wait");
			return 3;
		}
		printf("stop %d\n", i++);
		if (WIFEXITED(status)) {
			pr_err("exited, status=%d", WEXITSTATUS(status));
		} else if (WIFSIGNALED(status)) {
			pr_err("killed by signal %d", WTERMSIG(status));
		}
		ptrace(PTRACE_POKEDATA, pid, &xxx, 1);
		printf("cont\n");
		ptrace(PTRACE_CONT, pid, NULL, NULL);
	}


	return 0;
}