[PATCH 4/4] x86-64: Inline vdso clock_gettime helpers

[Andy Lutomirski]

This is about a 6% speedup on Sandy Bridge.

Signed-off-by: Andy Lutomirski <luto@xxxxxxxxxxxxxx>
---
 arch/x86/vdso/vclock_gettime.c |   17 +++++++++--------
 1 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index fe960ac..40eead5 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -82,7 +82,8 @@ notrace static inline long vgetns(void)
 	return (v * gtod->clock.mult) >> gtod->clock.shift;
 }
 
-notrace static noinline void do_realtime(struct timespec *ts)
+/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
+notrace static void __always_inline do_realtime(struct timespec *ts)
 {
 	unsigned long seq, ns;
 	do {
@@ -94,7 +95,7 @@ notrace static noinline void do_realtime(struct timespec *ts)
 	timespec_add_ns(ts, ns);
 }
 
-notrace static noinline u64 do_realtime_ns(void)
+notrace static u64 do_realtime_ns(void)
 {
 	unsigned long seq, ns;
 	do {
@@ -104,7 +105,7 @@ notrace static noinline u64 do_realtime_ns(void)
 	return ns;
 }
 
-notrace static noinline void do_monotonic(struct timespec *ts)
+notrace static void do_monotonic(struct timespec *ts)
 {
 	unsigned long seq, ns;
 	do {
@@ -116,7 +117,7 @@ notrace static noinline void do_monotonic(struct timespec *ts)
 	timespec_add_ns(ts, ns);
 }
 
-notrace static noinline u64 do_monotonic_ns(void)
+notrace static u64 do_monotonic_ns(void)
 {
 	unsigned long seq, ns;
 	do {
@@ -126,7 +127,7 @@ notrace static noinline u64 do_monotonic_ns(void)
 	return ns;
 }
 
-notrace static noinline void do_realtime_coarse(struct timespec *ts)
+notrace static void do_realtime_coarse(struct timespec *ts)
 {
 	unsigned long seq;
 	do {
@@ -136,13 +137,13 @@ notrace static noinline void do_realtime_coarse(struct timespec *ts)
 	} while (unlikely(read_seqretry(&gtod->lock, seq)));
 }
 
-notrace static noinline u64 do_realtime_coarse_ns(void)
+notrace static u64 do_realtime_coarse_ns(void)
 {
 	/* This is atomic on x86-64. */
 	return ACCESS_ONCE(gtod->wall_time_coarse_flat_ns);
 }
 
-notrace static noinline void do_monotonic_coarse(struct timespec *ts)
+notrace static void do_monotonic_coarse(struct timespec *ts)
 {
 	unsigned long seq;
 	do {
@@ -152,7 +153,7 @@ notrace static noinline void do_monotonic_coarse(struct timespec *ts)
 	} while (unlikely(read_seqretry(&gtod->lock, seq)));
 }
 
-notrace static noinline u64 do_monotonic_coarse_ns(void)
+notrace static u64 do_monotonic_coarse_ns(void)
 {
 	/* This is atomic on x86-64. */
 	return ACCESS_ONCE(gtod->monotonic_time_coarse_flat_ns);
-- 
1.7.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/