Re: Add prefetch switch stack hook in scheduler function

[Eric Dumazet]

Ingo Molnar a écrit :


> unroll prefetch_range() loops manually.
>
> Signed-off-by: Ingo Molnar <mingo@xxxxxxx>
>
>  include/linux/prefetch.h |   31 +++++++++++++++++++++++++++++--
>  1 files changed, 29 insertions(+), 2 deletions(-)
>
> Index: linux/include/linux/prefetch.h
> ===================================================================
> --- linux.orig/include/linux/prefetch.h
> +++ linux/include/linux/prefetch.h
> @@ -58,11 +58,38 @@ static inline void prefetchw(const void 
>  static inline void prefetch_range(void *addr, size_t len)
>  {
>  #ifdef ARCH_HAS_PREFETCH
> -	char *cp;
> +	char *cp = addr;
>  	char *end = addr + len;
>  
> -	for (cp = addr; cp < end; cp += PREFETCH_STRIDE)
> +	/*
> +	 * Unroll agressively:
> +	 */
> +	if (len <= PREFETCH_STRIDE)
>  		prefetch(cp);
> +	else if (len <= 2*PREFETCH_STRIDE) {
> +		prefetch(cp);
> +		prefetch(cp + PREFETCH_STRIDE);
> +	}
> +	else if (len <= 3*PREFETCH_STRIDE) {
> +		prefetch(cp);
> +		prefetch(cp + PREFETCH_STRIDE);
> +		prefetch(cp + 2*PREFETCH_STRIDE);
> +	}
> +	else if (len <= 4*PREFETCH_STRIDE) {
> +		prefetch(cp);
> +		prefetch(cp + PREFETCH_STRIDE);
> +		prefetch(cp + 2*PREFETCH_STRIDE);
> +		prefetch(cp + 3*PREFETCH_STRIDE);
> +	}
> +	else if (len <= 5*PREFETCH_STRIDE) {
> +		prefetch(cp);
> +		prefetch(cp + PREFETCH_STRIDE);
> +		prefetch(cp + 2*PREFETCH_STRIDE);
> +		prefetch(cp + 3*PREFETCH_STRIDE);
> +		prefetch(cp + 4*PREFETCH_STRIDE);
> +	} else
> +		for (; cp < end; cp += PREFETCH_STRIDE)
> +			prefetch(cp);
>  #endif
>  }
>  
> -

Please test that len is a constant, or else the inlining is too large for the non constant case.

Thank you
static inline void prefetch_range(void *addr, size_t len)
{
	char *cp;
	char *end = addr + len;

	if (__builtin_constant_p(len) && (len <= 5*PREFETCH_STRIDE)) {
		if (len <= PREFETCH_STRIDE)
	 		prefetch(cp);
		else if (len <= 2*PREFETCH_STRIDE) {
			prefetch(cp);
			prefetch(cp + PREFETCH_STRIDE);
		}
		else if (len <= 3*PREFETCH_STRIDE) {
			prefetch(cp);
			prefetch(cp + PREFETCH_STRIDE);
			prefetch(cp + 2*PREFETCH_STRIDE);
		}
		else if (len <= 4*PREFETCH_STRIDE) {
			prefetch(cp);
			prefetch(cp + PREFETCH_STRIDE);
			prefetch(cp + 2*PREFETCH_STRIDE);
			prefetch(cp + 3*PREFETCH_STRIDE);
		}
		else if (len <= 5*PREFETCH_STRIDE) {
			prefetch(cp);
			prefetch(cp + PREFETCH_STRIDE);
			prefetch(cp + 2*PREFETCH_STRIDE);
			prefetch(cp + 3*PREFETCH_STRIDE);
			prefetch(cp + 4*PREFETCH_STRIDE);
		}
	}
	else
		for (; cp < end; cp += PREFETCH_STRIDE)
			prefetch(cp);
}