Re: [PATCH 2/2] Customize sched domain via cpuset

[Peter Zijlstra]

On Tue, 2008-04-01 at 20:27 +0900, Hidetoshi Seto wrote:
> The implementation is here.
> 
>  - Add 2 new cpuset files:
>      sched_wake_idle_far
>      sched_balance_newidle_far
> 
>  - Modify partition_sched_domains() and build_sched_domains()
>    to take flags parameter passed from cpuset.
> 
>  - Fill newidle_idx for node domains which currently unused but
>    might be required for sched_balance_newidle_far.

Just to be clear; the same effect can be had by poking into:

 /proc/sys/kernel/sched_domain/$cpu/$domain/flags

but this interface you now propose gives a more stable interface in that
you'd have to re-do your setting after every cpuset change (admittedly
those are rare, but I see how it could be a nuisance).

Or do you actually add something that wasn't available through the
initial domain interface?

> Signed-off-by: Hidetoshi Seto <seto.hidetoshi@xxxxxxxxxxxxxx>
> 
> ---
>  include/asm-ia64/topology.h |    2
>  include/asm-sh/topology.h   |    2
>  include/asm-x86/topology.h  |    2
>  include/linux/sched.h       |    4 +
>  kernel/cpuset.c             |   89 ++++++++++++++++++++++++++++++++++++++++++--
>  kernel/sched.c              |   38 ++++++++++++++++--
>  kernel/sched_fair.c         |    4 +
>  7 files changed, 128 insertions(+), 13 deletions(-)
> 
> Index: GIT-torvalds/kernel/sched_fair.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/sched_fair.c
> +++ GIT-torvalds/kernel/sched_fair.c
> @@ -957,7 +957,9 @@ static int wake_idle(int cpu, struct tas
>  		return cpu;
> 
>  	for_each_domain(cpu, sd) {
> -		if (sd->flags & SD_WAKE_IDLE) {
> +		if ((sd->flags & SD_WAKE_IDLE)
> +		    || ((sd->flags & SD_WAKE_IDLE_FAR)
> +			&& !task_hot(p, task_rq(p)->clock, sd))) {
>  			cpus_and(tmp, sd->span, p->cpus_allowed);
>  			for_each_cpu_mask(i, tmp) {
>  				if (idle_cpu(i)) {
> Index: GIT-torvalds/kernel/cpuset.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/cpuset.c
> +++ GIT-torvalds/kernel/cpuset.c
> @@ -126,6 +126,8 @@ typedef enum {
>  	CS_MEM_EXCLUSIVE,
>  	CS_MEMORY_MIGRATE,
>  	CS_SCHED_LOAD_BALANCE,
> +	CS_SCHED_BALANCE_NEWIDLE_FAR,
> +	CS_SCHED_WAKE_IDLE_FAR,
>  	CS_SPREAD_PAGE,
>  	CS_SPREAD_SLAB,
>  } cpuset_flagbits_t;
> @@ -146,6 +148,16 @@ static inline int is_sched_load_balance(
>  	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
>  }
> 
> +static inline int is_sched_balance_newidle_far(const struct cpuset *cs)
> +{
> +	return test_bit(CS_SCHED_BALANCE_NEWIDLE_FAR, &cs->flags);
> +}
> +
> +static inline int is_sched_wake_idle_far(const struct cpuset *cs)
> +{
> +	return test_bit(CS_SCHED_WAKE_IDLE_FAR, &cs->flags);
> +}
> +
>  static inline int is_memory_migrate(const struct cpuset *cs)
>  {
>  	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
> @@ -161,6 +173,11 @@ static inline int is_spread_slab(const s
>  	return test_bit(CS_SPREAD_SLAB, &cs->flags);
>  }
> 
> +static inline int is_sched_custom_domain(const struct cpuset *cs)
> +{
> +	return is_sched_balance_newidle_far(cs) || is_sched_wake_idle_far(cs);
> +}
> +
>  /*
>   * Increment this integer everytime any cpuset changes its
>   * mems_allowed value.  Users of cpusets can track this generation
> @@ -553,12 +570,14 @@ static void rebuild_sched_domains(void)
>  	int csn;		/* how many cpuset ptrs in csa so far */
>  	int i, j, k;		/* indices for partition finding loops */
>  	cpumask_t *doms;	/* resulting partition; i.e. sched domains */
> +	int *flags;		/* flags for custom sched domains */
>  	int ndoms;		/* number of sched domains in result */
>  	int nslot;		/* next empty doms[] cpumask_t slot */
> 
>  	q = NULL;
>  	csa = NULL;
>  	doms = NULL;
> +	flags = NULL;
> 
>  	/* Special case for the 99% of systems with one, full, sched domain */
>  	if (is_sched_load_balance(&top_cpuset)) {
> @@ -566,6 +585,13 @@ static void rebuild_sched_domains(void)
>  		doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
>  		if (!doms)
>  			goto rebuild;
> +		if (is_sched_custom_domain(&top_cpuset)) {
> +			flags = kzalloc(sizeof(int), GFP_KERNEL);
> +			if (flags && is_sched_balance_newidle_far(&top_cpuset))
> +				*flags |= SD_BALANCE_NEWIDLE;
> +			if (flags && is_sched_wake_idle_far(&top_cpuset))
> +				*flags |= SD_WAKE_IDLE_FAR;
> +		}
>  		*doms = top_cpuset.cpus_allowed;
>  		goto rebuild;
>  	}
> @@ -622,6 +648,7 @@ restart:
>  	doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL);
>  	if (!doms)
>  		goto rebuild;
> +	flags = kzalloc(ndoms * sizeof(int), GFP_KERNEL);
> 
>  	for (nslot = 0, i = 0; i < csn; i++) {
>  		struct cpuset *a = csa[i];
> @@ -650,6 +677,13 @@ restart:
>  				if (apn == b->pn) {
>  					cpus_or(*dp, *dp, b->cpus_allowed);
>  					b->pn = -1;
> +					if (flags
> +					    && is_sched_balance_newidle_far(b))
> +						*(flags + nslot) |=
> +							SD_BALANCE_NEWIDLE;
> +					if (flags && is_sched_wake_idle_far(b))
> +						*(flags + nslot) |=
> +							SD_WAKE_IDLE_FAR;
>  				}
>  			}
>  			nslot++;
> @@ -660,7 +694,7 @@ restart:
>  rebuild:
>  	/* Have scheduler rebuild sched domains */
>  	get_online_cpus();
> -	partition_sched_domains(ndoms, doms);
> +	partition_sched_domains(ndoms, doms, flags);
>  	put_online_cpus();
> 
>  done:
> @@ -668,6 +702,7 @@ done:
>  		kfifo_free(q);
>  	kfree(csa);
>  	/* Don't kfree(doms) -- partition_sched_domains() does that. */
> +	/* Don't kfree(flags) -- partition_sched_domains() does that. */
>  }
> 
>  static inline int started_after_time(struct task_struct *t1,
> @@ -1011,10 +1046,26 @@ static int update_memory_pressure_enable
>  	return 0;
>  }
> 
> +static int need_rebuild_domains(struct cpuset *cs, struct cpuset *tcs)
> +{
> +	if (is_sched_load_balance(cs) != is_sched_load_balance(tcs))
> +		return 1;
> +	if (!is_sched_load_balance(tcs))
> +		return 0;
> +	if (is_sched_balance_newidle_far(cs) !=
> +					is_sched_balance_newidle_far(tcs))
> +		return 1;
> +	if (is_sched_wake_idle_far(cs) != is_sched_wake_idle_far(tcs))
> +		return 1;
> +	return 0;
> +}
> +
>  /*
>   * update_flag - read a 0 or a 1 in a file and update associated flag
>   * bit:	the bit to update (CS_CPU_EXCLUSIVE, CS_MEM_EXCLUSIVE,
>   *				CS_SCHED_LOAD_BALANCE,
> + *				CS_SCHED_BALANCE_NEW_IDLE_FAR,
> + *				CS_SCHED_WAKE_IDLE_FAR,
>   *				CS_NOTIFY_ON_RELEASE, CS_MEMORY_MIGRATE,
>   *				CS_SPREAD_PAGE, CS_SPREAD_SLAB)
>   * cs:	the cpuset to update
> @@ -1043,8 +1094,7 @@ static int update_flag(cpuset_flagbits_t
>  		return err;
> 
>  	cpus_nonempty = !cpus_empty(trialcs.cpus_allowed);
> -	balance_flag_changed = (is_sched_load_balance(cs) !=
> -		 			is_sched_load_balance(&trialcs));
> +	balance_flag_changed = need_rebuild_domains(cs, &trialcs);
> 
>  	mutex_lock(&callback_mutex);
>  	cs->flags = trialcs.flags;
> @@ -1202,6 +1252,8 @@ typedef enum {
>  	FILE_CPU_EXCLUSIVE,
>  	FILE_MEM_EXCLUSIVE,
>  	FILE_SCHED_LOAD_BALANCE,
> +	FILE_SCHED_BALANCE_NEWIDLE_FAR,
> +	FILE_SCHED_WAKE_IDLE_FAR,
>  	FILE_MEMORY_PRESSURE_ENABLED,
>  	FILE_MEMORY_PRESSURE,
>  	FILE_SPREAD_PAGE,
> @@ -1256,6 +1308,12 @@ static ssize_t cpuset_common_file_write(
>  	case FILE_SCHED_LOAD_BALANCE:
>  		retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, buffer);
>  		break;
> +	case FILE_SCHED_BALANCE_NEWIDLE_FAR:
> +		retval = update_flag(CS_SCHED_BALANCE_NEWIDLE_FAR, cs, buffer);
> +		break;
> +	case FILE_SCHED_WAKE_IDLE_FAR:
> +		retval = update_flag(CS_SCHED_WAKE_IDLE_FAR, cs, buffer);
> +		break;
>  	case FILE_MEMORY_MIGRATE:
>  		retval = update_flag(CS_MEMORY_MIGRATE, cs, buffer);
>  		break;
> @@ -1354,6 +1412,12 @@ static ssize_t cpuset_common_file_read(s
>  	case FILE_SCHED_LOAD_BALANCE:
>  		*s++ = is_sched_load_balance(cs) ? '1' : '0';
>  		break;
> +	case FILE_SCHED_BALANCE_NEWIDLE_FAR:
> +		*s++ = is_sched_balance_newidle_far(cs) ? '1' : '0';
> +		break;
> +	case FILE_SCHED_WAKE_IDLE_FAR:
> +		*s++ = is_sched_wake_idle_far(cs) ? '1' : '0';
> +		break;
>  	case FILE_MEMORY_MIGRATE:
>  		*s++ = is_memory_migrate(cs) ? '1' : '0';
>  		break;
> @@ -1424,6 +1488,20 @@ static struct cftype cft_sched_load_bala
>  	.private = FILE_SCHED_LOAD_BALANCE,
>  };
> 
> +static struct cftype cft_sched_balance_newidle_far = {
> +	.name = "sched_balance_newidle_far",
> +	.read = cpuset_common_file_read,
> +	.write = cpuset_common_file_write,
> +	.private = FILE_SCHED_BALANCE_NEWIDLE_FAR,
> +};
> +
> +static struct cftype cft_sched_wake_idle_far = {
> +	.name = "sched_wake_idle_far",
> +	.read = cpuset_common_file_read,
> +	.write = cpuset_common_file_write,
> +	.private = FILE_SCHED_WAKE_IDLE_FAR,
> +};
> +
>  static struct cftype cft_memory_migrate = {
>  	.name = "memory_migrate",
>  	.read = cpuset_common_file_read,
> @@ -1475,6 +1553,11 @@ static int cpuset_populate(struct cgroup
>  		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_sched_load_balance)) < 0)
>  		return err;
> +	if ((err = cgroup_add_file(cont, ss,
> +					&cft_sched_balance_newidle_far)) < 0)
> +		return err;
> +	if ((err = cgroup_add_file(cont, ss, &cft_sched_wake_idle_far)) < 0)
> +		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_memory_pressure)) < 0)
>  		return err;
>  	if ((err = cgroup_add_file(cont, ss, &cft_spread_page)) < 0)
> Index: GIT-torvalds/include/linux/sched.h
> ===================================================================
> --- GIT-torvalds.orig/include/linux/sched.h
> +++ GIT-torvalds/include/linux/sched.h
> @@ -704,6 +704,7 @@ enum cpu_idle_type {
>  #define SD_POWERSAVINGS_BALANCE	256	/* Balance for power savings */
>  #define SD_SHARE_PKG_RESOURCES	512	/* Domain members share cpu pkg resources */
>  #define SD_SERIALIZE		1024	/* Only a single load balancing instance */
> +#define SD_WAKE_IDLE_FAR	2048	/* Gain latency sacrificing cache hit */
> 
>  #define BALANCE_FOR_MC_POWER	\
>  	(sched_smt_power_savings ? SD_POWERSAVINGS_BALANCE : 0)
> @@ -789,7 +790,8 @@ struct sched_domain {
>  #endif
>  };
> 
> -extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new);
> +extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
> +				    int *flags_new);
>  extern int arch_reinit_sched_domains(void);
> 
>  #endif	/* CONFIG_SMP */
> Index: GIT-torvalds/kernel/sched.c
> ===================================================================
> --- GIT-torvalds.orig/kernel/sched.c
> +++ GIT-torvalds/kernel/sched.c
> @@ -6586,7 +6586,7 @@ static void init_sched_groups_power(int
>   * Build sched domains for a given set of cpus and attach the sched domains
>   * to the individual cpus
>   */
> -static int build_sched_domains(const cpumask_t *cpu_map)
> +static int __build_sched_domains(const cpumask_t *cpu_map, int flags)
>  {
>  	int i;
>  	struct root_domain *rd;
> @@ -6627,6 +6627,7 @@ static int build_sched_domains(const cpu
>  			sd = &per_cpu(allnodes_domains, i);
>  			*sd = SD_ALLNODES_INIT;
>  			sd->span = *cpu_map;
> +			/* prohibit "sd->flags |= flags" for allnodes_domain */
>  			cpu_to_allnodes_group(i, cpu_map, &sd->groups);
>  			p = sd;
>  			sd_allnodes = 1;
> @@ -6636,6 +6637,7 @@ static int build_sched_domains(const cpu
>  		sd = &per_cpu(node_domains, i);
>  		*sd = SD_NODE_INIT;
>  		sd->span = sched_domain_node_span(cpu_to_node(i));
> +		sd->flags |= flags;
>  		sd->parent = p;
>  		if (p)
>  			p->child = sd;
> @@ -6646,6 +6648,7 @@ static int build_sched_domains(const cpu
>  		sd = &per_cpu(phys_domains, i);
>  		*sd = SD_CPU_INIT;
>  		sd->span = nodemask;
> +		sd->flags |= flags;
>  		sd->parent = p;
>  		if (p)
>  			p->child = sd;
> @@ -6657,6 +6660,7 @@ static int build_sched_domains(const cpu
>  		*sd = SD_MC_INIT;
>  		sd->span = cpu_coregroup_map(i);
>  		cpus_and(sd->span, sd->span, *cpu_map);
> +		sd->flags |= flags;
>  		sd->parent = p;
>  		p->child = sd;
>  		cpu_to_core_group(i, cpu_map, &sd->groups);
> @@ -6668,6 +6672,7 @@ static int build_sched_domains(const cpu
>  		*sd = SD_SIBLING_INIT;
>  		sd->span = per_cpu(cpu_sibling_map, i);
>  		cpus_and(sd->span, sd->span, *cpu_map);
> +		sd->flags |= flags;
>  		sd->parent = p;
>  		p->child = sd;
>  		cpu_to_cpu_group(i, cpu_map, &sd->groups);
> @@ -6840,8 +6845,14 @@ error:
>  #endif
>  }
> 
> +static int build_sched_domains(const cpumask_t *cpu_map)
> +{
> +	return __build_sched_domains(cpu_map, 0);
> +}
> +
>  static cpumask_t *doms_cur;	/* current sched domains */
>  static int ndoms_cur;		/* number of sched domains in 'doms_cur' */
> +static int *flags_cur;		/* custom flags of domains in 'doms_cur' */
> 
>  /*
>   * Special case: If a kmalloc of a doms_cur partition (array of
> @@ -6868,6 +6879,7 @@ static int arch_init_sched_domains(const
>  	doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
>  	if (!doms_cur)
>  		doms_cur = &fallback_doms;
> +	flags_cur = NULL;
>  	cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
>  	err = build_sched_domains(doms_cur);
>  	register_sched_domain_sysctl();
> @@ -6896,6 +6908,16 @@ static void detach_destroy_domains(const
>  	arch_destroy_sched_domains(cpu_map);
>  }
> 
> +/* handle null as 0s array */
> +static inline int flags_equal(int *cur, int idx_cur, int *new, int idx_new)
> +{
> +	if (!new)
> +		return (!cur || !cur[idx_cur]);
> +	if (!cur)
> +		return (!new[idx_new]);
> +	return (cur[idx_cur] == new[idx_new]);
> +}
> +
>  /*
>   * Partition sched domains as specified by the 'ndoms_new'
>   * cpumasks in the array doms_new[] of cpumasks. This compares
> @@ -6917,7 +6939,7 @@ static void detach_destroy_domains(const
>   *
>   * Call with hotplug lock held
>   */
> -void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
> +void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, int *flags_new)
>  {
>  	int i, j;
> 
> @@ -6929,13 +6951,15 @@ void partition_sched_domains(int ndoms_n
>  	if (doms_new == NULL) {
>  		ndoms_new = 1;
>  		doms_new = &fallback_doms;
> +		flags_new = NULL;
>  		cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
>  	}
> 
>  	/* Destroy deleted domains */
>  	for (i = 0; i < ndoms_cur; i++) {
>  		for (j = 0; j < ndoms_new; j++) {
> -			if (cpus_equal(doms_cur[i], doms_new[j]))
> +			if (cpus_equal(doms_cur[i], doms_new[j])
> +			    && flags_equal(flags_cur, i, flags_new, j))
>  				goto match1;
>  		}
>  		/* no match - a current sched domain not in new doms_new[] */
> @@ -6947,11 +6971,13 @@ match1:
>  	/* Build new domains */
>  	for (i = 0; i < ndoms_new; i++) {
>  		for (j = 0; j < ndoms_cur; j++) {
> -			if (cpus_equal(doms_new[i], doms_cur[j]))
> +			if (cpus_equal(doms_new[i], doms_cur[j])
> +			    && flags_equal(flags_new, i, flags_cur, j))
>  				goto match2;
>  		}
>  		/* no match - add a new doms_new */
> -		build_sched_domains(doms_new + i);
> +		__build_sched_domains(doms_new + i,
> +					flags_new ? flags_new[i] : 0);
>  match2:
>  		;
>  	}
> @@ -6959,7 +6985,9 @@ match2:
>  	/* Remember the new sched domains */
>  	if (doms_cur != &fallback_doms)
>  		kfree(doms_cur);
> +	kfree(flags_cur);	/* kfree(NULL) is safe */
>  	doms_cur = doms_new;
> +	flags_cur = flags_new;
>  	ndoms_cur = ndoms_new;
> 
>  	register_sched_domain_sysctl();
> Index: GIT-torvalds/include/asm-ia64/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-ia64/topology.h
> +++ GIT-torvalds/include/asm-ia64/topology.h
> @@ -93,7 +93,7 @@ void build_cpu_to_node_map(void);
>  	.cache_nice_tries	= 2,			\
>  	.busy_idx		= 3,			\
>  	.idle_idx		= 2,			\
> -	.newidle_idx		= 0, /* unused */	\
> +	.newidle_idx		= 2,			\
>  	.wake_idx		= 1,			\
>  	.forkexec_idx		= 1,			\
>  	.flags			= SD_LOAD_BALANCE	\
> Index: GIT-torvalds/include/asm-sh/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-sh/topology.h
> +++ GIT-torvalds/include/asm-sh/topology.h
> @@ -16,7 +16,7 @@
>  	.cache_nice_tries	= 2,			\
>  	.busy_idx		= 3,			\
>  	.idle_idx		= 2,			\
> -	.newidle_idx		= 0,			\
> +	.newidle_idx		= 2,			\
>  	.wake_idx		= 1,			\
>  	.forkexec_idx		= 1,			\
>  	.flags			= SD_LOAD_BALANCE	\
> Index: GIT-torvalds/include/asm-x86/topology.h
> ===================================================================
> --- GIT-torvalds.orig/include/asm-x86/topology.h
> +++ GIT-torvalds/include/asm-x86/topology.h
> @@ -129,7 +129,7 @@ extern unsigned long node_remap_size[];
> 
>  # define SD_CACHE_NICE_TRIES	2
>  # define SD_IDLE_IDX		2
> -# define SD_NEWIDLE_IDX		0
> +# define SD_NEWIDLE_IDX		2
>  # define SD_FORKEXEC_IDX	1
> 
>  #endif
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/