Re: [PATCH linux-2.6.36-rc7] sunrpc/xprtrdma: clean up workqueueusage

[J. Bruce Fields]

Tom, could you take a look?

--b.

On Fri, Oct 15, 2010 at 05:49:27PM +0200, Tejun Heo wrote:
> * Create and use svc_rdma_wq instead of using the system workqueue and
>   flush_scheduled_work().  This workqueue is necessary to serve as
>   flushing domain for rdma->sc_work which is used to destroy itself
>   and thus can't be flushed explicitly.
> 
> * Replace cancel_delayed_work() + flush_scheduled_work() with
>   cancel_delayed_work_sync().
> 
> * Implement synchronous connect in xprt_rdma_connect() using
>   flush_delayed_work() on the rdma_connect work instead of using
>   flush_scheduled_work().
> 
> This is to prepare for the deprecation and removal of
> flush_scheduled_work().
> 
> Signed-off-by: Tejun Heo <tj@xxxxxxxxxx>
> ---
>  net/sunrpc/xprtrdma/svc_rdma.c           |   11 ++++++++++-
>  net/sunrpc/xprtrdma/svc_rdma_transport.c |    6 +++++-
>  net/sunrpc/xprtrdma/transport.c          |    5 ++---
>  3 files changed, 17 insertions(+), 5 deletions(-)
> 
> Index: work/net/sunrpc/xprtrdma/svc_rdma.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/svc_rdma.c
> +++ work/net/sunrpc/xprtrdma/svc_rdma.c
> @@ -43,6 +43,7 @@
>  #include <linux/slab.h>
>  #include <linux/fs.h>
>  #include <linux/sysctl.h>
> +#include <linux/workqueue.h>
>  #include <linux/sunrpc/clnt.h>
>  #include <linux/sunrpc/sched.h>
>  #include <linux/sunrpc/svc_rdma.h>
> @@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
>  struct kmem_cache *svc_rdma_map_cachep;
>  struct kmem_cache *svc_rdma_ctxt_cachep;
> 
> +struct workqueue_struct *svc_rdma_wq;
> +
>  /*
>   * This function implements reading and resetting an atomic_t stat
>   * variable through read/write to a proc file. Any write to the file
> @@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
>  void svc_rdma_cleanup(void)
>  {
>  	dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
> -	flush_scheduled_work();
> +	destroy_workqueue(svc_rdma_wq);
>  	if (svcrdma_table_header) {
>  		unregister_sysctl_table(svcrdma_table_header);
>  		svcrdma_table_header = NULL;
> @@ -249,6 +252,11 @@ int svc_rdma_init(void)
>  	dprintk("\tsq_depth         : %d\n",
>  		svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
>  	dprintk("\tmax_inline       : %d\n", svcrdma_max_req_size);
> +
> +	svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
> +	if (!svc_rdma_wq)
> +		return -ENOMEM;
> +
>  	if (!svcrdma_table_header)
>  		svcrdma_table_header =
>  			register_sysctl_table(svcrdma_root_table);
> @@ -283,6 +291,7 @@ int svc_rdma_init(void)
>  	kmem_cache_destroy(svc_rdma_map_cachep);
>   err0:
>  	unregister_sysctl_table(svcrdma_table_header);
> +	destroy_workqueue(svc_rdma_wq);
>  	return -ENOMEM;
>  }
>  MODULE_AUTHOR("Tom Tucker <tom@xxxxxxxxxxxxxxxxxxxxx>");
> Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/svc_rdma_transport.c
> +++ work/net/sunrpc/xprtrdma/svc_rdma_transport.c
> @@ -45,6 +45,7 @@
>  #include <linux/sched.h>
>  #include <linux/slab.h>
>  #include <linux/spinlock.h>
> +#include <linux/workqueue.h>
>  #include <rdma/ib_verbs.h>
>  #include <rdma/rdma_cm.h>
>  #include <linux/sunrpc/svc_rdma.h>
> @@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
>  /* WR context cache. Created in svc_rdma.c  */
>  extern struct kmem_cache *svc_rdma_ctxt_cachep;
> 
> +/* Workqueue created in svc_rdma.c */
> +extern struct workqueue_struct *svc_rdma_wq;
> +
>  struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
>  {
>  	struct svc_rdma_op_ctxt *ctxt;
> @@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
>  	struct svcxprt_rdma *rdma =
>  		container_of(xprt, struct svcxprt_rdma, sc_xprt);
>  	INIT_WORK(&rdma->sc_work, __svc_rdma_free);
> -	schedule_work(&rdma->sc_work);
> +	queue_work(svc_rdma_wq, &rdma->sc_work);
>  }
> 
>  static int svc_rdma_has_wspace(struct svc_xprt *xprt)
> Index: work/net/sunrpc/xprtrdma/transport.c
> ===================================================================
> --- work.orig/net/sunrpc/xprtrdma/transport.c
> +++ work/net/sunrpc/xprtrdma/transport.c
> @@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
> 
>  	dprintk("RPC:       %s: called\n", __func__);
> 
> -	cancel_delayed_work(&r_xprt->rdma_connect);
> -	flush_scheduled_work();
> +	cancel_delayed_work_sync(&r_xprt->rdma_connect);
> 
>  	xprt_clear_connected(xprt);
> 
> @@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
>  	} else {
>  		schedule_delayed_work(&r_xprt->rdma_connect, 0);
>  		if (!RPC_IS_ASYNC(task))
> -			flush_scheduled_work();
> +			flush_delayed_work(&r_xprt->rdma_connect);
>  	}
>  }
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/