/*

 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.

 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.

 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.

 *

 * This software is available to you under a choice of one of two

 * licenses.  You may choose to be licensed under the terms of the GNU

 * General Public License (GPL) Version 2, available from the file

 * COPYING in the main directory of this source tree, or the

 * OpenIB.org BSD license below:

 *

 *     Redistribution and use in source and binary forms, with or

 *     without modification, are permitted provided that the following

 *     conditions are met:

 *

 *      - Redistributions of source code must retain the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer.

 *

 *      - Redistributions in binary form must reproduce the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer in the documentation and/or other materials

 *        provided with the distribution.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

 * SOFTWARE.

 *

 */

/*

 * Abstract:

 * 	Declaration of dispatcher abstraction.

 */

#ifndef _CL_DISPATCHER_H_

#define _CL_DISPATCHER_H_

#include <complib/cl_atomic.h>

#include <complib/cl_threadpool.h>

#include <complib/cl_qlist.h>

#include <complib/cl_qpool.h>

#include <complib/cl_spinlock.h>

#include <complib/cl_ptr_vector.h>

#ifdef __cplusplus

#  define BEGIN_C_DECLS extern "C" {

#  define END_C_DECLS   }

#else				/* !__cplusplus */

#  define BEGIN_C_DECLS

#  define END_C_DECLS

#endif				/* __cplusplus */

BEGIN_C_DECLS

/****h* Component Library/Dispatcher

* NAME

*	Dispatcher

*

* DESCRIPTION

*	The Dispatcher provides a facility for message routing to

*	asynchronous worker threads.

*

*	The Dispatcher functions operate on a cl_dispatcher_t structure

*	which should be treated as opaque and should be manipulated

*	only through the provided functions.

*

* SEE ALSO

*	Structures:

*		cl_dispatcher_t

*

*	Initialization/Destruction:

*		cl_disp_construct, cl_disp_init, cl_disp_shutdown, cl_disp_destroy

*

*	Manipulation:

*		cl_disp_post, cl_disp_register, cl_disp_unregister

*********/

/****s* Component Library: Dispatcher/cl_disp_msgid_t

* NAME

*	cl_disp_msgid_t

*

* DESCRIPTION

*	Defines the type of dispatcher messages.

*

* SYNOPSIS

*/

typedef uint32_t cl_disp_msgid_t;

/**********/

/****s* Component Library: Dispatcher/CL_DISP_MSGID_NONE

* NAME

*	CL_DISP_MSGID_NONE

*

* DESCRIPTION

*	Defines a message value that means "no message".

*	This value is used during registration by Dispatcher clients

*	that do not wish to receive messages.

*

*	No Dispatcher message is allowed to have this value.

*

* SYNOPSIS

*/

#define CL_DISP_MSGID_NONE	0xFFFFFFFF

/**********/

/****s* Component Library: Dispatcher/CL_DISP_INVALID_HANDLE

* NAME

*	CL_DISP_INVALID_HANDLE

*

* DESCRIPTION

*	Defines the value of an invalid Dispatcher registration handle.

*

* SYNOPSIS

*/

#define CL_DISP_INVALID_HANDLE ((cl_disp_reg_handle_t)0)

/*********/

/****f* Component Library: Dispatcher/cl_pfn_msgrcv_cb_t

* NAME

*	cl_pfn_msgrcv_cb_t

*

* DESCRIPTION

*	This typedef defines the prototype for client functions invoked

*	by the Dispatcher.  The Dispatcher calls the corresponding

*	client function when delivering a message to the client.

*

*	The client function must be reentrant if the user creates a

*	Dispatcher with more than one worker thread.

*

* SYNOPSIS

*/

typedef void

 (*cl_pfn_msgrcv_cb_t) (IN void *context, IN void *p_data);

/*

* PARAMETERS

*	context

*		[in] Client specific context specified in a call to

*		cl_disp_register

*

*	p_data

*		[in] Pointer to the client specific data payload

*		of this message.

*

* RETURN VALUE

*	This function does not return a value.

*

* NOTES

*	This typedef provides a function prototype reference for

*	the function provided by Dispatcher clients as a parameter

*	to the cl_disp_register function.

*

* SEE ALSO

*	Dispatcher, cl_disp_register

*********/

/****f* Component Library: Dispatcher/cl_pfn_msgdone_cb_t

* NAME

*	cl_pfn_msgdone_cb_t

*

* DESCRIPTION

*	This typedef defines the prototype for client functions invoked

*	by the Dispatcher.  The Dispatcher calls the corresponding

*	client function after completing delivery of a message.

*

*	The client function must be reentrant if the user creates a

*	Dispatcher with more than one worker thread.

*

* SYNOPSIS

*/

typedef void

 (*cl_pfn_msgdone_cb_t) (IN void *context, IN void *p_data);

/*

* PARAMETERS

*	context

*		[in] Client specific context specified in a call to

*		cl_disp_post

*

*	p_data

*		[in] Pointer to the client specific data payload

*		of this message.

*

* RETURN VALUE

*	This function does not return a value.

*

* NOTES

*	This typedef provides a function prototype reference for

*	the function provided by Dispatcher clients as a parameter

*	to the cl_disp_post function.

*

* SEE ALSO

*	Dispatcher, cl_disp_post

*********/

/****s* Component Library: Dispatcher/cl_dispatcher_t

* NAME

*	cl_dispatcher_t

*

* DESCRIPTION

*	Dispatcher structure.

*

*	The Dispatcher is thread safe.

*

*	The cl_dispatcher_t structure should be treated as opaque and should

*	be manipulated only through the provided functions.

*

* SYNOPSIS

*/

typedef struct _cl_dispatcher {

	cl_spinlock_t lock;

	cl_ptr_vector_t reg_vec;

	cl_qlist_t reg_list;

	cl_thread_pool_t worker_threads;

	cl_qlist_t msg_fifo;

	cl_qpool_t msg_pool;

	uint64_t last_msg_queue_time_us;

} cl_dispatcher_t;

/*

* FIELDS

*	lock

*		Spinlock to guard internal structures.

*

*	reg_vec

*		Vector of registration info objects.  Indexed by message msg_id.

*

*	reg_list

*		List of registration info objects.

*

*	worker_threads

*		Thread pool of worker threads to dispose of posted messages.

*

*	msg_fifo

*		FIFO of messages being processed by the Dispatcher.  New

*		messages are posted to the tail of the FIFO.  Worker threads

*		pull messages from the front.

*

*	msg_pool

*		Pool of message objects to be processed through the FIFO.

*

*	last_msg_queue_time_us

*		The time that the last message spent in the Q in usec

*

* SEE ALSO

*	Dispatcher

*********/

/****s* Component Library: Dispatcher/cl_disp_reg_info_t

* NAME

*	cl_disp_reg_info_t

*

* DESCRIPTION

*	Defines the dispatcher registration object structure.

*

*	The cl_disp_reg_info_t structure is for internal use by the

*	Dispatcher only.

*

* SYNOPSIS

*/

typedef struct _cl_disp_reg_info {

	cl_list_item_t list_item;

	cl_pfn_msgrcv_cb_t pfn_rcv_callback;

	const void *context;

	atomic32_t ref_cnt;

	cl_disp_msgid_t msg_id;

	cl_dispatcher_t *p_disp;

} cl_disp_reg_info_t;

/*

* FIELDS

* 	list_item

* 		List linkage.  Must be first element in the structure!!

*

*	pfn_rcv_callback

*		Client's message receive callback.

*

*	context

*		Client's context for message receive callback.

*

*	ref_cnt

*		Reference count.

*

*	msg_id

*		Dispatcher message msg_id value for this registration object.

*

*	p_disp

*		Pointer to parent Dispatcher.

*

* SEE ALSO

*********/

/****s* Component Library: Dispatcher/cl_disp_msg_t

* NAME

*	cl_disp_msg_t

*

* DESCRIPTION

*	Defines the dispatcher message structure.

*

*	The cl_disp_msg_t structure is for internal use by the

*	Dispatcher only.

*

* SYNOPSIS

*/

typedef struct _cl_disp_msg {

	cl_pool_item_t item;

	const void *p_data;

	cl_disp_reg_info_t *p_src_reg;

	cl_disp_reg_info_t *p_dest_reg;

	cl_pfn_msgdone_cb_t pfn_xmt_callback;

	uint64_t in_time;

	const void *context;

} cl_disp_msg_t;

/*

* FIELDS

*	item

*		List & Pool linkage.  Must be first element in the structure!!

*

*	p_data

*		Pointer to the data payload for this message.  The payload

*		is opaque to the Dispatcher.

*

*	p_src_reg

*		Pointer to the registration info of the sender.

*

* 	p_dest_reg

* 		Pointer to the registration info of the recipient.

*

*	pfn_xmt_callback

*		Client's message done callback.

*

*       in_time

*               The absolute time the message was inserted into the queue

*

*	context

*		Client's message done callback context.

*

* SEE ALSO

*********/

/****s* Component Library: Dispatcher/cl_disp_reg_info_t

* NAME

*	cl_disp_reg_info_t

*

* DESCRIPTION

*	Defines the Dispatcher registration handle.  This handle

*	should be treated as opaque by the client.

*

* SYNOPSIS

*/

typedef const struct _cl_disp_reg_info *cl_disp_reg_handle_t;

/**********/

/****f* Component Library: Dispatcher/cl_disp_construct

* NAME

*	cl_disp_construct

*

* DESCRIPTION

*	This function constructs a Dispatcher object.

*

* SYNOPSIS

*/

void cl_disp_construct(IN cl_dispatcher_t * const p_disp);

/*

* PARAMETERS

*	p_disp

*		[in] Pointer to a Dispatcher.

*

* RETURN VALUE

*	This function does not return a value.

*

* NOTES

*	Allows calling cl_disp_init and cl_disp_destroy.

*

* SEE ALSO

*	Dispatcher, cl_disp_init, cl_disp_destroy

*********/

/****f* Component Library: Dispatcher/cl_disp_init

* NAME

*	cl_disp_init

*

* DESCRIPTION

*	This function initializes a Dispatcher object.

*

* SYNOPSIS

*/

cl_status_t

cl_disp_init(IN cl_dispatcher_t * const p_disp,

	     IN const uint32_t thread_count, IN const char *const name);

/*

* PARAMETERS

*	p_disp

*		[in] Pointer to a Dispatcher.

*

*	thread_count

*		[in] The number of worker threads to create in this Dispatcher.

*		A value of 0 causes the Dispatcher to create one worker thread

*		per CPU in the system.  When the Dispatcher is created with

*		only one thread, the Dispatcher guarantees to deliver posted

*		messages in order.  When the Dispatcher is created with more

*		than one thread, messages may be delivered out of order.

*

*	name

*		[in] Name to associate with the threads.  The name may be up to 16

*		characters, including a terminating null character.  All threads

*		created in the Dispatcher have the same name.

*

* RETURN VALUE

*	CL_SUCCESS if the operation is successful.

*

* SEE ALSO

*	Dispatcher, cl_disp_destroy, cl_disp_register, cl_disp_unregister,

*	cl_disp_post

*********/

/****f* Component Library: Dispatcher/cl_disp_shutdown

* NAME

*	cl_disp_shutdown

*

* DESCRIPTION

*	This function shutdown a Dispatcher object. So it unreg all messages and

*  clears the fifo and waits for the threads to exit

*

* SYNOPSIS

*/

void cl_disp_shutdown(IN cl_dispatcher_t * const p_disp);

/*

* PARAMETERS

*	p_disp

*		[in] Pointer to a Dispatcher.

*

* RETURN VALUE

*	This function does not return a value.

*

* NOTES

*	This function does not returns until all worker threads

*	have exited client callback functions and been successfully

*	shutdowned.

*

* SEE ALSO

*	Dispatcher, cl_disp_construct, cl_disp_init

*********/

/****f* Component Library: Dispatcher/cl_disp_destroy

* NAME

*	cl_disp_destroy

*

* DESCRIPTION

*	This function destroys a Dispatcher object.

*

* SYNOPSIS

*/

void cl_disp_destroy(IN cl_dispatcher_t * const p_disp);

/*

* PARAMETERS

*	p_disp

*		[in] Pointer to a Dispatcher.

*

* RETURN VALUE

*	This function does not return a value.

*

* SEE ALSO

*	Dispatcher, cl_disp_construct, cl_disp_init

*********/

/****f* Component Library: Dispatcher/cl_disp_register

* NAME

*	cl_disp_register

*

* DESCRIPTION

*	This function registers a client with a Dispatcher object.

*

* SYNOPSIS

*/

cl_disp_reg_handle_t

cl_disp_register(IN cl_dispatcher_t * const p_disp,

		 IN const cl_disp_msgid_t msg_id,

		 IN cl_pfn_msgrcv_cb_t pfn_callback OPTIONAL,

		 IN const void *const context);

/*

* PARAMETERS

*	p_disp

*		[in] Pointer to a Dispatcher.

*

*	msg_id

*		[in] Numberic message ID for which the client is registering.

*		If the client does not wish to receive any messages,

*		(a send-only client) then the caller should set this value

*		to CL_DISP_MSGID_NONE.  For efficiency, numeric message msg_id

*		values should start with 0 and should be contiguous, or nearly so.

*

*	pfn_callback

*		[in] Message receive callback.  The Dispatcher calls this

*		function after receiving a posted message with the

*		appropriate message msg_id value.  Send-only clients may specify

*		NULL for this value.

*

*	context

*		[in] Client context value passed to the cl_pfn_msgrcv_cb_t

*		function.

*

* RETURN VALUE

*	On success a Dispatcher registration handle.

*	CL_CL_DISP_INVALID_HANDLE otherwise.

*

* SEE ALSO

*	Dispatcher, cl_disp_unregister, cl_disp_post

*********/

/****f* Component Library: Dispatcher/cl_disp_unregister

* NAME

*	cl_disp_unregister

*

* DESCRIPTION

*	This function unregisters a client from a Dispatcher.

*

* SYNOPSIS

*/

void cl_disp_unregister(IN const cl_disp_reg_handle_t handle);

/*

* PARAMETERS

*	handle

*		[in] cl_disp_reg_handle_t value return by cl_disp_register.

*

* RETURN VALUE

*	This function does not return a value.

*

* NOTES

*	This function will not return until worker threads have exited

*	the callback functions for this client.  Do not invoke this

*	function from a callback.

*

* SEE ALSO

*	Dispatcher, cl_disp_register

*********/

/****f* Component Library: Dispatcher/cl_disp_post

* NAME

*	cl_disp_post

*

* DESCRIPTION

*	This function posts a message to a Dispatcher object.

*

* SYNOPSIS

*/

cl_status_t

cl_disp_post(IN const cl_disp_reg_handle_t handle,

	     IN const cl_disp_msgid_t msg_id,

	     IN const void *const p_data,

	     IN cl_pfn_msgdone_cb_t pfn_callback OPTIONAL,

	     IN const void *const context);

/*

* PARAMETERS

*	handle

*		[in] cl_disp_reg_handle_t value return by cl_disp_register.

*

*	msg_id

*		[in] Numeric message msg_id value associated with this message.

*

*	p_data

*		[in] Data payload for this message.

*

*	pfn_callback

*		[in] Pointer to a cl_pfn_msgdone_cb_t function.

*		The Dispatcher calls this function after the message has been

*		processed by the recipient.

*		The caller may pass NULL for this value, which indicates no

*		message done callback is necessary.

*

*	context

*		[in] Client context value passed to the cl_pfn_msgdone_cb_t

*		function.

*

* RETURN VALUE

*	CL_SUCCESS if the message was successfully queued in the Dispatcher.

*

* NOTES

*	The caller must not modify the memory pointed to by p_data until

*	the Dispatcher call the pfn_callback function.

*

* SEE ALSO

*	Dispatcher

*********/

/****f* Component Library: Dispatcher/cl_disp_get_queue_status

* NAME

*	cl_disp_get_queue_status

*

* DESCRIPTION

*	This function gets queue status of a Dispatcher object.

*

* SYNOPSIS

*/

void

cl_disp_get_queue_status(IN const cl_disp_reg_handle_t handle,

			 OUT uint32_t * p_num_queued_msgs,

			 OUT uint64_t * p_last_msg_queue_time_ms);

/*

* PARAMETERS

*   handle

*     [in] cl_disp_reg_handle_t value return by cl_disp_register.

*

*   p_num_queued_msgs

*     [out] number of messages in the queue

*

*   p_last_msg_queue_time_ms

*     [out] pointer to a variable to hold the time the last popped up message

*           spent in the queue

*

* RETURN VALUE

*	Thr time the last popped up message stayed in the queue, in msec

*

* NOTES

*	Extarnel Locking is not required.

*

* SEE ALSO

*	Dispatcher

*********/

END_C_DECLS

#endif				/* !defined(_CL_DISPATCHER_H_) */