|
Packit Service |
c5cf8c |
% Prototype Method Send State Machine
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% This state machine is targeted towards packet based systems such as VIA.
|
|
Packit Service |
c5cf8c |
% Some simplifications may be possible for stream based model like TCP.
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% Tasks for this state machine:
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - send message at the head of the queue
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - message data source exists in two forms:
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - provided by user - potentially needing to be packed but guaranteed to be
|
|
Packit Service |
c5cf8c |
% available when the CAR is enqueued
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - provided by associated recv CAR - may not be (completely) available
|
|
Packit Service |
c5cf8c |
% until later (i.e., send could stall)
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - track buffers provided by associated recv CAR
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - provide buffers to associated recv CAR
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% - enforce flow control
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
%
|
|
Packit Service |
c5cf8c |
% This version of the state machine only handles sending from a user provided
|
|
Packit Service |
c5cf8c |
% buffer. It needs to be expanded to include the forwarding cases.
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
\begin{verbatim}
|
|
Packit Service |
c5cf8c |
State [Awaiting CAR],[label="Awaiting\nCAR"]
|
|
Packit Service |
c5cf8c |
Event [CAR Available]
|
|
Packit Service |
c5cf8c |
Invoke_Action([Decide Msg Type],
|
|
Packit Service |
c5cf8c |
[label="got CAR"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Decide Msg Type]
|
|
Packit Service |
c5cf8c |
// Note: An otherwise eager message may get converted to a
|
|
Packit Service |
c5cf8c |
// rndv-RTS message if too many outstanding eager messages have already
|
|
Packit Service |
c5cf8c |
// been sent to the receiving process.
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if the type is eager
|
|
Packit Service |
c5cf8c |
// TODO: we may want to convert it to rndv (rts and eventually data)
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Buffers for Ctrl])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Check Buffers for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="Check Buffers\nfor Ctrl Pkt"]
|
|
Packit Service |
c5cf8c |
// get buffers if we don't already have some
|
|
Packit Service |
c5cf8c |
// always need a buffer for control information
|
|
Packit Service |
c5cf8c |
if buffers are not available
|
|
Packit Service |
c5cf8c |
Change_State([Awaiting Buffers for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="no bufs"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="bufs ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
State [Awaiting Buffers for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="Awaiting Buffers\nfor Ctrl Pkt"]
|
|
Packit Service |
c5cf8c |
Event [Buffers Available]
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="bufs ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Check Flow for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="Check Flow Control\nfor Ctrl Pkt"]
|
|
Packit Service |
c5cf8c |
if flow restricted // all tensed up
|
|
Packit Service |
c5cf8c |
Change_State([Awaiting Flow for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="flow\nrestricted"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Post Send Ctrl],
|
|
Packit Service |
c5cf8c |
[label="flow ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
State [Awaiting Flow for Ctrl],
|
|
Packit Service |
c5cf8c |
[label="Awaiting Flow\nfor Ctrl Pkt"]
|
|
Packit Service |
c5cf8c |
Event [Flow Enabled]
|
|
Packit Service |
c5cf8c |
Invoke_Action([Post Send Ctrl],
|
|
Packit Service |
c5cf8c |
[label="flow ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Post Send Ctrl],
|
|
Packit Service |
c5cf8c |
[label="Post Send\nfor Ctrl Pkt"]
|
|
Packit Service |
c5cf8c |
type = OD.type
|
|
Packit Service |
c5cf8c |
// pack meta-data into the packet buffer
|
|
Packit Service |
c5cf8c |
// meta-data may include the envelope information (context, src rank,
|
|
Packit Service |
c5cf8c |
// and tag) as well as the ready send bit
|
|
Packit Service |
c5cf8c |
put type in packet buffer
|
|
Packit Service |
c5cf8c |
if type is short or eager
|
|
Packit Service |
c5cf8c |
put the envelope info and rsend bit in the packet buffer
|
|
Packit Service |
c5cf8c |
else if type is rndv-rts message
|
|
Packit Service |
c5cf8c |
put the envelope and LIBA(OD) into the packet buffer
|
|
Packit Service |
c5cf8c |
OD.type = rndv-data
|
|
Packit Service |
c5cf8c |
else if type is rndv-cts
|
|
Packit Service |
c5cf8c |
put the two LIBAs in the packet buffer
|
|
Packit Service |
c5cf8c |
else if type is rndv-data
|
|
Packit Service |
c5cf8c |
put the LIBA in the packet buffer
|
|
Packit Service |
c5cf8c |
else if type is flow control
|
|
Packit Service |
c5cf8c |
// TODO
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
// pack data from DD into packet buffer (as much as will fit) if we have
|
|
Packit Service |
c5cf8c |
// any data to send
|
|
Packit Service |
c5cf8c |
if type is short, eager, or rndv-data and a DD exists
|
|
Packit Service |
c5cf8c |
if direct access flag is set
|
|
Packit Service |
c5cf8c |
point network send descriptor at the user data (which is pinned)
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
pack data into the packet buffer (from the data descriptor)
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
post the send to the network device
|
|
Packit Service |
c5cf8c |
if there are no data packets to send (i.e., short)
|
|
Packit Service |
c5cf8c |
Invoke_Action([Signal CAR Completion],
|
|
Packit Service |
c5cf8c |
[label="msg complete"])
|
|
Packit Service |
c5cf8c |
else if direct access flag is set
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="more data,\ndirect send"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Buffers for Data],
|
|
Packit Service |
c5cf8c |
[label="more data,\nbuffered send"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Check Buffers for Data],
|
|
Packit Service |
c5cf8c |
[label="Check Buffers\nfor Data Pkts"]
|
|
Packit Service |
c5cf8c |
// try to get buffers if we need them
|
|
Packit Service |
c5cf8c |
if buffers aren't available
|
|
Packit Service |
c5cf8c |
Change_State([Awaiting Buffers for Data],
|
|
Packit Service |
c5cf8c |
[label="no bufs"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="bufs ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
State [Awaiting Buffers for Data],
|
|
Packit Service |
c5cf8c |
[label="Awaiting Buffers\nfor Data Pkts"]
|
|
Packit Service |
c5cf8c |
Event [Buffers Available]
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="bufs ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Check Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="Check Flow Control\nfor Data Pkts"]
|
|
Packit Service |
c5cf8c |
if flow restricted
|
|
Packit Service |
c5cf8c |
Change_State([Awaiting Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="flow\nrestricted"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Post Send Data],
|
|
Packit Service |
c5cf8c |
[label="flow ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
State [Awaiting Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="Awaiting Flow Control\nfor Data Pkts"]
|
|
Packit Service |
c5cf8c |
Event [Flow Enabled]
|
|
Packit Service |
c5cf8c |
Invoke_Action([Post Send Data],
|
|
Packit Service |
c5cf8c |
[label="flow ok"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
// TODO: Add buffer packing into flow
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Post Send Data],
|
|
Packit Service |
c5cf8c |
[label="Post Send\nfor Data Pkts"]
|
|
Packit Service |
c5cf8c |
if direct access flag is set
|
|
Packit Service |
c5cf8c |
point network send descriptor at the user data (which is pinned)
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
pack data into the packet buffer (from the data descriptor)
|
|
Packit Service |
c5cf8c |
post the send to the network device
|
|
Packit Service |
c5cf8c |
// Note: In the implementation, we may want to post multiple packets sends
|
|
Packit Service |
c5cf8c |
// at one time, amortizing the cost of checking for buffers and checking
|
|
Packit Service |
c5cf8c |
// flow control over a set of outgoing packets. We are uncertain how
|
|
Packit Service |
c5cf8c |
// packing multiple buffers then posting multiple sends will affect
|
|
Packit Service |
c5cf8c |
// peformance, but clearly amortizing costs when sending straight out of
|
|
Packit Service |
c5cf8c |
// the user buffer will be a win.
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if there are more data packets to send
|
|
Packit Service |
c5cf8c |
if we are sending directly from the user buffer
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Flow for Data],
|
|
Packit Service |
c5cf8c |
[label="more data,\ndirect send"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Buffers for Data],
|
|
Packit Service |
c5cf8c |
[label="more data,\nbuffered send"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Invoke_Action([Signal CAR Completion],
|
|
Packit Service |
c5cf8c |
[label="msg complete"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Signal CAR Completion], [label="Check/Signal\nCAR Completion"]
|
|
Packit Service |
c5cf8c |
if we did not send all data directly from the user buffer
|
|
Packit Service |
c5cf8c |
reset (local) completion flag
|
|
Packit Service |
c5cf8c |
if we send some of the data directly from the user buffer
|
|
Packit Service |
c5cf8c |
lock CAR.mutex
|
|
Packit Service |
c5cf8c |
decrement CAR outstanding activities counter
|
|
Packit Service |
c5cf8c |
if outstanding activity counter reaches zero
|
|
Packit Service |
c5cf8c |
set completion flag
|
|
Packit Service |
c5cf8c |
unlock CAR.mutex
|
|
Packit Service |
c5cf8c |
else // all data sent using packet buffers
|
|
Packit Service |
c5cf8c |
set completion flag
|
|
Packit Service |
c5cf8c |
if completion flag is set
|
|
Packit Service |
c5cf8c |
decrement completion counter associated with CAR
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Invoke_Action([Check Ready CAR])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Action [Check Ready CAR],[label="Check for Ready\nSend CAR"]
|
|
Packit Service |
c5cf8c |
if more CARs are on the send queue
|
|
Packit Service |
c5cf8c |
Invoke_Action([Decide Msg Type],
|
|
Packit Service |
c5cf8c |
[label="ready CAR"])
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
Change_State([Awaiting CAR],
|
|
Packit Service |
c5cf8c |
[label="no more\nCARs"])
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
\end{verbatim}
|