|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Copyright (c) 2020 Red Hat, Inc.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is free software; you can redistribute it and/or
|
|
Packit Service |
310c69 |
* modify it under the terms of the GNU General Public License
|
|
Packit Service |
310c69 |
* as published by the Free Software Foundation; either version 2
|
|
Packit Service |
310c69 |
* of the License, or (at your option) any later version.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is distributed in the hope that it will be useful,
|
|
Packit Service |
310c69 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
310c69 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
Packit Service |
310c69 |
* GNU General Public License for more details.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* You should have received a copy of the GNU General Public License
|
|
Packit Service |
310c69 |
* along with this program; if not, write to the Free Software
|
|
Packit Service |
310c69 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
Packit Service |
310c69 |
* 02110-1301, USA.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/workQueueInternals.h#4 $
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#ifndef WORK_QUEUE_INTERNALS_H
|
|
Packit Service |
310c69 |
#define WORK_QUEUE_INTERNALS_H
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include <linux/completion.h>
|
|
Packit Service |
310c69 |
#include <linux/kobject.h>
|
|
Packit Service |
310c69 |
#include <linux/list.h>
|
|
Packit Service |
310c69 |
#include <linux/spinlock.h>
|
|
Packit Service |
310c69 |
#include <linux/wait.h>
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "workItemStats.h"
|
|
Packit Service |
310c69 |
#include "workQueueStats.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
typedef struct kvdoWorkItemList {
|
|
Packit Service |
310c69 |
KvdoWorkItem *tail;
|
|
Packit Service |
310c69 |
} KvdoWorkItemList;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Work queue definition.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* There are two types of work queues: simple, with one worker thread, and
|
|
Packit Service |
310c69 |
* round-robin, which uses a group of the former to do the work, and assigns
|
|
Packit Service |
310c69 |
* work to them in -- you guessed it -- round-robin fashion. Externally, both
|
|
Packit Service |
310c69 |
* are represented via the same common sub-structure, though there's actually
|
|
Packit Service |
310c69 |
* not a great deal of overlap between the two types internally.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
struct kvdoWorkQueue {
|
|
Packit Service |
310c69 |
/** Name of just the work queue (e.g., "cpuQ12") */
|
|
Packit Service |
310c69 |
char *name;
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Whether this is a round-robin work queue or a simple (one-thread)
|
|
Packit Service |
310c69 |
* work queue.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
bool roundRobinMode;
|
|
Packit Service |
310c69 |
/** A handle to a sysfs tree for reporting stats and other info */
|
|
Packit Service |
310c69 |
struct kobject kobj;
|
|
Packit Service |
310c69 |
/** The kernel layer owning this work queue */
|
|
Packit Service |
310c69 |
KernelLayer *owner;
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
typedef struct simpleWorkQueue SimpleWorkQueue;
|
|
Packit Service |
310c69 |
typedef struct roundRobinWorkQueue RoundRobinWorkQueue;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
struct simpleWorkQueue {
|
|
Packit Service |
310c69 |
/** Common work queue bits */
|
|
Packit Service |
310c69 |
KvdoWorkQueue common;
|
|
Packit Service |
310c69 |
/** A copy of .thread->pid, for safety in the sysfs support */
|
|
Packit Service |
310c69 |
atomic_t threadID;
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Number of priorities actually used, so we don't keep re-checking unused
|
|
Packit Service |
310c69 |
* funnel queues.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
unsigned int numPriorityLists;
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Map from action codes to priorities.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This mapping can be changed at run time in internal builds, for tuning
|
|
Packit Service |
310c69 |
* purposes.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
uint8_t priorityMap[WORK_QUEUE_ACTION_COUNT];
|
|
Packit Service |
310c69 |
/** The funnel queues */
|
|
Packit Service |
310c69 |
FunnelQueue *priorityLists[WORK_QUEUE_PRIORITY_COUNT];
|
|
Packit Service |
310c69 |
/** The kernel thread */
|
|
Packit Service |
310c69 |
struct task_struct *thread;
|
|
Packit Service |
310c69 |
/** Life cycle functions, etc */
|
|
Packit Service |
310c69 |
const KvdoWorkQueueType *type;
|
|
Packit Service |
310c69 |
/** Opaque private data pointer, defined by higher level code */
|
|
Packit Service |
310c69 |
void *private;
|
|
Packit Service |
310c69 |
/** In a subordinate work queue, a link back to the round-robin parent */
|
|
Packit Service |
310c69 |
KvdoWorkQueue *parentQueue;
|
|
Packit Service |
310c69 |
/** Padding for cache line separation */
|
|
Packit Service |
310c69 |
char pad[CACHE_LINE_BYTES - sizeof(KvdoWorkQueue *)];
|
|
Packit Service |
310c69 |
/** Lock protecting delayedItems, priorityMap, numPriorityLists, started */
|
|
Packit Service |
310c69 |
spinlock_t lock;
|
|
Packit Service |
310c69 |
/** Any worker threads (zero or one) waiting for new work to do */
|
|
Packit Service |
310c69 |
wait_queue_head_t waitingWorkerThreads;
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Hack to reduce wakeup calls if the worker thread is running. See comments
|
|
Packit Service |
310c69 |
* in workQueue.c.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* There is a lot of redundancy with "firstWakeup", though, and the pair
|
|
Packit Service |
310c69 |
* should be re-examined.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
atomic_t idle;
|
|
Packit Service |
310c69 |
/** Wait list for synchronization during worker thread startup */
|
|
Packit Service |
310c69 |
wait_queue_head_t startWaiters;
|
|
Packit Service |
310c69 |
/** Worker thread status (boolean) */
|
|
Packit Service |
310c69 |
bool started;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/** List of delayed work items; usually only one, if any */
|
|
Packit Service |
310c69 |
KvdoWorkItemList delayedItems;
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Timer for pulling delayed work items off their list and submitting them to
|
|
Packit Service |
310c69 |
* run.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* If the spinlock "lock" above is not held, this timer is scheduled (or
|
|
Packit Service |
310c69 |
* currently firing and the callback about to acquire the lock) iff
|
|
Packit Service |
310c69 |
* delayedItems is nonempty.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
struct timer_list delayedItemsTimer;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Timestamp (ns) from the submitting thread that decided to wake us up; also
|
|
Packit Service |
310c69 |
* used as a flag to indicate whether a wakeup is needed.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* Written by submitting threads with atomic64_cmpxchg, and by the worker
|
|
Packit Service |
310c69 |
* thread setting to 0.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* If the value is 0, the worker is probably asleep; the submitting thread
|
|
Packit Service |
310c69 |
* stores a non-zero value and becomes responsible for calling wake_up on the
|
|
Packit Service |
310c69 |
* worker thread. If the value is non-zero, either the worker is running or
|
|
Packit Service |
310c69 |
* another thread has the responsibility for issuing the wakeup.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* The "sleep" mode has periodic wakeups and the worker thread may happen to
|
|
Packit Service |
310c69 |
* wake up while a work item is being enqueued. If that happens, the wakeup
|
|
Packit Service |
310c69 |
* may be unneeded but will be attempted anyway.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* So the return value from cmpxchg(firstWakeup,0,nonzero) can always be
|
|
Packit Service |
310c69 |
* done, and will tell the submitting thread whether to issue the wakeup or
|
|
Packit Service |
310c69 |
* not; cmpxchg is atomic, so no other synchronization is needed.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* A timestamp is used rather than, say, 1, so that the worker thread can
|
|
Packit Service |
310c69 |
* record stats on how long it takes to actually get the worker thread
|
|
Packit Service |
310c69 |
* running.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* There is some redundancy between this and "idle" above.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
atomic64_t firstWakeup;
|
|
Packit Service |
310c69 |
/** Padding for cache line separation */
|
|
Packit Service |
310c69 |
char pad2[CACHE_LINE_BYTES - sizeof(atomic64_t)];
|
|
Packit Service |
310c69 |
/** Scheduling and work-function statistics */
|
|
Packit Service |
310c69 |
KvdoWorkQueueStats stats;
|
|
Packit Service |
310c69 |
/** Last time (ns) the scheduler actually woke us up */
|
|
Packit Service |
310c69 |
uint64_t mostRecentWakeup;
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
struct roundRobinWorkQueue {
|
|
Packit Service |
310c69 |
/** Common work queue bits */
|
|
Packit Service |
310c69 |
KvdoWorkQueue common;
|
|
Packit Service |
310c69 |
/** Simple work queues, for actually getting stuff done */
|
|
Packit Service |
310c69 |
SimpleWorkQueue **serviceQueues;
|
|
Packit Service |
310c69 |
/** Number of subordinate work queues */
|
|
Packit Service |
310c69 |
unsigned int numServiceQueues;
|
|
Packit Service |
310c69 |
/** Padding for cache line separation */
|
|
Packit Service |
310c69 |
char pad[CACHE_LINE_BYTES - sizeof(unsigned int)];
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Rotor used for dispatching across subordinate service queues.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* Used and updated by submitting threads. (Not atomically or with locking,
|
|
Packit Service |
310c69 |
* because we don't really care about it being precise, only about getting a
|
|
Packit Service |
310c69 |
* roughly even spread; if an increment is missed here and there, it's not a
|
|
Packit Service |
310c69 |
* problem.)
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
unsigned int serviceQueueRotor;
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
static inline SimpleWorkQueue *asSimpleWorkQueue(KvdoWorkQueue *queue)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ((queue == NULL)
|
|
Packit Service |
310c69 |
? NULL
|
|
Packit Service |
310c69 |
: container_of(queue, SimpleWorkQueue, common));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
static inline const SimpleWorkQueue *
|
|
Packit Service |
310c69 |
asConstSimpleWorkQueue(const KvdoWorkQueue *queue)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ((queue == NULL)
|
|
Packit Service |
310c69 |
? NULL
|
|
Packit Service |
310c69 |
: container_of(queue, SimpleWorkQueue, common));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
static inline RoundRobinWorkQueue *asRoundRobinWorkQueue(KvdoWorkQueue *queue)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ((queue == NULL)
|
|
Packit Service |
310c69 |
? NULL
|
|
Packit Service |
310c69 |
: container_of(queue, RoundRobinWorkQueue, common));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#endif // WORK_QUEUE_INTERNALS_H
|