/*

 * Copyright (c) 2020 Red Hat, Inc.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 * 

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 * 

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

 * 02110-1301, USA. 

 *

 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/workQueueInternals.h#4 $

 */

#ifndef WORK_QUEUE_INTERNALS_H

#define WORK_QUEUE_INTERNALS_H

#include <linux/completion.h>

#include <linux/kobject.h>

#include <linux/list.h>

#include <linux/spinlock.h>

#include <linux/wait.h>

#include "workItemStats.h"

#include "workQueueStats.h"

typedef struct kvdoWorkItemList {

  KvdoWorkItem *tail;

} KvdoWorkItemList;

/**

 * Work queue definition.

 *

 * There are two types of work queues: simple, with one worker thread, and

 * round-robin, which uses a group of the former to do the work, and assigns

 * work to them in -- you guessed it -- round-robin fashion. Externally, both

 * are represented via the same common sub-structure, though there's actually

 * not a great deal of overlap between the two types internally.

 **/

struct kvdoWorkQueue {

  /** Name of just the work queue (e.g., "cpuQ12") */

  char           *name;

  /**

   * Whether this is a round-robin work queue or a simple (one-thread)

   * work queue.

   **/

  bool            roundRobinMode;

  /** A handle to a sysfs tree for reporting stats and other info */

  struct kobject  kobj;

  /** The kernel layer owning this work queue */

  KernelLayer    *owner;

};

typedef struct simpleWorkQueue     SimpleWorkQueue;

typedef struct roundRobinWorkQueue RoundRobinWorkQueue;

struct simpleWorkQueue {

  /** Common work queue bits */

  KvdoWorkQueue            common;

  /** A copy of .thread->pid, for safety in the sysfs support */

  atomic_t                 threadID;

  /**

   * Number of priorities actually used, so we don't keep re-checking unused

   * funnel queues.

   **/

  unsigned int             numPriorityLists;

  /**

   * Map from action codes to priorities.

   *

   * This mapping can be changed at run time in internal builds, for tuning

   * purposes.

   **/

  uint8_t                  priorityMap[WORK_QUEUE_ACTION_COUNT];

  /** The funnel queues */

  FunnelQueue             *priorityLists[WORK_QUEUE_PRIORITY_COUNT];

  /** The kernel thread */

  struct task_struct      *thread;

  /** Life cycle functions, etc */

  const KvdoWorkQueueType *type;

  /** Opaque private data pointer, defined by higher level code */

  void                    *private;

  /** In a subordinate work queue, a link back to the round-robin parent */

  KvdoWorkQueue           *parentQueue;

  /** Padding for cache line separation */

  char                     pad[CACHE_LINE_BYTES - sizeof(KvdoWorkQueue *)];

  /** Lock protecting delayedItems, priorityMap, numPriorityLists, started */

  spinlock_t               lock;

  /** Any worker threads (zero or one) waiting for new work to do */

  wait_queue_head_t        waitingWorkerThreads;

  /**

   * Hack to reduce wakeup calls if the worker thread is running. See comments

   * in workQueue.c.

   *

   * There is a lot of redundancy with "firstWakeup", though, and the pair

   * should be re-examined.

   **/

  atomic_t                 idle;

  /** Wait list for synchronization during worker thread startup */

  wait_queue_head_t        startWaiters;

  /** Worker thread status (boolean) */

  bool                     started;

  /** List of delayed work items; usually only one, if any */

  KvdoWorkItemList         delayedItems;

  /**

   * Timer for pulling delayed work items off their list and submitting them to

   * run.

   *

   * If the spinlock "lock" above is not held, this timer is scheduled (or

   * currently firing and the callback about to acquire the lock) iff

   * delayedItems is nonempty.

   **/

  struct timer_list        delayedItemsTimer;

  /**

   * Timestamp (ns) from the submitting thread that decided to wake us up; also

   * used as a flag to indicate whether a wakeup is needed.

   *

   * Written by submitting threads with atomic64_cmpxchg, and by the worker

   * thread setting to 0.

   *

   * If the value is 0, the worker is probably asleep; the submitting thread

   * stores a non-zero value and becomes responsible for calling wake_up on the

   * worker thread. If the value is non-zero, either the worker is running or

   * another thread has the responsibility for issuing the wakeup.

   *

   * The "sleep" mode has periodic wakeups and the worker thread may happen to

   * wake up while a work item is being enqueued. If that happens, the wakeup

   * may be unneeded but will be attempted anyway.

   *

   * So the return value from cmpxchg(firstWakeup,0,nonzero) can always be

   * done, and will tell the submitting thread whether to issue the wakeup or

   * not; cmpxchg is atomic, so no other synchronization is needed.

   *

   * A timestamp is used rather than, say, 1, so that the worker thread can

   * record stats on how long it takes to actually get the worker thread

   * running.

   *

   * There is some redundancy between this and "idle" above.

   **/

  atomic64_t               firstWakeup;

  /** Padding for cache line separation */

  char                     pad2[CACHE_LINE_BYTES - sizeof(atomic64_t)];

  /** Scheduling and work-function statistics */

  KvdoWorkQueueStats       stats;

  /** Last time (ns) the scheduler actually woke us up */

  uint64_t                 mostRecentWakeup;

};

struct roundRobinWorkQueue {

  /** Common work queue bits */

  KvdoWorkQueue     common;

  /** Simple work queues, for actually getting stuff done */

  SimpleWorkQueue **serviceQueues;

  /** Number of subordinate work queues */

  unsigned int      numServiceQueues;

  /** Padding for cache line separation */

  char              pad[CACHE_LINE_BYTES - sizeof(unsigned int)];

  /**

   * Rotor used for dispatching across subordinate service queues.

   *

   * Used and updated by submitting threads. (Not atomically or with locking,

   * because we don't really care about it being precise, only about getting a

   * roughly even spread; if an increment is missed here and there, it's not a

   * problem.)

   **/

  unsigned int      serviceQueueRotor;

};

static inline SimpleWorkQueue *asSimpleWorkQueue(KvdoWorkQueue *queue)

{

  return ((queue == NULL)

          ? NULL

          : container_of(queue, SimpleWorkQueue, common));

}

static inline const SimpleWorkQueue *

asConstSimpleWorkQueue(const KvdoWorkQueue *queue)

{

  return ((queue == NULL)

          ? NULL

          : container_of(queue, SimpleWorkQueue, common));

}

static inline RoundRobinWorkQueue *asRoundRobinWorkQueue(KvdoWorkQueue *queue)

{

  return ((queue == NULL)

          ? NULL

          : container_of(queue, RoundRobinWorkQueue, common));

}

#endif // WORK_QUEUE_INTERNALS_H