/*
 * Copyright (c) 2020 Red Hat, Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA. 
 *
 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/workItemStats.h#2 $
 */

#ifndef WORK_ITEM_STATS_H
#define WORK_ITEM_STATS_H

#include "timeUtils.h"

#include "workQueue.h"

enum {
  // Whether to enable tracking of per-work-function run-time stats.
  ENABLE_PER_FUNCTION_TIMING_STATS = 0,
  // How many work function/priority pairs to track call stats for
  NUM_WORK_QUEUE_ITEM_STATS        = 18,
};

typedef struct simpleStats {
  uint64_t count;
  uint64_t sum;
  uint64_t min;
  uint64_t max;
} SimpleStats;

/*
 * We track numbers of work items handled (and optionally the
 * wall-clock time to run the work functions), broken down by
 * individual work functions (or alternate functions that the caller
 * wants recorded, like the VIO completion callback function if we're
 * just enqueueing a work function that invokes that indirectly) and
 * priority.
 *
 * The first part of this structure manages the function/priority
 * pairs, and is read frequently but updated rarely (once for each
 * pair, plus possibly spin lock contention).
 *
 * The second part holds counters, and is updated often; different
 * parts are updated by various threads as described below. The last
 * element of each array, index NUM_WORK_QUEUE_ITEM_STATS, is updated
 * only if we have filled the arrays and can't add the current work
 * function/priority. See how the statTableIndex field is set in
 * workItemStats.c.
 *
 * All fields may additionally be read when reporting statistics
 * (including optionally reporting stats when the worker thread shuts
 * down), but that's rare and shouldn't significantly affect cache
 * contention issues.
 *
 * There is no "pending" count per work function here. For reporting
 * statistics, it can be approximated by looking at the other fields.
 * Do not rely on them being precise and synchronized, though.
 */
typedef struct kvdoWorkItemStatsFunctionTable {
  /*
   * The spin lock is used to protect .functions and .priorities
   * during updates. All three are modified by producers (enqueueing
   * threads) but only rarely. The .functions and .priorities arrays
   * are read by producers very frequently.
   */
  spinlock_t       lock;
  KvdoWorkFunction functions[NUM_WORK_QUEUE_ITEM_STATS];
  uint8_t          priorities[NUM_WORK_QUEUE_ITEM_STATS];
} KvdoWorkFunctionTable;

typedef struct kvdoWorkItemStats {
  /*
   * Table of functions and priorities, for determining the index to
   * use into the counter arrays below.
   *
   * This table is read by producers (usually multiple entries) for
   * every work item enqueued, and when reporting stats. It is updated
   * by producers, and only the first time a new (work-function,
   * priority) combination is seen.
   */
  KvdoWorkFunctionTable  functionTable;
  // Skip to (somewhere on) the next cache line
  char                   pad[CACHE_LINE_BYTES - sizeof(atomic64_t)];
  /*
   * The .enqueued field is updated by producers only, once per work
   * item processed; __sync operations are used to update these
   * values.
   */
  atomic64_t             enqueued[NUM_WORK_QUEUE_ITEM_STATS + 1];
  // Skip to (somewhere on) the next cache line
  char                   pad2[CACHE_LINE_BYTES - sizeof(atomic64_t)];
  /*
   * These values are updated only by the consumer (worker thread). We
   * overload the .times[].count field as a count of items processed,
   * so if we're not doing the optional processing-time tracking
   * (controlled via an option in workQueue.c), we need to explicitly
   * update the count.
   *
   * Since only one thread can ever update these values, no
   * synchronization is used.
   */
  SimpleStats            times[NUM_WORK_QUEUE_ITEM_STATS + 1];
} KvdoWorkItemStats;

/**
 * Initialize a statistics structure for tracking sample
 * values. Assumes the storage was already zeroed out at allocation
 * time.
 *
 * @param stats    The statistics structure
 **/
static inline void initSimpleStats(SimpleStats *stats)
{
  // Assume other fields are initialized to zero at allocation.
  stats->min = UINT64_MAX;
}

/**
 * Update the statistics being tracked for a new sample value.
 *
 * @param stats    The statistics structure
 * @param value    The new value to be folded in
 **/
static inline void addSample(SimpleStats *stats, uint64_t value)
{
  stats->count++;
  stats->sum += value;
  if (stats->min > value) {
    stats->min = value;
  }
  if (stats->max < value) {
    stats->max = value;
  }
}

/**
 * Return the average of the samples collected.
 *
 * @param stats    The statistics structure
 *
 * @return         The average sample value
 **/
static inline uint64_t getSampleAverage(const SimpleStats *stats)
{
  uint64_t slop = stats->count / 2;
  return (stats->sum + slop) / stats->count;
}

/**
 * Update all work queue statistics (work-item and otherwise) after
 * enqueueing a work item.
 *
 * @param  stats     The statistics structure
 * @param  item      The work item enqueued
 * @param  priority  The work item's priority
 **/
void updateWorkItemStatsForEnqueue(KvdoWorkItemStats *stats,
                                   KvdoWorkItem      *item,
                                   int                priority);

/**
 * Update all work queue statistics (work-item and otherwise) after enqueueing
 * a work item.
 *
 * This is a very lightweight function (after optimizing away conditionals and
 * no-ops) and is called for every work item processed, hence the inline
 * definition.
 *
 * This function requires that recordStartTime and
 * updateWorkItemStatsForWorkTime below both get called as well; in some cases
 * counters may be updated in updateWorkItemStatsForWorkTime rather than here.
 *
 * @param  stats  The statistics structure
 * @param  item   The work item enqueued
 **/
static inline void updateWorkItemStatsForDequeue(KvdoWorkItemStats *stats,
                                                 KvdoWorkItem      *item)
{
  // The times[].count field is overloaded as a count of items
  // processed.
  if (!ENABLE_PER_FUNCTION_TIMING_STATS) {
    stats->times[item->statTableIndex].count++;
  } else {
    // In this case, updateWorkItemStatsForWorkTime will bump the counter.
  }
}

/**
 * Record the starting time for processing a work item, if timing
 * stats are enabled and if we haven't run out of room for recording
 * stats in the table.
 *
 * @param  index  The work item's index into the internal array
 *
 * @return    The current time, or zero
 **/
static inline uint64_t recordStartTime(unsigned int index)
{
  return (ENABLE_PER_FUNCTION_TIMING_STATS ? currentTime(CLOCK_MONOTONIC) : 0);
}

/**
 * Update the work queue statistics with the wall-clock time for
 * processing a work item, if timing stats are enabled and if we
 * haven't run out of room for recording stats in the table.
 *
 * @param  stats      The statistics structure
 * @param  index      The work item's index into the internal array
 * @param  startTime  The start time as reported by recordStartTime
 **/
static inline void updateWorkItemStatsForWorkTime(KvdoWorkItemStats *stats,
                                                  unsigned int       index,
                                                  uint64_t           startTime)
{
  if (ENABLE_PER_FUNCTION_TIMING_STATS) {
    uint64_t endTime = currentTime(CLOCK_MONOTONIC);
    addSample(&stats->times[index], endTime - startTime);
  }
}

/**
 * Convert the pointer into a string representation, using a function
 * name if available.
 *
 * @param pointer       The pointer to be converted
 * @param buffer        The output buffer
 * @param bufferLength  The size of the output buffer
 **/
char *getFunctionName(void *pointer, char *buffer, size_t bufferLength);

/**
 * Dump statistics broken down by work function and priority into the
 * kernel log.
 *
 * @param  stats  The statistics structure
 **/
void logWorkItemStats(const KvdoWorkItemStats *stats);

/**
 * Format counters for per-work-function stats for reporting via /sys.
 *
 * @param [in]  stats   The statistics structure
 * @param [out] buffer  The output buffer
 * @param [in]  length  The size of the output buffer
 *
 * @return  The size of the string actually written
 **/
size_t formatWorkItemStats(const KvdoWorkItemStats *stats,
                           char                    *buffer,
                           size_t                   length);

#endif // WORK_ITEM_STATS_H