/*
 * Copyright (c) 2020 Red Hat, Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA. 
 *
 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/workItemStats.c#4 $
 */

#include "workItemStats.h"

#include "atomic.h"
#include "logger.h"

/**
 * Scan the work queue stats table for the provided work function and
 * priority value. If it's not found, see if an empty slot is
 * available.
 *
 * @param table       The work queue's function table
 * @param work        The function we want to record stats for
 * @param priority    The priority of the work item
 *
 * @return   The index of the slot to use (matching or empty), or
 *           NUM_WORK_QUEUE_ITEM_STATS if the table is full of
 *           non-matching entries.
 **/
static inline unsigned int scanStatTable(const KvdoWorkFunctionTable *table,
                                         KvdoWorkFunction             work,
                                         unsigned int                 priority)
{
  unsigned int i;
  /*
   * See comments in getStatTableIndex regarding order of memory
   * accesses. Work function first, then a barrier, then priority.
   */
  for (i = 0; i < NUM_WORK_QUEUE_ITEM_STATS; i++) {
    if (table->functions[i] == NULL) {
      return i;
    } else if (table->functions[i] == work) {
      smp_rmb();
      if (table->priorities[i] == priority) {
        return i;
      }
    }
  }
  return NUM_WORK_QUEUE_ITEM_STATS;
}

/**
 * Scan the work queue stats table for the provided work function and
 * priority value. Assign an empty slot if necessary.
 *
 * @param stats       The stats structure
 * @param work        The function we want to record stats for
 * @param priority    The priority of the work item
 *
 * @return   The index of the matching slot, or NUM_WORK_QUEUE_ITEM_STATS
 *           if the table is full of non-matching entries.
 **/
static unsigned int getStatTableIndex(KvdoWorkItemStats *stats,
                                      KvdoWorkFunction   work,
                                      unsigned int       priority)
{
  KvdoWorkFunctionTable *functionTable = &stats->functionTable;

  unsigned int index = scanStatTable(functionTable, work, priority);
  if (unlikely(index == NUM_WORK_QUEUE_ITEM_STATS)
      || likely(functionTable->functions[index] != NULL)) {
    return index;
  }

  unsigned long flags = 0;
  // The delayed-work-item processing uses queue->lock in some cases,
  // and one case may call into this function, so we can't reuse
  // queue->lock here.
  spin_lock_irqsave(&functionTable->lock, flags);
  // Recheck now that we've got the lock...
  index = scanStatTable(functionTable, work, priority);
  if ((index == NUM_WORK_QUEUE_ITEM_STATS)
      || (functionTable->functions[index] != NULL)) {
    spin_unlock_irqrestore(&functionTable->lock, flags);
    return index;
  }

  /*
   * An uninitialized priority is indistinguishable from a zero
   * priority. So store the priority first, and enforce the ordering,
   * so that a non-null work function pointer indicates we've finished
   * filling in the value. (And, to make this work, we have to read
   * the work function first and priority second, when comparing.)
   */
  functionTable->priorities[index] = priority;
  smp_wmb();
  functionTable->functions[index] = work;
  spin_unlock_irqrestore(&functionTable->lock, flags);
  return index;
}

/**
 * Get counters on work items, identified by index into the internal
 * array.
 *
 * @param [in]  stats         The collected statistics
 * @param [in]  index         The index
 * @param [out] enqueuedPtr   The total work items enqueued
 * @param [out] processedPtr  The number of work items processed
 * @param [out] pendingPtr    The number of work items still pending
 **/
static void getWorkItemCountsByItem(const KvdoWorkItemStats *stats,
                                    unsigned int             index,
                                    uint64_t                *enqueuedPtr,
                                    uint64_t                *processedPtr,
                                    unsigned int            *pendingPtr)
{
  uint64_t enqueued  = atomic64_read(&stats->enqueued[index]);
  uint64_t processed = stats->times[index].count;
  unsigned int pending;
  if (enqueued < processed) {
    // Probably just out of sync.
    pending = 1;
  } else {
    pending = enqueued - processed;
    // Pedantic paranoia: Check for overflow of the 32-bit "pending".
    if ((pending + processed) < enqueued) {
      pending = UINT_MAX;
    }
  }
  *enqueuedPtr  = enqueued;
  *processedPtr = processed;
  *pendingPtr   = pending;
}

/**
 * Get counters on work items not covered by any index value.
 *
 * @param [in]  stats         The collected statistics
 * @param [out] enqueuedPtr   The total work items enqueued
 * @param [out] processedPtr  The number of work items processed
 **/
static void getOtherWorkItemCounts(const KvdoWorkItemStats *stats,
                                   uint64_t                *enqueuedPtr,
                                   uint64_t                *processedPtr)
{
  unsigned int pending;
  getWorkItemCountsByItem(stats, NUM_WORK_QUEUE_ITEM_STATS,
                          enqueuedPtr, processedPtr, &pending);
}

/**
 * Get timing stats on work items, identified by index into the
 * internal array.
 *
 * @param [in]  stats  The collected statistics
 * @param [in]  index  The index into the array
 * @param [out] min    The minimum execution time
 * @param [out] mean   The mean execution time
 * @param [out] max    The maximum execution time
 **/
static void getWorkItemTimesByItem(const KvdoWorkItemStats *stats,
                                   unsigned int             index,
                                   uint64_t                *min,
                                   uint64_t                *mean,
                                   uint64_t                *max)
{
  *min  = stats->times[index].min;
  *mean = getSampleAverage(&stats->times[index]);
  *max  = stats->times[index].max;
}

/**********************************************************************/
void updateWorkItemStatsForEnqueue(KvdoWorkItemStats *stats,
                                   KvdoWorkItem      *item,
                                   int                priority)
{
  item->statTableIndex = getStatTableIndex(stats, item->statsFunction,
                                           priority);
  atomic64_add(1, &stats->enqueued[item->statTableIndex]);
}

/**********************************************************************/
char *getFunctionName(void *pointer, char *buffer, size_t bufferLength)
{
  if (pointer == NULL) {
    /*
     * Format "%ps" logs a null pointer as "(null)" with a bunch of
     * leading spaces. We sometimes use this when logging lots of
     * data; don't be so verbose.
     */
    strncpy(buffer, "-", bufferLength);
  } else {
    /*
     * Use a non-const array instead of a string literal below to
     * defeat gcc's format checking, which doesn't understand that
     * "%ps" actually does support a precision spec in Linux kernel
     * code.
     */
    static char truncatedFunctionNameFormatString[] = "%.*ps";
    snprintf(buffer, bufferLength,
             truncatedFunctionNameFormatString,
             bufferLength - 1,
             pointer);

    char *space = strchr(buffer, ' ');
    if (space != NULL) {
      *space = '\0';
    }
  }

  return buffer;
}

/**********************************************************************/
size_t formatWorkItemStats(const KvdoWorkItemStats *stats,
                           char                    *buffer,
                           size_t                   length)
{
  const KvdoWorkFunctionTable *functionIDs = &stats->functionTable;
  size_t currentOffset = 0;

  uint64_t enqueued, processed;
  int i;
  for (i = 0; i < NUM_WORK_QUEUE_ITEM_STATS; i++) {
    if (functionIDs->functions[i] == NULL) {
      break;
    }
    if (atomic64_read(&stats->enqueued[i]) == 0) {
      continue;
    }
    /*
     * The reporting of all of "pending", "enqueued" and "processed"
     * here seems redundant, but "pending" is limited to 0 in the case
     * where "processed" exceeds "enqueued", either through current
     * activity and a lack of synchronization when fetching stats, or
     * a coding bug. This report is intended largely for debugging, so
     * we'll go ahead and print the not-necessarily-redundant values.
     */
    unsigned int pending;
    getWorkItemCountsByItem(stats, i, &enqueued, &processed, &pending);

    // Format: fn prio enq proc timeo [ min max mean ]
    if (ENABLE_PER_FUNCTION_TIMING_STATS) {
      uint64_t min, mean, max;
      getWorkItemTimesByItem(stats, i, &min, &mean, &max);
      currentOffset += snprintf(buffer + currentOffset,
                                length - currentOffset,
                                "%-36ps %d %10llu %10" PRIu64
                                " %10llu %10llu %10" PRIu64
                                "\n",
                                functionIDs->functions[i],
                                functionIDs->priorities[i],
                                enqueued, processed,
                                min, max, mean);
    } else {
      currentOffset += snprintf(buffer + currentOffset,
                                length - currentOffset,
                                "%-36ps %d %10llu %10" PRIu64
                                "\n",
                                functionIDs->functions[i],
                                functionIDs->priorities[i],
                                enqueued, processed);
    }
    if (currentOffset >= length) {
      break;
    }
  }
  if ((i == NUM_WORK_QUEUE_ITEM_STATS) && (currentOffset < length)) {
    uint64_t enqueued, processed;
    getOtherWorkItemCounts(stats, &enqueued, &processed);
    if (enqueued > 0) {
      currentOffset += snprintf(buffer + currentOffset,
                                length - currentOffset,
                                "%-36s %d %10llu %10" PRIu64
                                "\n",
                                "OTHER", 0,
                                enqueued, processed);
    }
  }
  return currentOffset;
}

/**********************************************************************/
void logWorkItemStats(const KvdoWorkItemStats *stats)
{
  uint64_t totalEnqueued = 0;
  uint64_t totalProcessed = 0;

  const KvdoWorkFunctionTable *functionIDs = &stats->functionTable;

  int i;
  for (i = 0; i < NUM_WORK_QUEUE_ITEM_STATS; i++) {
    if (functionIDs->functions[i] == NULL) {
      break;
    }
    if (atomic64_read(&stats->enqueued[i]) == 0) {
      continue;
    }
    /*
     * The reporting of all of "pending", "enqueued" and "processed"
     * here seems redundant, but "pending" is limited to 0 in the case
     * where "processed" exceeds "enqueued", either through current
     * activity and a lack of synchronization when fetching stats, or
     * a coding bug. This report is intended largely for debugging, so
     * we'll go ahead and print the not-necessarily-redundant values.
     */
    uint64_t enqueued, processed;
    unsigned int pending;
    getWorkItemCountsByItem(stats, i, &enqueued, &processed, &pending);
    totalEnqueued  += enqueued;
    totalProcessed += processed;

    static char work[256]; // arbitrary size
    getFunctionName(functionIDs->functions[i], work, sizeof(work));

    if (ENABLE_PER_FUNCTION_TIMING_STATS) {
      uint64_t min, mean, max;
      getWorkItemTimesByItem(stats, i, &min, &mean, &max);
      logInfo("  priority %d: %u pending"
              " %llu enqueued %llu processed"
              " %s"
              " times %llu/%llu/%lluns",
              functionIDs->priorities[i],
              pending, enqueued, processed, work,
              min, mean, max);
    } else {
      logInfo("  priority %d: %u pending"
              " %llu enqueued %llu processed"
              " %s",
              functionIDs->priorities[i],
              pending, enqueued, processed, work);
    }
  }
  if (i == NUM_WORK_QUEUE_ITEM_STATS) {
    uint64_t enqueued, processed;
    getOtherWorkItemCounts(stats, &enqueued, &processed);
    if (enqueued > 0) {
      totalEnqueued  += enqueued;
      totalProcessed += processed;
      logInfo("  ... others: %llu enqueued %llu processed",
              enqueued, processed);
    }
  }
  logInfo("  total: %llu enqueued %llu processed",
          totalEnqueued, totalProcessed);
}