/*
* Copyright (c) 2020 Red Hat, Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/workItemStats.h#2 $
*/
#ifndef WORK_ITEM_STATS_H
#define WORK_ITEM_STATS_H
#include "timeUtils.h"
#include "workQueue.h"
enum {
// Whether to enable tracking of per-work-function run-time stats.
ENABLE_PER_FUNCTION_TIMING_STATS = 0,
// How many work function/priority pairs to track call stats for
NUM_WORK_QUEUE_ITEM_STATS = 18,
};
typedef struct simpleStats {
uint64_t count;
uint64_t sum;
uint64_t min;
uint64_t max;
} SimpleStats;
/*
* We track numbers of work items handled (and optionally the
* wall-clock time to run the work functions), broken down by
* individual work functions (or alternate functions that the caller
* wants recorded, like the VIO completion callback function if we're
* just enqueueing a work function that invokes that indirectly) and
* priority.
*
* The first part of this structure manages the function/priority
* pairs, and is read frequently but updated rarely (once for each
* pair, plus possibly spin lock contention).
*
* The second part holds counters, and is updated often; different
* parts are updated by various threads as described below. The last
* element of each array, index NUM_WORK_QUEUE_ITEM_STATS, is updated
* only if we have filled the arrays and can't add the current work
* function/priority. See how the statTableIndex field is set in
* workItemStats.c.
*
* All fields may additionally be read when reporting statistics
* (including optionally reporting stats when the worker thread shuts
* down), but that's rare and shouldn't significantly affect cache
* contention issues.
*
* There is no "pending" count per work function here. For reporting
* statistics, it can be approximated by looking at the other fields.
* Do not rely on them being precise and synchronized, though.
*/
typedef struct kvdoWorkItemStatsFunctionTable {
/*
* The spin lock is used to protect .functions and .priorities
* during updates. All three are modified by producers (enqueueing
* threads) but only rarely. The .functions and .priorities arrays
* are read by producers very frequently.
*/
spinlock_t lock;
KvdoWorkFunction functions[NUM_WORK_QUEUE_ITEM_STATS];
uint8_t priorities[NUM_WORK_QUEUE_ITEM_STATS];
} KvdoWorkFunctionTable;
typedef struct kvdoWorkItemStats {
/*
* Table of functions and priorities, for determining the index to
* use into the counter arrays below.
*
* This table is read by producers (usually multiple entries) for
* every work item enqueued, and when reporting stats. It is updated
* by producers, and only the first time a new (work-function,
* priority) combination is seen.
*/
KvdoWorkFunctionTable functionTable;
// Skip to (somewhere on) the next cache line
char pad[CACHE_LINE_BYTES - sizeof(atomic64_t)];
/*
* The .enqueued field is updated by producers only, once per work
* item processed; __sync operations are used to update these
* values.
*/
atomic64_t enqueued[NUM_WORK_QUEUE_ITEM_STATS + 1];
// Skip to (somewhere on) the next cache line
char pad2[CACHE_LINE_BYTES - sizeof(atomic64_t)];
/*
* These values are updated only by the consumer (worker thread). We
* overload the .times[].count field as a count of items processed,
* so if we're not doing the optional processing-time tracking
* (controlled via an option in workQueue.c), we need to explicitly
* update the count.
*
* Since only one thread can ever update these values, no
* synchronization is used.
*/
SimpleStats times[NUM_WORK_QUEUE_ITEM_STATS + 1];
} KvdoWorkItemStats;
/**
* Initialize a statistics structure for tracking sample
* values. Assumes the storage was already zeroed out at allocation
* time.
*
* @param stats The statistics structure
**/
static inline void initSimpleStats(SimpleStats *stats)
{
// Assume other fields are initialized to zero at allocation.
stats->min = UINT64_MAX;
}
/**
* Update the statistics being tracked for a new sample value.
*
* @param stats The statistics structure
* @param value The new value to be folded in
**/
static inline void addSample(SimpleStats *stats, uint64_t value)
{
stats->count++;
stats->sum += value;
if (stats->min > value) {
stats->min = value;
}
if (stats->max < value) {
stats->max = value;
}
}
/**
* Return the average of the samples collected.
*
* @param stats The statistics structure
*
* @return The average sample value
**/
static inline uint64_t getSampleAverage(const SimpleStats *stats)
{
uint64_t slop = stats->count / 2;
return (stats->sum + slop) / stats->count;
}
/**
* Update all work queue statistics (work-item and otherwise) after
* enqueueing a work item.
*
* @param stats The statistics structure
* @param item The work item enqueued
* @param priority The work item's priority
**/
void updateWorkItemStatsForEnqueue(KvdoWorkItemStats *stats,
KvdoWorkItem *item,
int priority);
/**
* Update all work queue statistics (work-item and otherwise) after enqueueing
* a work item.
*
* This is a very lightweight function (after optimizing away conditionals and
* no-ops) and is called for every work item processed, hence the inline
* definition.
*
* This function requires that recordStartTime and
* updateWorkItemStatsForWorkTime below both get called as well; in some cases
* counters may be updated in updateWorkItemStatsForWorkTime rather than here.
*
* @param stats The statistics structure
* @param item The work item enqueued
**/
static inline void updateWorkItemStatsForDequeue(KvdoWorkItemStats *stats,
KvdoWorkItem *item)
{
// The times[].count field is overloaded as a count of items
// processed.
if (!ENABLE_PER_FUNCTION_TIMING_STATS) {
stats->times[item->statTableIndex].count++;
} else {
// In this case, updateWorkItemStatsForWorkTime will bump the counter.
}
}
/**
* Record the starting time for processing a work item, if timing
* stats are enabled and if we haven't run out of room for recording
* stats in the table.
*
* @param index The work item's index into the internal array
*
* @return The current time, or zero
**/
static inline uint64_t recordStartTime(unsigned int index)
{
return (ENABLE_PER_FUNCTION_TIMING_STATS ? currentTime(CLOCK_MONOTONIC) : 0);
}
/**
* Update the work queue statistics with the wall-clock time for
* processing a work item, if timing stats are enabled and if we
* haven't run out of room for recording stats in the table.
*
* @param stats The statistics structure
* @param index The work item's index into the internal array
* @param startTime The start time as reported by recordStartTime
**/
static inline void updateWorkItemStatsForWorkTime(KvdoWorkItemStats *stats,
unsigned int index,
uint64_t startTime)
{
if (ENABLE_PER_FUNCTION_TIMING_STATS) {
uint64_t endTime = currentTime(CLOCK_MONOTONIC);
addSample(&stats->times[index], endTime - startTime);
}
}
/**
* Convert the pointer into a string representation, using a function
* name if available.
*
* @param pointer The pointer to be converted
* @param buffer The output buffer
* @param bufferLength The size of the output buffer
**/
char *getFunctionName(void *pointer, char *buffer, size_t bufferLength);
/**
* Dump statistics broken down by work function and priority into the
* kernel log.
*
* @param stats The statistics structure
**/
void logWorkItemStats(const KvdoWorkItemStats *stats);
/**
* Format counters for per-work-function stats for reporting via /sys.
*
* @param [in] stats The statistics structure
* @param [out] buffer The output buffer
* @param [in] length The size of the output buffer
*
* @return The size of the string actually written
**/
size_t formatWorkItemStats(const KvdoWorkItemStats *stats,
char *buffer,
size_t length);
#endif // WORK_ITEM_STATS_H