|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Copyright (c) 2020 Red Hat, Inc.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is free software; you can redistribute it and/or
|
|
Packit Service |
310c69 |
* modify it under the terms of the GNU General Public License
|
|
Packit Service |
310c69 |
* as published by the Free Software Foundation; either version 2
|
|
Packit Service |
310c69 |
* of the License, or (at your option) any later version.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is distributed in the hope that it will be useful,
|
|
Packit Service |
310c69 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
310c69 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
Packit Service |
310c69 |
* GNU General Public License for more details.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* You should have received a copy of the GNU General Public License
|
|
Packit Service |
310c69 |
* along with this program; if not, write to the Free Software
|
|
Packit Service |
310c69 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
Packit Service |
310c69 |
* 02110-1301, USA.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/base/vdoPageCache.c#11 $
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "vdoPageCacheInternals.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#if __KERNEL__
|
|
Packit Service |
310c69 |
#include <linux/ratelimit.h>
|
|
Packit Service |
310c69 |
#endif
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "errors.h"
|
|
Packit Service |
310c69 |
#include "logger.h"
|
|
Packit Service |
310c69 |
#include "memoryAlloc.h"
|
|
Packit Service |
310c69 |
#include "permassert.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "adminState.h"
|
|
Packit Service |
310c69 |
#include "constants.h"
|
|
Packit Service |
310c69 |
#include "numUtils.h"
|
|
Packit Service |
310c69 |
#include "readOnlyNotifier.h"
|
|
Packit Service |
310c69 |
#include "statusCodes.h"
|
|
Packit Service |
310c69 |
#include "types.h"
|
|
Packit Service |
310c69 |
#include "vio.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
enum {
|
|
Packit Service |
310c69 |
LOG_INTERVAL = 4000,
|
|
Packit Service |
310c69 |
DISPLAY_INTERVAL = 100000,
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static char *getPageBuffer(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE];
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Allocate components of the cache which require their own allocation. The
|
|
Packit Service |
310c69 |
* caller is responsible for all clean up on errors.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache The cache being constructed
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return VDO_SUCCESS or an error code
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static int allocateCacheComponents(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = ALLOCATE(cache->pageCount, PageInfo, "page infos",
|
|
Packit Service |
310c69 |
&cache->infos);
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
uint64_t size = cache->pageCount * (uint64_t) VDO_BLOCK_SIZE;
|
|
Packit Service |
310c69 |
result = allocateMemory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages);
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
return makeIntMap(cache->pageCount, 0, &cache->pageMap);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Initialize all page info structures and put them on the free list.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache The cache to initialize
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return VDO_SUCCESS or an error
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static int initializeInfo(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
initializeRing(&cache->freeList);
|
|
Packit Service |
310c69 |
PageInfo *info;
|
|
Packit Service |
310c69 |
for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
|
|
Packit Service |
310c69 |
info->cache = cache;
|
|
Packit Service |
310c69 |
info->state = PS_FREE;
|
|
Packit Service |
310c69 |
info->pbn = NO_PAGE;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (cache->layer->createMetadataVIO != NULL) {
|
|
Packit Service |
310c69 |
int result = createVIO(cache->layer, VIO_TYPE_BLOCK_MAP,
|
|
Packit Service |
310c69 |
VIO_PRIORITY_METADATA, info, getPageBuffer(info),
|
|
Packit Service |
310c69 |
&info->vio);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// The thread ID should never change.
|
|
Packit Service |
310c69 |
info->vio->completion.callbackThreadID = cache->zone->threadID;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
initializeRing(&info->listNode);
|
|
Packit Service |
310c69 |
pushRingNode(&cache->freeList, &info->listNode);
|
|
Packit Service |
310c69 |
initializeRing(&info->lruNode);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
relaxedStore64(&cache->stats.counts.freePages, cache->pageCount);
|
|
Packit Service |
310c69 |
return VDO_SUCCESS;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static void writeDirtyPagesCallback(RingNode *node, void *context);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
int makeVDOPageCache(PhysicalLayer *layer,
|
|
Packit Service |
310c69 |
PageCount pageCount,
|
|
Packit Service |
310c69 |
VDOPageReadFunction *readHook,
|
|
Packit Service |
310c69 |
VDOPageWriteFunction *writeHook,
|
|
Packit Service |
310c69 |
size_t pageContextSize,
|
|
Packit Service |
310c69 |
BlockCount maximumAge,
|
|
Packit Service |
310c69 |
BlockMapZone *zone,
|
|
Packit Service |
310c69 |
VDOPageCache **cachePtr)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = ASSERT(pageContextSize <= MAX_PAGE_CONTEXT_SIZE,
|
|
Packit Service |
310c69 |
"page context size %zu cannot exceed %u bytes",
|
|
Packit Service |
310c69 |
pageContextSize, MAX_PAGE_CONTEXT_SIZE);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
VDOPageCache *cache;
|
|
Packit Service |
310c69 |
result = ALLOCATE(1, VDOPageCache, "page cache", &cache);
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
cache->layer = layer;
|
|
Packit Service |
310c69 |
cache->pageCount = pageCount;
|
|
Packit Service |
310c69 |
cache->readHook = readHook;
|
|
Packit Service |
310c69 |
cache->writeHook = writeHook;
|
|
Packit Service |
310c69 |
cache->zone = zone;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = allocateCacheComponents(cache);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
freeVDOPageCache(&cache);
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = initializeInfo(cache);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
freeVDOPageCache(&cache);
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = makeDirtyLists(maximumAge, writeDirtyPagesCallback, cache,
|
|
Packit Service |
310c69 |
&cache->dirtyLists);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
freeVDOPageCache(&cache);
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// initialize empty circular queues
|
|
Packit Service |
310c69 |
initializeRing(&cache->lruList);
|
|
Packit Service |
310c69 |
initializeRing(&cache->outgoingList);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
*cachePtr = cache;
|
|
Packit Service |
310c69 |
return VDO_SUCCESS;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void freeVDOPageCache(VDOPageCache **cachePtr)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = *cachePtr;
|
|
Packit Service |
310c69 |
if (cache == NULL) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (cache->infos != NULL) {
|
|
Packit Service |
310c69 |
PageInfo *info;
|
|
Packit Service |
310c69 |
for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
|
|
Packit Service |
310c69 |
freeVIO(&info->vio);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
freeDirtyLists(&cache->dirtyLists);
|
|
Packit Service |
310c69 |
freeIntMap(&cache->pageMap);
|
|
Packit Service |
310c69 |
FREE(cache->infos);
|
|
Packit Service |
310c69 |
FREE(cache->pages);
|
|
Packit Service |
310c69 |
FREE(cache);
|
|
Packit Service |
310c69 |
*cachePtr = NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void setVDOPageCacheInitialPeriod(VDOPageCache *cache, SequenceNumber period)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
setCurrentPeriod(cache->dirtyLists, period);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void setVDOPageCacheRebuildMode(VDOPageCache *cache, bool rebuilding)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
cache->rebuilding = rebuilding;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Assert that a function has been called on the VDO page cache's thread.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
* @param functionName the name of the function
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static inline void assertOnCacheThread(VDOPageCache *cache,
|
|
Packit Service |
310c69 |
const char *functionName)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ThreadID threadID = getCallbackThreadID();
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY((threadID == cache->zone->threadID),
|
|
Packit Service |
310c69 |
"%s() must only be called on cache thread %d, not thread %d",
|
|
Packit Service |
310c69 |
functionName, cache->zone->threadID, threadID);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Assert that a page cache may issue I/O.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static inline void assertIOAllowed(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(!isQuiescent(&cache->zone->state),
|
|
Packit Service |
310c69 |
"VDO page cache may issue I/O");
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Log and, if enabled, report cache pressure.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void reportCachePressure(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.cachePressure, 1);
|
|
Packit Service |
310c69 |
if (cache->waiterCount > cache->pageCount) {
|
|
Packit Service |
310c69 |
if ((cache->pressureReport % LOG_INTERVAL) == 0) {
|
|
Packit Service |
310c69 |
logInfo("page cache pressure %llu",
|
|
Packit Service |
310c69 |
relaxedLoad64(&cache->stats.cachePressure));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (++cache->pressureReport >= DISPLAY_INTERVAL) {
|
|
Packit Service |
310c69 |
cache->pressureReport = 0;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
const char *vpcPageStateName(PageState state)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
static const char *stateNames[] = {
|
|
Packit Service |
310c69 |
"FREE",
|
|
Packit Service |
310c69 |
"INCOMING",
|
|
Packit Service |
310c69 |
"FAILED",
|
|
Packit Service |
310c69 |
"RESIDENT",
|
|
Packit Service |
310c69 |
"DIRTY",
|
|
Packit Service |
310c69 |
"OUTGOING"
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
STATIC_ASSERT(COUNT_OF(stateNames) == PAGE_STATE_COUNT);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = ASSERT(state < COUNT_OF(stateNames),
|
|
Packit Service |
310c69 |
"Unknown PageState value %d", state);
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return "[UNKNOWN PAGE STATE]";
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
return stateNames[state];
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Update the counter associated with a given state.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info the page info to count
|
|
Packit Service |
310c69 |
* @param delta the delta to apply to the counter
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void updateCounter(PageInfo *info, int32_t delta)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
switch (info->state) {
|
|
Packit Service |
310c69 |
case PS_FREE:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.freePages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_INCOMING:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.incomingPages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_OUTGOING:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.outgoingPages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_FAILED:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.failedPages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_RESIDENT:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.cleanPages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_DIRTY:
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.counts.dirtyPages, delta);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
default:
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Update the lru information for an active page.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void updateLru(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (cache->lruList.prev != &info->lruNode) {
|
|
Packit Service |
310c69 |
pushRingNode(&cache->lruList, &info->lruNode);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Set the state of a PageInfo and put it on the right list, adjusting
|
|
Packit Service |
310c69 |
* counters.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info the PageInfo to modify
|
|
Packit Service |
310c69 |
* @param newState the new state for the PageInfo
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void setInfoState(PageInfo *info, PageState newState)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (newState == info->state) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
updateCounter(info, -1);
|
|
Packit Service |
310c69 |
info->state = newState;
|
|
Packit Service |
310c69 |
updateCounter(info, 1);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
switch (info->state) {
|
|
Packit Service |
310c69 |
case PS_FREE:
|
|
Packit Service |
310c69 |
case PS_FAILED:
|
|
Packit Service |
310c69 |
pushRingNode(&info->cache->freeList, &info->listNode);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_OUTGOING:
|
|
Packit Service |
310c69 |
pushRingNode(&info->cache->outgoingList, &info->listNode);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case PS_DIRTY:
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
default:
|
|
Packit Service |
310c69 |
unspliceRingNode(&info->listNode);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Set the pbn for an info, updating the map as needed.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info The page info
|
|
Packit Service |
310c69 |
* @param pbn The physical block number to set
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static int setInfoPBN(PageInfo *info, PhysicalBlockNumber pbn)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Either the new or the old page number must be NO_PAGE.
|
|
Packit Service |
310c69 |
int result = ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE),
|
|
Packit Service |
310c69 |
"Must free a page before reusing it.");
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (info->pbn != NO_PAGE) {
|
|
Packit Service |
310c69 |
intMapRemove(cache->pageMap, info->pbn);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
info->pbn = pbn;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (pbn != NO_PAGE) {
|
|
Packit Service |
310c69 |
result = intMapPut(cache->pageMap, pbn, info, true, NULL);
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
return VDO_SUCCESS;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Reset page info to represent an unallocated page.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static int resetPageInfo(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = ASSERT(info->busy == 0, "VDO Page must not be busy");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = ASSERT(!hasWaiters(&info->waiting),
|
|
Packit Service |
310c69 |
"VDO Page must not have waiters");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = setInfoPBN(info, NO_PAGE);
|
|
Packit Service |
310c69 |
setInfoState(info, PS_FREE);
|
|
Packit Service |
310c69 |
unspliceRingNode(&info->lruNode);
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Find a free page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return a pointer to the page info structure (if found), NULL otherwise
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static PageInfo *findFreePage(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (cache->freeList.next == &cache->freeList) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
PageInfo *info = pageInfoFromListNode(cache->freeList.next);
|
|
Packit Service |
310c69 |
unspliceRingNode(&info->listNode);
|
|
Packit Service |
310c69 |
return info;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
PageInfo *vpcFindPage(VDOPageCache *cache, PhysicalBlockNumber pbn)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if ((cache->lastFound != NULL)
|
|
Packit Service |
310c69 |
&& (cache->lastFound->pbn == pbn)) {
|
|
Packit Service |
310c69 |
return cache->lastFound;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
cache->lastFound = intMapGet(cache->pageMap, pbn);
|
|
Packit Service |
310c69 |
return cache->lastFound;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Determine which page is least recently used.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache structure
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return a pointer to the info structure for a relevant page,
|
|
Packit Service |
310c69 |
* or NULL if no such page can be found. The page can be
|
|
Packit Service |
310c69 |
* dirty or resident.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @note Picks the least recently used from among the non-busy entries
|
|
Packit Service |
310c69 |
* at the front of each of the lru ring.
|
|
Packit Service |
310c69 |
* Since whenever we mark a page busy we also put it to the end
|
|
Packit Service |
310c69 |
* of the ring it is unlikely that the entries at the front
|
|
Packit Service |
310c69 |
* are busy unless the queue is very short, but not impossible.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static PageInfo *selectLRUPage(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfoNode *lru;
|
|
Packit Service |
310c69 |
for (lru = cache->lruList.next;
|
|
Packit Service |
310c69 |
lru != &cache->lruList;
|
|
Packit Service |
310c69 |
lru = lru->next) {
|
|
Packit Service |
310c69 |
PageInfo *info = pageInfoFromLRUNode(lru);
|
|
Packit Service |
310c69 |
if ((info->busy == 0) && !isInFlight(info)) {
|
|
Packit Service |
310c69 |
return info;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
AtomicPageCacheStatistics *getVDOPageCacheStatistics(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return &cache->stats;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// ASYNCHRONOUS INTERFACE BEYOND THIS POINT
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Helper to complete the VDO Page Completion request successfully.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info the page info representing the result page
|
|
Packit Service |
310c69 |
* @param vdoPageComp the VDO page completion to complete
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void completeWithPage(PageInfo *info, VDOPageCompletion *vdoPageComp)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
bool available = vdoPageComp->writable ? isPresent(info) : isValid(info);
|
|
Packit Service |
310c69 |
if (!available) {
|
|
Packit Service |
310c69 |
logErrorWithStringError(VDO_BAD_PAGE,
|
|
Packit Service |
310c69 |
"Requested cache page %llu in state %s is"
|
|
Packit Service |
310c69 |
" not %s",
|
|
Packit Service |
310c69 |
info->pbn, vpcPageStateName(info->state),
|
|
Packit Service |
310c69 |
vdoPageComp->writable ? "present" : "valid");
|
|
Packit Service |
310c69 |
finishCompletion(&vdoPageComp->completion, VDO_BAD_PAGE);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
vdoPageComp->info = info;
|
|
Packit Service |
310c69 |
vdoPageComp->ready = true;
|
|
Packit Service |
310c69 |
finishCompletion(&vdoPageComp->completion, VDO_SUCCESS);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Complete a page completion with an error code. Implements WaiterCallback.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param waiter The page completion, as a waiter
|
|
Packit Service |
310c69 |
* @param resultPtr A pointer to the error code.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void completeWaiterWithError(Waiter *waiter, void *resultPtr)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int *result = resultPtr;
|
|
Packit Service |
310c69 |
VDOPageCompletion *completion = pageCompletionFromWaiter(waiter);
|
|
Packit Service |
310c69 |
finishCompletion(&completion->completion, *result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Complete a queue of VDOPageCompletions with an error code.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param [in] result the error result
|
|
Packit Service |
310c69 |
* @param [in, out] queue a pointer to the queue
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @note upon completion the queue will be empty
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void distributeErrorOverQueue(int result, WaitQueue *queue)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
notifyAllWaiters(queue, completeWaiterWithError, &result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Complete a page completion with a page. Implements WaiterCallback.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param waiter The page completion, as a waiter
|
|
Packit Service |
310c69 |
* @param pageInfo The page info to complete with
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void completeWaiterWithPage(Waiter *waiter, void *pageInfo)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = pageInfo;
|
|
Packit Service |
310c69 |
VDOPageCompletion *completion = pageCompletionFromWaiter(waiter);
|
|
Packit Service |
310c69 |
completeWithPage(info, completion);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Complete a queue of VDOPageCompletions with a page result.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param [in] info the page info describing the page
|
|
Packit Service |
310c69 |
* @param [in, out] queue a pointer to a queue of waiters
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return the number of pages distributed
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @note upon completion the queue will be empty
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static unsigned int distributePageOverQueue(PageInfo *info, WaitQueue *queue)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
updateLru(info);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
size_t pages = countWaiters(queue);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Increment the busy count once for each pending completion so that
|
|
Packit Service |
310c69 |
* this page does not stop being busy until all completions have
|
|
Packit Service |
310c69 |
* been processed (VDO-83).
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
info->busy += pages;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
notifyAllWaiters(queue, completeWaiterWithPage, info);
|
|
Packit Service |
310c69 |
return pages;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Set a persistent error which all requests will receive in the future.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
* @param context a string describing what triggered the error
|
|
Packit Service |
310c69 |
* @param result the error result
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* Once triggered, all enqueued completions will get this error.
|
|
Packit Service |
310c69 |
* Any future requests will result in this error as well.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void setPersistentError(VDOPageCache *cache,
|
|
Packit Service |
310c69 |
const char *context,
|
|
Packit Service |
310c69 |
int result)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
// If we're already read-only, there's no need to log.
|
|
Packit Service |
310c69 |
ReadOnlyNotifier *notifier = cache->zone->readOnlyNotifier;
|
|
Packit Service |
310c69 |
if ((result != VDO_READ_ONLY) && !isReadOnly(notifier)) {
|
|
Packit Service |
310c69 |
logErrorWithStringError(result, "VDO Page Cache persistent error: %s",
|
|
Packit Service |
310c69 |
context);
|
|
Packit Service |
310c69 |
enterReadOnlyMode(notifier, result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
distributeErrorOverQueue(result, &cache->freeWaiters);
|
|
Packit Service |
310c69 |
cache->waiterCount = 0;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *info;
|
|
Packit Service |
310c69 |
for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
|
|
Packit Service |
310c69 |
distributeErrorOverQueue(result, &info->waiting);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void initVDOPageCompletion(VDOPageCompletion *pageCompletion,
|
|
Packit Service |
310c69 |
VDOPageCache *cache,
|
|
Packit Service |
310c69 |
PhysicalBlockNumber pbn,
|
|
Packit Service |
310c69 |
bool writable,
|
|
Packit Service |
310c69 |
void *parent,
|
|
Packit Service |
310c69 |
VDOAction *callback,
|
|
Packit Service |
310c69 |
VDOAction *errorHandler)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY((pageCompletion->waiter.nextWaiter == NULL),
|
|
Packit Service |
310c69 |
"New page completion was not already on a wait queue");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
*pageCompletion = (VDOPageCompletion) {
|
|
Packit Service |
310c69 |
.pbn = pbn,
|
|
Packit Service |
310c69 |
.writable = writable,
|
|
Packit Service |
310c69 |
.cache = cache,
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
VDOCompletion *completion = &pageCompletion->completion;
|
|
Packit Service |
310c69 |
initializeCompletion(completion, VDO_PAGE_COMPLETION, cache->layer);
|
|
Packit Service |
310c69 |
prepareCompletion(completion, callback, errorHandler, cache->zone->threadID,
|
|
Packit Service |
310c69 |
parent);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Helper function to check that a completion represents a successfully
|
|
Packit Service |
310c69 |
* completed VDO Page Completion referring to a valid page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion a VDO completion
|
|
Packit Service |
310c69 |
* @param writable whether a writable page is required
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return the embedding completion if valid, NULL if not
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static VDOPageCompletion *validateCompletedPage(VDOCompletion *completion,
|
|
Packit Service |
310c69 |
bool writable)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCompletion *vpc = asVDOPageCompletion(completion);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = ASSERT(vpc->ready, "VDO Page completion not ready");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = ASSERT(vpc->info != NULL, "VDO Page Completion must be complete");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = ASSERT(vpc->info->pbn == vpc->pbn,
|
|
Packit Service |
310c69 |
"VDO Page Completion pbn must be consistent");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = ASSERT(isValid(vpc->info),
|
|
Packit Service |
310c69 |
"VDO Page Completion page must be valid");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (writable) {
|
|
Packit Service |
310c69 |
result = ASSERT(vpc->writable, "VDO Page Completion is writable");
|
|
Packit Service |
310c69 |
if (result != UDS_SUCCESS) {
|
|
Packit Service |
310c69 |
return NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
return vpc;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
bool isPageCacheActive(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ((cache->outstandingReads != 0) || (cache->outstandingWrites != 0));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* VIO callback used when a page has been loaded.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion A completion for the VIO, the parent of which is a
|
|
Packit Service |
310c69 |
* PageInfo.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void pageIsLoaded(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setInfoState(info, PS_RESIDENT);
|
|
Packit Service |
310c69 |
distributePageOverQueue(info, &info->waiting);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Don't decrement until right before calling checkForDrainComplete() to
|
|
Packit Service |
310c69 |
* ensure that the above work can't cause the page cache to be freed out from
|
|
Packit Service |
310c69 |
* under us.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
cache->outstandingReads--;
|
|
Packit Service |
310c69 |
checkForDrainComplete(cache->zone);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Handle page load errors.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The page read VIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void handleLoadError(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = completion->result;
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
enterReadOnlyMode(cache->zone->readOnlyNotifier, result);
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.failedReads, 1);
|
|
Packit Service |
310c69 |
setInfoState(info, PS_FAILED);
|
|
Packit Service |
310c69 |
distributeErrorOverQueue(result, &info->waiting);
|
|
Packit Service |
310c69 |
resetPageInfo(info);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Don't decrement until right before calling checkForDrainComplete() to
|
|
Packit Service |
310c69 |
* ensure that the above work can't cause the page cache to be freed out from
|
|
Packit Service |
310c69 |
* under us.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
cache->outstandingReads--;
|
|
Packit Service |
310c69 |
checkForDrainComplete(cache->zone);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Run the read hook after a page is loaded. This callback is registered in
|
|
Packit Service |
310c69 |
* launchPageLoad() when there is a read hook.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The page load completion
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void runReadHook(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
completion->callback = pageIsLoaded;
|
|
Packit Service |
310c69 |
resetCompletion(completion);
|
|
Packit Service |
310c69 |
int result = info->cache->readHook(getPageBuffer(info), info->pbn,
|
|
Packit Service |
310c69 |
info->cache->zone, info->context);
|
|
Packit Service |
310c69 |
continueCompletion(completion, result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Handle a read error during a read-only rebuild.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The page load completion
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void handleRebuildReadError(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// We are doing a read-only rebuild, so treat this as a successful read
|
|
Packit Service |
310c69 |
// of an uninitialized page.
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.failedReads, 1);
|
|
Packit Service |
310c69 |
memset(getPageBuffer(info), 0, VDO_BLOCK_SIZE);
|
|
Packit Service |
310c69 |
resetCompletion(completion);
|
|
Packit Service |
310c69 |
if (cache->readHook != NULL) {
|
|
Packit Service |
310c69 |
runReadHook(completion);
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
pageIsLoaded(completion);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Begin the process of loading a page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info the page info representing where to load the page
|
|
Packit Service |
310c69 |
* @param pbn the absolute pbn of the desired page
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return VDO_SUCCESS or an error code
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
__attribute__((warn_unused_result))
|
|
Packit Service |
310c69 |
static int launchPageLoad(PageInfo *info, PhysicalBlockNumber pbn)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
assertIOAllowed(cache);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = setInfoPBN(info, pbn);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = ASSERT((info->busy == 0), "Page is not busy before loading.");
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setInfoState(info, PS_INCOMING);
|
|
Packit Service |
310c69 |
cache->outstandingReads++;
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.pagesLoaded, 1);
|
|
Packit Service |
310c69 |
launchReadMetadataVIO(info->vio, pbn,
|
|
Packit Service |
310c69 |
(cache->readHook != NULL) ? runReadHook : pageIsLoaded,
|
|
Packit Service |
310c69 |
(cache->rebuilding
|
|
Packit Service |
310c69 |
? handleRebuildReadError : handleLoadError));
|
|
Packit Service |
310c69 |
return VDO_SUCCESS;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static void writePages(VDOCompletion *completion);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Handle errors flushing the layer.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The flush VIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void handleFlushError(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = ((PageInfo *) completion->parent)->cache;
|
|
Packit Service |
310c69 |
setPersistentError(cache, "flush failed", completion->result);
|
|
Packit Service |
310c69 |
writePages(completion);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Attempt to save the outgoing pages by first flushing the layer.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache The cache
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void savePages(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if ((cache->pagesInFlush > 0) || (cache->pagesToFlush == 0)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
assertIOAllowed(cache);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *info = pageInfoFromListNode(cache->outgoingList.next);
|
|
Packit Service |
310c69 |
cache->pagesInFlush = cache->pagesToFlush;
|
|
Packit Service |
310c69 |
cache->pagesToFlush = 0;
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.flushCount, 1);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
VIO *vio = info->vio;
|
|
Packit Service |
310c69 |
PhysicalLayer *layer = vio->completion.layer;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* We must make sure that the recovery journal entries that changed these
|
|
Packit Service |
310c69 |
* pages were successfully persisted, and thus must issue a flush before
|
|
Packit Service |
310c69 |
* each batch of pages is written to ensure this. However, in sync mode,
|
|
Packit Service |
310c69 |
* every journal block is written with FUA, thus guaranteeing the journal
|
|
Packit Service |
310c69 |
* persisted already.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
if (layer->getWritePolicy(layer) != WRITE_POLICY_SYNC) {
|
|
Packit Service |
310c69 |
launchFlush(vio, writePages, handleFlushError);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
writePages(&vio->completion);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Add a page to the outgoing list of pages waiting to be saved. Once in the
|
|
Packit Service |
310c69 |
* list, a page may not be used until it has been written out.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info The page to save
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void schedulePageSave(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (info->busy > 0) {
|
|
Packit Service |
310c69 |
info->writeStatus = WRITE_STATUS_DEFERRED;
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
info->cache->pagesToFlush++;
|
|
Packit Service |
310c69 |
info->cache->outstandingWrites++;
|
|
Packit Service |
310c69 |
setInfoState(info, PS_OUTGOING);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static void writeDirtyPagesCallback(RingNode *expired, void *context)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
while (!isRingEmpty(expired)) {
|
|
Packit Service |
310c69 |
schedulePageSave(pageInfoFromListNode(chopRingNode(expired)));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
savePages((VDOPageCache *) context);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Add a page to outgoing pages waiting to be saved, and then start saving
|
|
Packit Service |
310c69 |
* pages if another save is not in progress.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info The page to save
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void launchPageSave(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
schedulePageSave(info);
|
|
Packit Service |
310c69 |
savePages(info->cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Determine whether a given VDOPageCompletion (as a waiter) is requesting a
|
|
Packit Service |
310c69 |
* given page number. Implements WaiterMatch.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param waiter The page completion in question
|
|
Packit Service |
310c69 |
* @param context A pointer to the pbn of the desired page
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return true if the page completion is for the desired page number
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static bool completionNeedsPage(Waiter *waiter, void *context)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PhysicalBlockNumber *pbn = context;
|
|
Packit Service |
310c69 |
return (pageCompletionFromWaiter(waiter)->pbn == *pbn);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Allocate a free page to the first completion in the waiting queue,
|
|
Packit Service |
310c69 |
* and any other completions that match it in page number.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void allocateFreePage(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!hasWaiters(&cache->freeWaiters)) {
|
|
Packit Service |
310c69 |
if (relaxedLoad64(&cache->stats.cachePressure) > 0) {
|
|
Packit Service |
310c69 |
logInfo("page cache pressure relieved");
|
|
Packit Service |
310c69 |
relaxedStore64(&cache->stats.cachePressure, 0);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = resetPageInfo(info);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
setPersistentError(cache, "cannot reset page info", result);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
Waiter *oldestWaiter = getFirstWaiter(&cache->freeWaiters);
|
|
Packit Service |
310c69 |
PhysicalBlockNumber pbn = pageCompletionFromWaiter(oldestWaiter)->pbn;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Remove all entries which match the page number in question
|
|
Packit Service |
310c69 |
// and push them onto the page info's wait queue.
|
|
Packit Service |
310c69 |
dequeueMatchingWaiters(&cache->freeWaiters, completionNeedsPage,
|
|
Packit Service |
310c69 |
&pbn, &info->waiting);
|
|
Packit Service |
310c69 |
cache->waiterCount -= countWaiters(&info->waiting);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = launchPageLoad(info, pbn);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
distributeErrorOverQueue(result, &info->waiting);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Begin the process of discarding a page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @note If no page is discardable, increments a count of deferred frees so
|
|
Packit Service |
310c69 |
* that the next release of a page which is no longer busy will kick
|
|
Packit Service |
310c69 |
* off another discard cycle. This is an indication that the cache is
|
|
Packit Service |
310c69 |
* not big enough.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @note If the selected page is not dirty, immediately allocates the page
|
|
Packit Service |
310c69 |
* to the oldest completion waiting for a free page.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void discardAPage(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = selectLRUPage(cache);
|
|
Packit Service |
310c69 |
if (info == NULL) {
|
|
Packit Service |
310c69 |
reportCachePressure(cache);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!isDirty(info)) {
|
|
Packit Service |
310c69 |
allocateFreePage(info);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(!isInFlight(info),
|
|
Packit Service |
310c69 |
"page selected for discard is not in flight");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
++cache->discardCount;
|
|
Packit Service |
310c69 |
info->writeStatus = WRITE_STATUS_DISCARD;
|
|
Packit Service |
310c69 |
launchPageSave(info);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Helper used to trigger a discard so that the completion can get a different
|
|
Packit Service |
310c69 |
* page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param vdoPageComp the VDO Page completion
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void discardPageForCompletion(VDOPageCompletion *vdoPageComp)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = vdoPageComp->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
++cache->waiterCount;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = enqueueWaiter(&cache->freeWaiters, &vdoPageComp->waiter);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
setPersistentError(cache, "cannot enqueue waiter", result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
discardAPage(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Helper used to trigger a discard if the cache needs another free page.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param cache the page cache
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void discardPageIfNeeded(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (cache->waiterCount > cache->discardCount) {
|
|
Packit Service |
310c69 |
discardAPage(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void advanceVDOPageCachePeriod(VDOPageCache *cache, SequenceNumber period)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
advancePeriod(cache->dirtyLists, period);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Inform the cache that a write has finished (possibly with an error).
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info The info structure for the page whose write just completed
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return true if the page write was a discard
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static bool writeHasFinished(PageInfo *info)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
assertOnCacheThread(info->cache, __func__);
|
|
Packit Service |
310c69 |
info->cache->outstandingWrites--;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
bool wasDiscard = (info->writeStatus == WRITE_STATUS_DISCARD);
|
|
Packit Service |
310c69 |
info->writeStatus = WRITE_STATUS_NORMAL;
|
|
Packit Service |
310c69 |
return wasDiscard;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Handler for page write errors.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The page write VIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void handlePageWriteError(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = completion->result;
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// If we're already read-only, write failures are to be expected.
|
|
Packit Service |
310c69 |
if (result != VDO_READ_ONLY) {
|
|
Packit Service |
310c69 |
#if __KERNEL__
|
|
Packit Service |
310c69 |
static DEFINE_RATELIMIT_STATE(errorLimiter, DEFAULT_RATELIMIT_INTERVAL,
|
|
Packit Service |
310c69 |
DEFAULT_RATELIMIT_BURST);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (__ratelimit(&errorLimiter)) {
|
|
Packit Service |
310c69 |
logError("failed to write block map page %llu", info->pbn);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
#else
|
|
Packit Service |
310c69 |
logError("failed to write block map page %llu", info->pbn);
|
|
Packit Service |
310c69 |
#endif
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setInfoState(info, PS_DIRTY);
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.failedWrites, 1);
|
|
Packit Service |
310c69 |
setPersistentError(cache, "cannot write page", result);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!writeHasFinished(info)) {
|
|
Packit Service |
310c69 |
discardPageIfNeeded(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
checkForDrainComplete(cache->zone);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* VIO callback used when a page has been written out.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion A completion for the VIO, the parent of which
|
|
Packit Service |
310c69 |
* is embedded in PageInfo.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void pageIsWrittenOut(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
PageInfo *info = completion->parent;
|
|
Packit Service |
310c69 |
VDOPageCache *cache = info->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (cache->writeHook != NULL) {
|
|
Packit Service |
310c69 |
bool rewrite = cache->writeHook(getPageBuffer(info), cache->zone,
|
|
Packit Service |
310c69 |
info->context);
|
|
Packit Service |
310c69 |
if (rewrite) {
|
|
Packit Service |
310c69 |
launchWriteMetadataVIOWithFlush(info->vio, info->pbn, pageIsWrittenOut,
|
|
Packit Service |
310c69 |
handlePageWriteError, true, false);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
bool wasDiscard = writeHasFinished(info);
|
|
Packit Service |
310c69 |
bool reclaimed = (!wasDiscard || (info->busy > 0)
|
|
Packit Service |
310c69 |
|| hasWaiters(&info->waiting));
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setInfoState(info, PS_RESIDENT);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
uint32_t reclamations = distributePageOverQueue(info, &info->waiting);
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.reclaimed, reclamations);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (wasDiscard) {
|
|
Packit Service |
310c69 |
cache->discardCount--;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (reclaimed) {
|
|
Packit Service |
310c69 |
discardPageIfNeeded(cache);
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
allocateFreePage(info);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
checkForDrainComplete(cache->zone);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Write the batch of pages which were covered by the layer flush which just
|
|
Packit Service |
310c69 |
* completed. This callback is registered in savePages().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param flushCompletion The flush VIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void writePages(VDOCompletion *flushCompletion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCache *cache = ((PageInfo *) flushCompletion->parent)->cache;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* We need to cache these two values on the stack since in the error case
|
|
Packit Service |
310c69 |
* below, it is possible for the last page info to cause the page cache to
|
|
Packit Service |
310c69 |
* get freed. Hence once we launch the last page, it may be unsafe to
|
|
Packit Service |
310c69 |
* dereference the cache [VDO-4724].
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
bool hasUnflushedPages = (cache->pagesToFlush > 0);
|
|
Packit Service |
310c69 |
PageCount pagesInFlush = cache->pagesInFlush;
|
|
Packit Service |
310c69 |
cache->pagesInFlush = 0;
|
|
Packit Service |
310c69 |
while (pagesInFlush-- > 0) {
|
|
Packit Service |
310c69 |
PageInfo *info = pageInfoFromListNode(chopRingNode(&cache->outgoingList));
|
|
Packit Service |
310c69 |
if (isReadOnly(info->cache->zone->readOnlyNotifier)) {
|
|
Packit Service |
310c69 |
VDOCompletion *completion = &info->vio->completion;
|
|
Packit Service |
310c69 |
resetCompletion(completion);
|
|
Packit Service |
310c69 |
completion->callback = pageIsWrittenOut;
|
|
Packit Service |
310c69 |
completion->errorHandler = handlePageWriteError;
|
|
Packit Service |
310c69 |
finishCompletion(completion, VDO_READ_ONLY);
|
|
Packit Service |
310c69 |
continue;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
relaxedAdd64(&info->cache->stats.pagesSaved, 1);
|
|
Packit Service |
310c69 |
launchWriteMetadataVIO(info->vio, info->pbn, pageIsWrittenOut,
|
|
Packit Service |
310c69 |
handlePageWriteError);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (hasUnflushedPages) {
|
|
Packit Service |
310c69 |
// If there are unflushed pages, the cache can't have been freed, so this
|
|
Packit Service |
310c69 |
// call is safe.
|
|
Packit Service |
310c69 |
savePages(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void releaseVDOPageCompletion(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (completion == NULL) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *discardInfo = NULL;
|
|
Packit Service |
310c69 |
VDOPageCompletion *pageCompletion;
|
|
Packit Service |
310c69 |
if (completion->result == VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
pageCompletion = validateCompletedPage(completion, false);
|
|
Packit Service |
310c69 |
if (--pageCompletion->info->busy == 0) {
|
|
Packit Service |
310c69 |
discardInfo = pageCompletion->info;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
// Do not check for errors if the completion was not successful.
|
|
Packit Service |
310c69 |
pageCompletion = asVDOPageCompletion(completion);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY((pageCompletion->waiter.nextWaiter == NULL),
|
|
Packit Service |
310c69 |
"Page being released after leaving all queues");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
VDOPageCache *cache = pageCompletion->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
memset(pageCompletion, 0, sizeof(VDOPageCompletion));
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (discardInfo != NULL) {
|
|
Packit Service |
310c69 |
if (discardInfo->writeStatus == WRITE_STATUS_DEFERRED) {
|
|
Packit Service |
310c69 |
discardInfo->writeStatus = WRITE_STATUS_NORMAL;
|
|
Packit Service |
310c69 |
launchPageSave(discardInfo);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// if there are excess requests for pages (that have not already started
|
|
Packit Service |
310c69 |
// discards) we need to discard some page (which may be this one)
|
|
Packit Service |
310c69 |
discardPageIfNeeded(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Helper function to load a page as described by a VDO Page Completion.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param info the page info representing where to load the page
|
|
Packit Service |
310c69 |
* @param vdoPageComp the VDO Page Completion describing the page
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void loadPageForCompletion(PageInfo *info,
|
|
Packit Service |
310c69 |
VDOPageCompletion *vdoPageComp)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
int result = enqueueWaiter(&info->waiting, &vdoPageComp->waiter);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
finishCompletion(&vdoPageComp->completion, result);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
result = launchPageLoad(info, vdoPageComp->pbn);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
distributeErrorOverQueue(result, &info->waiting);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void getVDOPageAsync(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCompletion *vdoPageComp = asVDOPageCompletion(completion);
|
|
Packit Service |
310c69 |
VDOPageCache *cache = vdoPageComp->cache;
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (vdoPageComp->writable && isReadOnly(cache->zone->readOnlyNotifier)) {
|
|
Packit Service |
310c69 |
finishCompletion(completion, VDO_READ_ONLY);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (vdoPageComp->writable) {
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.writeCount, 1);
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.readCount, 1);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *info = vpcFindPage(cache, vdoPageComp->pbn);
|
|
Packit Service |
310c69 |
if (info != NULL) {
|
|
Packit Service |
310c69 |
// The page is in the cache already.
|
|
Packit Service |
310c69 |
if ((info->writeStatus == WRITE_STATUS_DEFERRED) || isIncoming(info)
|
|
Packit Service |
310c69 |
|| (isOutgoing(info) && vdoPageComp->writable)) {
|
|
Packit Service |
310c69 |
// The page is unusable until it has finished I/O.
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.waitForPage, 1);
|
|
Packit Service |
310c69 |
int result = enqueueWaiter(&info->waiting, &vdoPageComp->waiter);
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
finishCompletion(&vdoPageComp->completion, result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (isValid(info)) {
|
|
Packit Service |
310c69 |
// The page is usable.
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.foundInCache, 1);
|
|
Packit Service |
310c69 |
if (!isPresent(info)) {
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.readOutgoing, 1);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
updateLru(info);
|
|
Packit Service |
310c69 |
++info->busy;
|
|
Packit Service |
310c69 |
completeWithPage(info, vdoPageComp);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// Something horrible has gone wrong.
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(false, "Info found in a usable state.");
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// The page must be fetched.
|
|
Packit Service |
310c69 |
info = findFreePage(cache);
|
|
Packit Service |
310c69 |
if (info != NULL) {
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.fetchRequired, 1);
|
|
Packit Service |
310c69 |
loadPageForCompletion(info, vdoPageComp);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// The page must wait for a page to be discarded.
|
|
Packit Service |
310c69 |
relaxedAdd64(&cache->stats.discardRequired, 1);
|
|
Packit Service |
310c69 |
discardPageForCompletion(vdoPageComp);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void markCompletedVDOPageDirty(VDOCompletion *completion,
|
|
Packit Service |
310c69 |
SequenceNumber oldDirtyPeriod,
|
|
Packit Service |
310c69 |
SequenceNumber newDirtyPeriod)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCompletion *vdoPageComp = validateCompletedPage(completion, true);
|
|
Packit Service |
310c69 |
if (vdoPageComp == NULL) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *info = vdoPageComp->info;
|
|
Packit Service |
310c69 |
setInfoState(info, PS_DIRTY);
|
|
Packit Service |
310c69 |
addToDirtyLists(info->cache->dirtyLists, &info->listNode, oldDirtyPeriod,
|
|
Packit Service |
310c69 |
newDirtyPeriod);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void requestVDOPageWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCompletion *vdoPageComp = validateCompletedPage(completion, true);
|
|
Packit Service |
310c69 |
if (vdoPageComp == NULL) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
PageInfo *info = vdoPageComp->info;
|
|
Packit Service |
310c69 |
setInfoState(info, PS_DIRTY);
|
|
Packit Service |
310c69 |
launchPageSave(info);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static void *dereferencePageCompletion(VDOPageCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ((completion != NULL) ? getPageBuffer(completion->info) : NULL);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
const void *dereferenceReadableVDOPage(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return dereferencePageCompletion(validateCompletedPage(completion, false));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void *dereferenceWritableVDOPage(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return dereferencePageCompletion(validateCompletedPage(completion, true));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void *getVDOPageCompletionContext(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
VDOPageCompletion *pageCompletion = asVDOPageCompletion(completion);
|
|
Packit Service |
310c69 |
PageInfo *info = ((pageCompletion != NULL) ? pageCompletion->info : NULL);
|
|
Packit Service |
310c69 |
return (((info != NULL) && isValid(info)) ? info->context : NULL);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void drainVDOPageCache(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(isDraining(&cache->zone->state),
|
|
Packit Service |
310c69 |
"drainVDOPageCache() called during block map drain");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!isSuspending(&cache->zone->state)) {
|
|
Packit Service |
310c69 |
flushDirtyLists(cache->dirtyLists);
|
|
Packit Service |
310c69 |
savePages(cache);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
int invalidateVDOPageCache(VDOPageCache *cache)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
assertOnCacheThread(cache, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Make sure we don't throw away any dirty pages.
|
|
Packit Service |
310c69 |
PageInfo *info;
|
|
Packit Service |
310c69 |
for (info = cache->infos; info < cache->infos + cache->pageCount; info++) {
|
|
Packit Service |
310c69 |
int result = ASSERT(!isDirty(info), "cache must have no dirty pages");
|
|
Packit Service |
310c69 |
if (result != VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Reset the pageMap by re-allocating it.
|
|
Packit Service |
310c69 |
freeIntMap(&cache->pageMap);
|
|
Packit Service |
310c69 |
return makeIntMap(cache->pageCount, 0, &cache->pageMap);
|
|
Packit Service |
310c69 |
}
|