Blame source/vdo/base/vdoPageCache.c

Packit Service 310c69
/*
Packit Service 310c69
 * Copyright (c) 2020 Red Hat, Inc.
Packit Service 310c69
 *
Packit Service 310c69
 * This program is free software; you can redistribute it and/or
Packit Service 310c69
 * modify it under the terms of the GNU General Public License
Packit Service 310c69
 * as published by the Free Software Foundation; either version 2
Packit Service 310c69
 * of the License, or (at your option) any later version.
Packit Service 310c69
 * 
Packit Service 310c69
 * This program is distributed in the hope that it will be useful,
Packit Service 310c69
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Packit Service 310c69
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Packit Service 310c69
 * GNU General Public License for more details.
Packit Service 310c69
 * 
Packit Service 310c69
 * You should have received a copy of the GNU General Public License
Packit Service 310c69
 * along with this program; if not, write to the Free Software
Packit Service 310c69
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
Packit Service 310c69
 * 02110-1301, USA. 
Packit Service 310c69
 *
Packit Service 310c69
 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/base/vdoPageCache.c#11 $
Packit Service 310c69
 */
Packit Service 310c69
Packit Service 310c69
#include "vdoPageCacheInternals.h"
Packit Service 310c69
Packit Service 310c69
#if __KERNEL__
Packit Service 310c69
#include <linux/ratelimit.h>
Packit Service 310c69
#endif
Packit Service 310c69
Packit Service 310c69
#include "errors.h"
Packit Service 310c69
#include "logger.h"
Packit Service 310c69
#include "memoryAlloc.h"
Packit Service 310c69
#include "permassert.h"
Packit Service 310c69
Packit Service 310c69
#include "adminState.h"
Packit Service 310c69
#include "constants.h"
Packit Service 310c69
#include "numUtils.h"
Packit Service 310c69
#include "readOnlyNotifier.h"
Packit Service 310c69
#include "statusCodes.h"
Packit Service 310c69
#include "types.h"
Packit Service 310c69
#include "vio.h"
Packit Service 310c69
Packit Service 310c69
enum {
Packit Service 310c69
  LOG_INTERVAL                = 4000,
Packit Service 310c69
  DISPLAY_INTERVAL            = 100000,
Packit Service 310c69
};
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
static char *getPageBuffer(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
  return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE];
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Allocate components of the cache which require their own allocation. The
Packit Service 310c69
 * caller is responsible for all clean up on errors.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache     The cache being constructed
Packit Service 310c69
 *
Packit Service 310c69
 * @return VDO_SUCCESS or an error code
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static int allocateCacheComponents(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  int result = ALLOCATE(cache->pageCount, PageInfo, "page infos",
Packit Service 310c69
                        &cache->infos);
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  uint64_t size = cache->pageCount * (uint64_t) VDO_BLOCK_SIZE;
Packit Service 310c69
  result = allocateMemory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages);
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  return makeIntMap(cache->pageCount, 0, &cache->pageMap);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Initialize all page info structures and put them on the free list.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache  The cache to initialize
Packit Service 310c69
 *
Packit Service 310c69
 * @return VDO_SUCCESS or an error
Packit Service 310c69
 **/
Packit Service 310c69
static int initializeInfo(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  initializeRing(&cache->freeList);
Packit Service 310c69
  PageInfo *info;
Packit Service 310c69
  for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
Packit Service 310c69
    info->cache = cache;
Packit Service 310c69
    info->state = PS_FREE;
Packit Service 310c69
    info->pbn   = NO_PAGE;
Packit Service 310c69
Packit Service 310c69
    if (cache->layer->createMetadataVIO != NULL) {
Packit Service 310c69
      int result = createVIO(cache->layer, VIO_TYPE_BLOCK_MAP,
Packit Service 310c69
                             VIO_PRIORITY_METADATA, info, getPageBuffer(info),
Packit Service 310c69
                             &info->vio);
Packit Service 310c69
      if (result != VDO_SUCCESS) {
Packit Service 310c69
        return result;
Packit Service 310c69
      }
Packit Service 310c69
Packit Service 310c69
      // The thread ID should never change.
Packit Service 310c69
      info->vio->completion.callbackThreadID = cache->zone->threadID;
Packit Service 310c69
    }
Packit Service 310c69
Packit Service 310c69
    initializeRing(&info->listNode);
Packit Service 310c69
    pushRingNode(&cache->freeList, &info->listNode);
Packit Service 310c69
    initializeRing(&info->lruNode);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  relaxedStore64(&cache->stats.counts.freePages, cache->pageCount);
Packit Service 310c69
  return VDO_SUCCESS;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
static void writeDirtyPagesCallback(RingNode *node, void *context);
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
int makeVDOPageCache(PhysicalLayer         *layer,
Packit Service 310c69
                     PageCount              pageCount,
Packit Service 310c69
                     VDOPageReadFunction   *readHook,
Packit Service 310c69
                     VDOPageWriteFunction  *writeHook,
Packit Service 310c69
                     size_t                 pageContextSize,
Packit Service 310c69
                     BlockCount             maximumAge,
Packit Service 310c69
                     BlockMapZone          *zone,
Packit Service 310c69
                     VDOPageCache         **cachePtr)
Packit Service 310c69
{
Packit Service 310c69
  int result = ASSERT(pageContextSize <= MAX_PAGE_CONTEXT_SIZE,
Packit Service 310c69
                      "page context size %zu cannot exceed %u bytes",
Packit Service 310c69
                      pageContextSize, MAX_PAGE_CONTEXT_SIZE);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  VDOPageCache *cache;
Packit Service 310c69
  result = ALLOCATE(1, VDOPageCache, "page cache", &cache);
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  cache->layer            = layer;
Packit Service 310c69
  cache->pageCount        = pageCount;
Packit Service 310c69
  cache->readHook         = readHook;
Packit Service 310c69
  cache->writeHook        = writeHook;
Packit Service 310c69
  cache->zone             = zone;
Packit Service 310c69
Packit Service 310c69
  result = allocateCacheComponents(cache);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    freeVDOPageCache(&cache);
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = initializeInfo(cache);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    freeVDOPageCache(&cache);
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = makeDirtyLists(maximumAge, writeDirtyPagesCallback, cache,
Packit Service 310c69
                          &cache->dirtyLists);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    freeVDOPageCache(&cache);
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  // initialize empty circular queues
Packit Service 310c69
  initializeRing(&cache->lruList);
Packit Service 310c69
  initializeRing(&cache->outgoingList);
Packit Service 310c69
Packit Service 310c69
  *cachePtr = cache;
Packit Service 310c69
  return VDO_SUCCESS;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void freeVDOPageCache(VDOPageCache **cachePtr)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = *cachePtr;
Packit Service 310c69
  if (cache == NULL) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (cache->infos != NULL) {
Packit Service 310c69
    PageInfo *info;
Packit Service 310c69
    for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
Packit Service 310c69
      freeVIO(&info->vio);
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  freeDirtyLists(&cache->dirtyLists);
Packit Service 310c69
  freeIntMap(&cache->pageMap);
Packit Service 310c69
  FREE(cache->infos);
Packit Service 310c69
  FREE(cache->pages);
Packit Service 310c69
  FREE(cache);
Packit Service 310c69
  *cachePtr = NULL;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void setVDOPageCacheInitialPeriod(VDOPageCache *cache, SequenceNumber period)
Packit Service 310c69
{
Packit Service 310c69
  setCurrentPeriod(cache->dirtyLists, period);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void setVDOPageCacheRebuildMode(VDOPageCache *cache, bool rebuilding)
Packit Service 310c69
{
Packit Service 310c69
  cache->rebuilding = rebuilding;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Assert that a function has been called on the VDO page cache's thread.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 * @param functionName  the name of the function
Packit Service 310c69
 **/
Packit Service 310c69
static inline void assertOnCacheThread(VDOPageCache *cache,
Packit Service 310c69
                                       const char   *functionName)
Packit Service 310c69
{
Packit Service 310c69
  ThreadID threadID = getCallbackThreadID();
Packit Service 310c69
  ASSERT_LOG_ONLY((threadID == cache->zone->threadID),
Packit Service 310c69
                  "%s() must only be called on cache thread %d, not thread %d",
Packit Service 310c69
                  functionName, cache->zone->threadID, threadID);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Assert that a page cache may issue I/O.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache  the page cache
Packit Service 310c69
 **/
Packit Service 310c69
static inline void assertIOAllowed(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  ASSERT_LOG_ONLY(!isQuiescent(&cache->zone->state),
Packit Service 310c69
                  "VDO page cache may issue I/O");
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Log and, if enabled, report cache pressure.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 **/
Packit Service 310c69
static void reportCachePressure(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  relaxedAdd64(&cache->stats.cachePressure, 1);
Packit Service 310c69
  if (cache->waiterCount > cache->pageCount) {
Packit Service 310c69
    if ((cache->pressureReport % LOG_INTERVAL) == 0) {
Packit Service 310c69
      logInfo("page cache pressure %llu",
Packit Service 310c69
              relaxedLoad64(&cache->stats.cachePressure));
Packit Service 310c69
    }
Packit Service 310c69
Packit Service 310c69
    if (++cache->pressureReport >= DISPLAY_INTERVAL) {
Packit Service 310c69
      cache->pressureReport = 0;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
const char *vpcPageStateName(PageState state)
Packit Service 310c69
{
Packit Service 310c69
  static const char *stateNames[] = {
Packit Service 310c69
    "FREE",
Packit Service 310c69
    "INCOMING",
Packit Service 310c69
    "FAILED",
Packit Service 310c69
    "RESIDENT",
Packit Service 310c69
    "DIRTY",
Packit Service 310c69
    "OUTGOING"
Packit Service 310c69
  };
Packit Service 310c69
  STATIC_ASSERT(COUNT_OF(stateNames) == PAGE_STATE_COUNT);
Packit Service 310c69
Packit Service 310c69
  int result = ASSERT(state < COUNT_OF(stateNames),
Packit Service 310c69
                      "Unknown PageState value %d", state);
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return "[UNKNOWN PAGE STATE]";
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  return stateNames[state];
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Update the counter associated with a given state.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info   the page info to count
Packit Service 310c69
 * @param delta  the delta to apply to the counter
Packit Service 310c69
 **/
Packit Service 310c69
static void updateCounter(PageInfo *info, int32_t delta)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
  switch (info->state) {
Packit Service 310c69
    case PS_FREE:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.freePages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    case PS_INCOMING:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.incomingPages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    case PS_OUTGOING:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.outgoingPages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    case PS_FAILED:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.failedPages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    case PS_RESIDENT:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.cleanPages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    case PS_DIRTY:
Packit Service 310c69
      relaxedAdd64(&cache->stats.counts.dirtyPages, delta);
Packit Service 310c69
      return;
Packit Service 310c69
Packit Service 310c69
    default:
Packit Service 310c69
      return;
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Update the lru information for an active page.
Packit Service 310c69
 **/
Packit Service 310c69
static void updateLru(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
Packit Service 310c69
  if (cache->lruList.prev != &info->lruNode) {
Packit Service 310c69
    pushRingNode(&cache->lruList, &info->lruNode);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Set the state of a PageInfo and put it on the right list, adjusting
Packit Service 310c69
 * counters.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info      the PageInfo to modify
Packit Service 310c69
 * @param newState  the new state for the PageInfo
Packit Service 310c69
 **/
Packit Service 310c69
static void setInfoState(PageInfo *info, PageState newState)
Packit Service 310c69
{
Packit Service 310c69
  if (newState == info->state) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  updateCounter(info, -1);
Packit Service 310c69
  info->state = newState;
Packit Service 310c69
  updateCounter(info, 1);
Packit Service 310c69
Packit Service 310c69
  switch (info->state) {
Packit Service 310c69
  case PS_FREE:
Packit Service 310c69
  case PS_FAILED:
Packit Service 310c69
    pushRingNode(&info->cache->freeList, &info->listNode);
Packit Service 310c69
    return;
Packit Service 310c69
Packit Service 310c69
  case PS_OUTGOING:
Packit Service 310c69
    pushRingNode(&info->cache->outgoingList, &info->listNode);
Packit Service 310c69
    return;
Packit Service 310c69
Packit Service 310c69
  case PS_DIRTY:
Packit Service 310c69
    return;
Packit Service 310c69
Packit Service 310c69
  default:
Packit Service 310c69
    unspliceRingNode(&info->listNode);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Set the pbn for an info, updating the map as needed.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info  The page info
Packit Service 310c69
 * @param pbn   The physical block number to set
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static int setInfoPBN(PageInfo *info, PhysicalBlockNumber pbn)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
Packit Service 310c69
  // Either the new or the old page number must be NO_PAGE.
Packit Service 310c69
  int result = ASSERT((pbn == NO_PAGE) || (info->pbn == NO_PAGE),
Packit Service 310c69
                      "Must free a page before reusing it.");
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (info->pbn != NO_PAGE) {
Packit Service 310c69
    intMapRemove(cache->pageMap, info->pbn);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  info->pbn = pbn;
Packit Service 310c69
Packit Service 310c69
  if (pbn != NO_PAGE) {
Packit Service 310c69
    result = intMapPut(cache->pageMap, pbn, info, true, NULL);
Packit Service 310c69
    if (result != UDS_SUCCESS) {
Packit Service 310c69
      return result;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
  return VDO_SUCCESS;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Reset page info to represent an unallocated page.
Packit Service 310c69
 **/
Packit Service 310c69
static int resetPageInfo(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  int result = ASSERT(info->busy == 0, "VDO Page must not be busy");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = ASSERT(!hasWaiters(&info->waiting),
Packit Service 310c69
                  "VDO Page must not have waiters");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = setInfoPBN(info, NO_PAGE);
Packit Service 310c69
  setInfoState(info, PS_FREE);
Packit Service 310c69
  unspliceRingNode(&info->lruNode);
Packit Service 310c69
  return result;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Find a free page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 *
Packit Service 310c69
 * @return a pointer to the page info structure (if found), NULL otherwise
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static PageInfo *findFreePage(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  if (cache->freeList.next == &cache->freeList) {
Packit Service 310c69
    return NULL;
Packit Service 310c69
  }
Packit Service 310c69
  PageInfo *info = pageInfoFromListNode(cache->freeList.next);
Packit Service 310c69
  unspliceRingNode(&info->listNode);
Packit Service 310c69
  return info;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
PageInfo *vpcFindPage(VDOPageCache *cache, PhysicalBlockNumber pbn)
Packit Service 310c69
{
Packit Service 310c69
  if ((cache->lastFound != NULL)
Packit Service 310c69
      && (cache->lastFound->pbn == pbn)) {
Packit Service 310c69
    return cache->lastFound;
Packit Service 310c69
  }
Packit Service 310c69
  cache->lastFound = intMapGet(cache->pageMap, pbn);
Packit Service 310c69
  return cache->lastFound;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Determine which page is least recently used.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache structure
Packit Service 310c69
 *
Packit Service 310c69
 * @return a pointer to the info structure for a relevant page,
Packit Service 310c69
 *         or NULL if no such page can be found. The page can be
Packit Service 310c69
 *         dirty or resident.
Packit Service 310c69
 *
Packit Service 310c69
 * @note Picks the least recently used from among the non-busy entries
Packit Service 310c69
 *       at the front of each of the lru ring.
Packit Service 310c69
 *       Since whenever we mark a page busy we also put it to the end
Packit Service 310c69
 *       of the ring it is unlikely that the entries at the front
Packit Service 310c69
 *       are busy unless the queue is very short, but not impossible.
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static PageInfo *selectLRUPage(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  PageInfoNode *lru;
Packit Service 310c69
  for (lru = cache->lruList.next;
Packit Service 310c69
       lru != &cache->lruList;
Packit Service 310c69
       lru = lru->next) {
Packit Service 310c69
    PageInfo *info = pageInfoFromLRUNode(lru);
Packit Service 310c69
    if ((info->busy == 0) && !isInFlight(info)) {
Packit Service 310c69
      return info;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  return NULL;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
AtomicPageCacheStatistics *getVDOPageCacheStatistics(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  return &cache->stats;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
// ASYNCHRONOUS INTERFACE BEYOND THIS POINT
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Helper to complete the VDO Page Completion request successfully.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info          the page info representing the result page
Packit Service 310c69
 * @param vdoPageComp   the VDO page completion to complete
Packit Service 310c69
 **/
Packit Service 310c69
static void completeWithPage(PageInfo *info, VDOPageCompletion *vdoPageComp)
Packit Service 310c69
{
Packit Service 310c69
  bool available = vdoPageComp->writable ? isPresent(info) : isValid(info);
Packit Service 310c69
  if (!available) {
Packit Service 310c69
    logErrorWithStringError(VDO_BAD_PAGE,
Packit Service 310c69
                            "Requested cache page %llu in state %s is"
Packit Service 310c69
                            " not %s",
Packit Service 310c69
                            info->pbn, vpcPageStateName(info->state),
Packit Service 310c69
                            vdoPageComp->writable ? "present" : "valid");
Packit Service 310c69
    finishCompletion(&vdoPageComp->completion, VDO_BAD_PAGE);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  vdoPageComp->info = info;
Packit Service 310c69
  vdoPageComp->ready = true;
Packit Service 310c69
  finishCompletion(&vdoPageComp->completion, VDO_SUCCESS);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Complete a page completion with an error code. Implements WaiterCallback.
Packit Service 310c69
 *
Packit Service 310c69
 * @param waiter        The page completion, as a waiter
Packit Service 310c69
 * @param resultPtr     A pointer to the error code.
Packit Service 310c69
 **/
Packit Service 310c69
static void completeWaiterWithError(Waiter *waiter, void *resultPtr)
Packit Service 310c69
{
Packit Service 310c69
  int               *result     = resultPtr;
Packit Service 310c69
  VDOPageCompletion *completion = pageCompletionFromWaiter(waiter);
Packit Service 310c69
  finishCompletion(&completion->completion, *result);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Complete a queue of VDOPageCompletions with an error code.
Packit Service 310c69
 *
Packit Service 310c69
 * @param [in]      result      the error result
Packit Service 310c69
 * @param [in, out] queue       a pointer to the queue
Packit Service 310c69
 *
Packit Service 310c69
 * @note upon completion the queue will be empty
Packit Service 310c69
 **/
Packit Service 310c69
static void distributeErrorOverQueue(int result, WaitQueue *queue)
Packit Service 310c69
{
Packit Service 310c69
  notifyAllWaiters(queue, completeWaiterWithError, &result);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Complete a page completion with a page. Implements WaiterCallback.
Packit Service 310c69
 *
Packit Service 310c69
 * @param waiter        The page completion, as a waiter
Packit Service 310c69
 * @param pageInfo      The page info to complete with
Packit Service 310c69
 **/
Packit Service 310c69
static void completeWaiterWithPage(Waiter *waiter, void *pageInfo)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo *info = pageInfo;
Packit Service 310c69
  VDOPageCompletion *completion = pageCompletionFromWaiter(waiter);
Packit Service 310c69
  completeWithPage(info, completion);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Complete a queue of VDOPageCompletions with a page result.
Packit Service 310c69
 *
Packit Service 310c69
 * @param [in]      info        the page info describing the page
Packit Service 310c69
 * @param [in, out] queue       a pointer to a queue of waiters
Packit Service 310c69
 *
Packit Service 310c69
 * @return the number of pages distributed
Packit Service 310c69
 *
Packit Service 310c69
 * @note upon completion the queue will be empty
Packit Service 310c69
 *
Packit Service 310c69
 **/
Packit Service 310c69
static unsigned int distributePageOverQueue(PageInfo *info, WaitQueue *queue)
Packit Service 310c69
{
Packit Service 310c69
  updateLru(info);
Packit Service 310c69
Packit Service 310c69
  size_t pages = countWaiters(queue);
Packit Service 310c69
Packit Service 310c69
  /*
Packit Service 310c69
   * Increment the busy count once for each pending completion so that
Packit Service 310c69
   * this page does not stop being busy until all completions have
Packit Service 310c69
   * been processed (VDO-83).
Packit Service 310c69
   */
Packit Service 310c69
  info->busy += pages;
Packit Service 310c69
Packit Service 310c69
  notifyAllWaiters(queue, completeWaiterWithPage, info);
Packit Service 310c69
  return pages;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Set a persistent error which all requests will receive in the future.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 * @param context       a string describing what triggered the error
Packit Service 310c69
 * @param result        the error result
Packit Service 310c69
 *
Packit Service 310c69
 * Once triggered, all enqueued completions will get this error.
Packit Service 310c69
 * Any future requests will result in this error as well.
Packit Service 310c69
 **/
Packit Service 310c69
static void setPersistentError(VDOPageCache *cache,
Packit Service 310c69
                               const char   *context,
Packit Service 310c69
                               int           result)
Packit Service 310c69
{
Packit Service 310c69
  // If we're already read-only, there's no need to log.
Packit Service 310c69
  ReadOnlyNotifier *notifier = cache->zone->readOnlyNotifier;
Packit Service 310c69
  if ((result != VDO_READ_ONLY) && !isReadOnly(notifier)) {
Packit Service 310c69
    logErrorWithStringError(result, "VDO Page Cache persistent error: %s",
Packit Service 310c69
                            context);
Packit Service 310c69
    enterReadOnlyMode(notifier, result);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  distributeErrorOverQueue(result, &cache->freeWaiters);
Packit Service 310c69
  cache->waiterCount = 0;
Packit Service 310c69
Packit Service 310c69
  PageInfo *info;
Packit Service 310c69
  for (info = cache->infos; info < cache->infos + cache->pageCount; ++info) {
Packit Service 310c69
    distributeErrorOverQueue(result, &info->waiting);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void initVDOPageCompletion(VDOPageCompletion   *pageCompletion,
Packit Service 310c69
                           VDOPageCache        *cache,
Packit Service 310c69
                           PhysicalBlockNumber  pbn,
Packit Service 310c69
                           bool                 writable,
Packit Service 310c69
                           void                *parent,
Packit Service 310c69
                           VDOAction           *callback,
Packit Service 310c69
                           VDOAction           *errorHandler)
Packit Service 310c69
{
Packit Service 310c69
  ASSERT_LOG_ONLY((pageCompletion->waiter.nextWaiter == NULL),
Packit Service 310c69
                  "New page completion was not already on a wait queue");
Packit Service 310c69
Packit Service 310c69
  *pageCompletion = (VDOPageCompletion) {
Packit Service 310c69
    .pbn      = pbn,
Packit Service 310c69
    .writable = writable,
Packit Service 310c69
    .cache    = cache,
Packit Service 310c69
  };
Packit Service 310c69
Packit Service 310c69
  VDOCompletion *completion = &pageCompletion->completion;
Packit Service 310c69
  initializeCompletion(completion, VDO_PAGE_COMPLETION, cache->layer);
Packit Service 310c69
  prepareCompletion(completion, callback, errorHandler, cache->zone->threadID,
Packit Service 310c69
                    parent);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Helper function to check that a completion represents a successfully
Packit Service 310c69
 * completed VDO Page Completion referring to a valid page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion    a VDO completion
Packit Service 310c69
 * @param writable      whether a writable page is required
Packit Service 310c69
 *
Packit Service 310c69
 * @return the embedding completion if valid, NULL if not
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static VDOPageCompletion *validateCompletedPage(VDOCompletion *completion,
Packit Service 310c69
                                                bool           writable)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCompletion *vpc = asVDOPageCompletion(completion);
Packit Service 310c69
Packit Service 310c69
  int result = ASSERT(vpc->ready, "VDO Page completion not ready");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return NULL;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = ASSERT(vpc->info != NULL, "VDO Page Completion must be complete");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return NULL;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = ASSERT(vpc->info->pbn == vpc->pbn,
Packit Service 310c69
                  "VDO Page Completion pbn must be consistent");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return NULL;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = ASSERT(isValid(vpc->info),
Packit Service 310c69
                  "VDO Page Completion page must be valid");
Packit Service 310c69
  if (result != UDS_SUCCESS) {
Packit Service 310c69
    return NULL;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (writable) {
Packit Service 310c69
    result = ASSERT(vpc->writable, "VDO Page Completion is writable");
Packit Service 310c69
    if (result != UDS_SUCCESS) {
Packit Service 310c69
      return NULL;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  return vpc;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
bool isPageCacheActive(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  return ((cache->outstandingReads != 0) || (cache->outstandingWrites != 0));
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * VIO callback used when a page has been loaded.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  A completion for the VIO, the parent of which is a
Packit Service 310c69
 *                    PageInfo.
Packit Service 310c69
 **/
Packit Service 310c69
static void pageIsLoaded(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo     *info   = completion->parent;
Packit Service 310c69
  VDOPageCache *cache  = info->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  setInfoState(info, PS_RESIDENT);
Packit Service 310c69
  distributePageOverQueue(info, &info->waiting);
Packit Service 310c69
Packit Service 310c69
  /*
Packit Service 310c69
   * Don't decrement until right before calling checkForDrainComplete() to
Packit Service 310c69
   * ensure that the above work can't cause the page cache to be freed out from
Packit Service 310c69
   * under us.
Packit Service 310c69
   */
Packit Service 310c69
  cache->outstandingReads--;
Packit Service 310c69
  checkForDrainComplete(cache->zone);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Handle page load errors.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  The page read VIO
Packit Service 310c69
 **/
Packit Service 310c69
static void handleLoadError(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  int           result = completion->result;
Packit Service 310c69
  PageInfo     *info   = completion->parent;
Packit Service 310c69
  VDOPageCache *cache  = info->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  enterReadOnlyMode(cache->zone->readOnlyNotifier, result);
Packit Service 310c69
  relaxedAdd64(&cache->stats.failedReads, 1);
Packit Service 310c69
  setInfoState(info, PS_FAILED);
Packit Service 310c69
  distributeErrorOverQueue(result, &info->waiting);
Packit Service 310c69
  resetPageInfo(info);
Packit Service 310c69
Packit Service 310c69
  /*
Packit Service 310c69
   * Don't decrement until right before calling checkForDrainComplete() to
Packit Service 310c69
   * ensure that the above work can't cause the page cache to be freed out from
Packit Service 310c69
   * under us.
Packit Service 310c69
   */
Packit Service 310c69
  cache->outstandingReads--;
Packit Service 310c69
  checkForDrainComplete(cache->zone);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Run the read hook after a page is loaded. This callback is registered in
Packit Service 310c69
 * launchPageLoad() when there is a read hook.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  The page load completion
Packit Service 310c69
 **/
Packit Service 310c69
static void runReadHook(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo *info       = completion->parent;
Packit Service 310c69
  completion->callback = pageIsLoaded;
Packit Service 310c69
  resetCompletion(completion);
Packit Service 310c69
  int result = info->cache->readHook(getPageBuffer(info), info->pbn,
Packit Service 310c69
                                     info->cache->zone, info->context);
Packit Service 310c69
  continueCompletion(completion, result);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Handle a read error during a read-only rebuild.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  The page load completion
Packit Service 310c69
 **/
Packit Service 310c69
static void handleRebuildReadError(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo     *info   = completion->parent;
Packit Service 310c69
  VDOPageCache *cache  = info->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  // We are doing a read-only rebuild, so treat this as a successful read
Packit Service 310c69
  // of an uninitialized page.
Packit Service 310c69
  relaxedAdd64(&cache->stats.failedReads, 1);
Packit Service 310c69
  memset(getPageBuffer(info), 0, VDO_BLOCK_SIZE);
Packit Service 310c69
  resetCompletion(completion);
Packit Service 310c69
  if (cache->readHook != NULL) {
Packit Service 310c69
    runReadHook(completion);
Packit Service 310c69
  } else {
Packit Service 310c69
    pageIsLoaded(completion);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Begin the process of loading a page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info  the page info representing where to load the page
Packit Service 310c69
 * @param pbn   the absolute pbn of the desired page
Packit Service 310c69
 *
Packit Service 310c69
 * @return VDO_SUCCESS or an error code
Packit Service 310c69
 **/
Packit Service 310c69
__attribute__((warn_unused_result))
Packit Service 310c69
static int launchPageLoad(PageInfo *info, PhysicalBlockNumber pbn)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
  assertIOAllowed(cache);
Packit Service 310c69
Packit Service 310c69
  int result = setInfoPBN(info, pbn);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = ASSERT((info->busy == 0), "Page is not busy before loading.");
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    return result;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  setInfoState(info, PS_INCOMING);
Packit Service 310c69
  cache->outstandingReads++;
Packit Service 310c69
  relaxedAdd64(&cache->stats.pagesLoaded, 1);
Packit Service 310c69
  launchReadMetadataVIO(info->vio, pbn,
Packit Service 310c69
                        (cache->readHook != NULL) ? runReadHook : pageIsLoaded,
Packit Service 310c69
                        (cache->rebuilding
Packit Service 310c69
                         ? handleRebuildReadError : handleLoadError));
Packit Service 310c69
  return VDO_SUCCESS;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
static void writePages(VDOCompletion *completion);
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Handle errors flushing the layer.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  The flush VIO
Packit Service 310c69
 **/
Packit Service 310c69
static void handleFlushError(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = ((PageInfo *) completion->parent)->cache;
Packit Service 310c69
  setPersistentError(cache, "flush failed", completion->result);
Packit Service 310c69
  writePages(completion);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Attempt to save the outgoing pages by first flushing the layer.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache  The cache
Packit Service 310c69
 **/
Packit Service 310c69
static void savePages(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  if ((cache->pagesInFlush > 0) || (cache->pagesToFlush == 0)) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  assertIOAllowed(cache);
Packit Service 310c69
Packit Service 310c69
  PageInfo *info      = pageInfoFromListNode(cache->outgoingList.next);
Packit Service 310c69
  cache->pagesInFlush = cache->pagesToFlush;
Packit Service 310c69
  cache->pagesToFlush = 0;
Packit Service 310c69
  relaxedAdd64(&cache->stats.flushCount, 1);
Packit Service 310c69
Packit Service 310c69
  VIO           *vio   = info->vio;
Packit Service 310c69
  PhysicalLayer *layer = vio->completion.layer;
Packit Service 310c69
Packit Service 310c69
  /*
Packit Service 310c69
   * We must make sure that the recovery journal entries that changed these
Packit Service 310c69
   * pages were successfully persisted, and thus must issue a flush before
Packit Service 310c69
   * each batch of pages is written to ensure this. However, in sync mode,
Packit Service 310c69
   * every journal block is written with FUA, thus guaranteeing the journal
Packit Service 310c69
   * persisted already.
Packit Service 310c69
   */
Packit Service 310c69
  if (layer->getWritePolicy(layer) != WRITE_POLICY_SYNC) {
Packit Service 310c69
    launchFlush(vio, writePages, handleFlushError);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  writePages(&vio->completion);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Add a page to the outgoing list of pages waiting to be saved. Once in the
Packit Service 310c69
 * list, a page may not be used until it has been written out.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info  The page to save
Packit Service 310c69
 **/
Packit Service 310c69
static void schedulePageSave(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  if (info->busy > 0) {
Packit Service 310c69
    info->writeStatus = WRITE_STATUS_DEFERRED;
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  info->cache->pagesToFlush++;
Packit Service 310c69
  info->cache->outstandingWrites++;
Packit Service 310c69
  setInfoState(info, PS_OUTGOING);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
static void writeDirtyPagesCallback(RingNode *expired, void *context)
Packit Service 310c69
{
Packit Service 310c69
  while (!isRingEmpty(expired)) {
Packit Service 310c69
    schedulePageSave(pageInfoFromListNode(chopRingNode(expired)));
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  savePages((VDOPageCache *) context);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Add a page to outgoing pages waiting to be saved, and then start saving
Packit Service 310c69
 * pages if another save is not in progress.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info  The page to save
Packit Service 310c69
 **/
Packit Service 310c69
static void launchPageSave(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  schedulePageSave(info);
Packit Service 310c69
  savePages(info->cache);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Determine whether a given VDOPageCompletion (as a waiter) is requesting a
Packit Service 310c69
 * given page number. Implements WaiterMatch.
Packit Service 310c69
 *
Packit Service 310c69
 * @param waiter        The page completion in question
Packit Service 310c69
 * @param context       A pointer to the pbn of the desired page
Packit Service 310c69
 *
Packit Service 310c69
 * @return true if the page completion is for the desired page number
Packit Service 310c69
 **/
Packit Service 310c69
static bool completionNeedsPage(Waiter *waiter, void *context)
Packit Service 310c69
{
Packit Service 310c69
  PhysicalBlockNumber *pbn = context;
Packit Service 310c69
  return (pageCompletionFromWaiter(waiter)->pbn == *pbn);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Allocate a free page to the first completion in the waiting queue,
Packit Service 310c69
 * and any other completions that match it in page number.
Packit Service 310c69
 **/
Packit Service 310c69
static void allocateFreePage(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  if (!hasWaiters(&cache->freeWaiters)) {
Packit Service 310c69
    if (relaxedLoad64(&cache->stats.cachePressure) > 0) {
Packit Service 310c69
      logInfo("page cache pressure relieved");
Packit Service 310c69
      relaxedStore64(&cache->stats.cachePressure, 0);
Packit Service 310c69
    }
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  int result = resetPageInfo(info);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    setPersistentError(cache, "cannot reset page info", result);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  Waiter *oldestWaiter = getFirstWaiter(&cache->freeWaiters);
Packit Service 310c69
  PhysicalBlockNumber pbn = pageCompletionFromWaiter(oldestWaiter)->pbn;
Packit Service 310c69
Packit Service 310c69
  // Remove all entries which match the page number in question
Packit Service 310c69
  // and push them onto the page info's wait queue.
Packit Service 310c69
  dequeueMatchingWaiters(&cache->freeWaiters, completionNeedsPage,
Packit Service 310c69
                         &pbn, &info->waiting);
Packit Service 310c69
  cache->waiterCount -= countWaiters(&info->waiting);
Packit Service 310c69
Packit Service 310c69
  result = launchPageLoad(info, pbn);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    distributeErrorOverQueue(result, &info->waiting);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Begin the process of discarding a page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 *
Packit Service 310c69
 * @note If no page is discardable, increments a count of deferred frees so
Packit Service 310c69
 *       that the next release of a page which is no longer busy will kick
Packit Service 310c69
 *       off another discard cycle. This is an indication that the cache is
Packit Service 310c69
 *       not big enough.
Packit Service 310c69
 *
Packit Service 310c69
 * @note If the selected page is not dirty, immediately allocates the page
Packit Service 310c69
 *       to the oldest completion waiting for a free page.
Packit Service 310c69
 **/
Packit Service 310c69
static void discardAPage(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo *info = selectLRUPage(cache);
Packit Service 310c69
  if (info == NULL) {
Packit Service 310c69
    reportCachePressure(cache);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (!isDirty(info)) {
Packit Service 310c69
    allocateFreePage(info);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  ASSERT_LOG_ONLY(!isInFlight(info),
Packit Service 310c69
                  "page selected for discard is not in flight");
Packit Service 310c69
Packit Service 310c69
  ++cache->discardCount;
Packit Service 310c69
  info->writeStatus = WRITE_STATUS_DISCARD;
Packit Service 310c69
  launchPageSave(info);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Helper used to trigger a discard so that the completion can get a different
Packit Service 310c69
 * page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param vdoPageComp   the VDO Page completion
Packit Service 310c69
 **/
Packit Service 310c69
static void discardPageForCompletion(VDOPageCompletion *vdoPageComp)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = vdoPageComp->cache;
Packit Service 310c69
Packit Service 310c69
  ++cache->waiterCount;
Packit Service 310c69
Packit Service 310c69
  int result = enqueueWaiter(&cache->freeWaiters, &vdoPageComp->waiter);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    setPersistentError(cache, "cannot enqueue waiter", result);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  discardAPage(cache);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Helper used to trigger a discard if the cache needs another free page.
Packit Service 310c69
 *
Packit Service 310c69
 * @param cache         the page cache
Packit Service 310c69
 **/
Packit Service 310c69
static void discardPageIfNeeded(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  if (cache->waiterCount > cache->discardCount) {
Packit Service 310c69
    discardAPage(cache);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void advanceVDOPageCachePeriod(VDOPageCache *cache, SequenceNumber period)
Packit Service 310c69
{
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
  advancePeriod(cache->dirtyLists, period);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Inform the cache that a write has finished (possibly with an error).
Packit Service 310c69
 *
Packit Service 310c69
 * @param info  The info structure for the page whose write just completed
Packit Service 310c69
 *
Packit Service 310c69
 * @return true if the page write was a discard
Packit Service 310c69
 **/
Packit Service 310c69
static bool writeHasFinished(PageInfo *info)
Packit Service 310c69
{
Packit Service 310c69
  assertOnCacheThread(info->cache, __func__);
Packit Service 310c69
  info->cache->outstandingWrites--;
Packit Service 310c69
Packit Service 310c69
  bool wasDiscard = (info->writeStatus == WRITE_STATUS_DISCARD);
Packit Service 310c69
  info->writeStatus = WRITE_STATUS_NORMAL;
Packit Service 310c69
  return wasDiscard;
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Handler for page write errors.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion  The page write VIO
Packit Service 310c69
 **/
Packit Service 310c69
static void handlePageWriteError(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  int           result = completion->result;
Packit Service 310c69
  PageInfo     *info   = completion->parent;
Packit Service 310c69
  VDOPageCache *cache  = info->cache;
Packit Service 310c69
Packit Service 310c69
  // If we're already read-only, write failures are to be expected.
Packit Service 310c69
  if (result != VDO_READ_ONLY) {
Packit Service 310c69
#if __KERNEL__
Packit Service 310c69
    static DEFINE_RATELIMIT_STATE(errorLimiter, DEFAULT_RATELIMIT_INTERVAL,
Packit Service 310c69
                                  DEFAULT_RATELIMIT_BURST);
Packit Service 310c69
Packit Service 310c69
    if (__ratelimit(&errorLimiter)) {
Packit Service 310c69
      logError("failed to write block map page %llu", info->pbn);
Packit Service 310c69
    }
Packit Service 310c69
#else
Packit Service 310c69
    logError("failed to write block map page %llu", info->pbn);
Packit Service 310c69
#endif
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  setInfoState(info, PS_DIRTY);
Packit Service 310c69
  relaxedAdd64(&cache->stats.failedWrites, 1);
Packit Service 310c69
  setPersistentError(cache, "cannot write page", result);
Packit Service 310c69
Packit Service 310c69
  if (!writeHasFinished(info)) {
Packit Service 310c69
    discardPageIfNeeded(cache);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  checkForDrainComplete(cache->zone);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * VIO callback used when a page has been written out.
Packit Service 310c69
 *
Packit Service 310c69
 * @param completion    A completion for the VIO, the parent of which
Packit Service 310c69
 *                      is embedded in PageInfo.
Packit Service 310c69
 **/
Packit Service 310c69
static void pageIsWrittenOut(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  PageInfo     *info  = completion->parent;
Packit Service 310c69
  VDOPageCache *cache = info->cache;
Packit Service 310c69
Packit Service 310c69
  if (cache->writeHook != NULL) {
Packit Service 310c69
    bool rewrite = cache->writeHook(getPageBuffer(info), cache->zone,
Packit Service 310c69
                                    info->context);
Packit Service 310c69
    if (rewrite) {
Packit Service 310c69
      launchWriteMetadataVIOWithFlush(info->vio, info->pbn, pageIsWrittenOut,
Packit Service 310c69
                                      handlePageWriteError, true, false);
Packit Service 310c69
      return;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  bool wasDiscard = writeHasFinished(info);
Packit Service 310c69
  bool reclaimed  = (!wasDiscard || (info->busy > 0)
Packit Service 310c69
                     || hasWaiters(&info->waiting));
Packit Service 310c69
Packit Service 310c69
  setInfoState(info, PS_RESIDENT);
Packit Service 310c69
Packit Service 310c69
  uint32_t reclamations = distributePageOverQueue(info, &info->waiting);
Packit Service 310c69
  relaxedAdd64(&cache->stats.reclaimed, reclamations);
Packit Service 310c69
Packit Service 310c69
  if (wasDiscard) {
Packit Service 310c69
    cache->discardCount--;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (reclaimed) {
Packit Service 310c69
    discardPageIfNeeded(cache);
Packit Service 310c69
  } else {
Packit Service 310c69
    allocateFreePage(info);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  checkForDrainComplete(cache->zone);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Write the batch of pages which were covered by the layer flush which just
Packit Service 310c69
 * completed. This callback is registered in savePages().
Packit Service 310c69
 *
Packit Service 310c69
 * @param flushCompletion  The flush VIO
Packit Service 310c69
 **/
Packit Service 310c69
static void writePages(VDOCompletion *flushCompletion)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCache *cache = ((PageInfo *) flushCompletion->parent)->cache;
Packit Service 310c69
Packit Service 310c69
  /*
Packit Service 310c69
   * We need to cache these two values on the stack since in the error case
Packit Service 310c69
   * below, it is possible for the last page info to cause the page cache to
Packit Service 310c69
   * get freed. Hence once we launch the last page, it may be unsafe to
Packit Service 310c69
   * dereference the cache [VDO-4724].
Packit Service 310c69
   */
Packit Service 310c69
  bool      hasUnflushedPages = (cache->pagesToFlush > 0);
Packit Service 310c69
  PageCount pagesInFlush      = cache->pagesInFlush;
Packit Service 310c69
  cache->pagesInFlush         = 0;
Packit Service 310c69
  while (pagesInFlush-- > 0) {
Packit Service 310c69
    PageInfo *info = pageInfoFromListNode(chopRingNode(&cache->outgoingList));
Packit Service 310c69
    if (isReadOnly(info->cache->zone->readOnlyNotifier)) {
Packit Service 310c69
      VDOCompletion *completion = &info->vio->completion;
Packit Service 310c69
      resetCompletion(completion);
Packit Service 310c69
      completion->callback     = pageIsWrittenOut;
Packit Service 310c69
      completion->errorHandler = handlePageWriteError;
Packit Service 310c69
      finishCompletion(completion, VDO_READ_ONLY);
Packit Service 310c69
      continue;
Packit Service 310c69
    }
Packit Service 310c69
    relaxedAdd64(&info->cache->stats.pagesSaved, 1);
Packit Service 310c69
    launchWriteMetadataVIO(info->vio, info->pbn, pageIsWrittenOut,
Packit Service 310c69
                           handlePageWriteError);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (hasUnflushedPages) {
Packit Service 310c69
    // If there are unflushed pages, the cache can't have been freed, so this
Packit Service 310c69
    // call is safe.
Packit Service 310c69
    savePages(cache);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void releaseVDOPageCompletion(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  if (completion == NULL) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  PageInfo *discardInfo = NULL;
Packit Service 310c69
  VDOPageCompletion *pageCompletion;
Packit Service 310c69
  if (completion->result == VDO_SUCCESS) {
Packit Service 310c69
    pageCompletion = validateCompletedPage(completion, false);
Packit Service 310c69
    if (--pageCompletion->info->busy == 0) {
Packit Service 310c69
      discardInfo = pageCompletion->info;
Packit Service 310c69
    }
Packit Service 310c69
  } else {
Packit Service 310c69
    // Do not check for errors if the completion was not successful.
Packit Service 310c69
    pageCompletion = asVDOPageCompletion(completion);
Packit Service 310c69
  }
Packit Service 310c69
  ASSERT_LOG_ONLY((pageCompletion->waiter.nextWaiter == NULL),
Packit Service 310c69
                  "Page being released after leaving all queues");
Packit Service 310c69
Packit Service 310c69
  VDOPageCache *cache = pageCompletion->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
  memset(pageCompletion, 0, sizeof(VDOPageCompletion));
Packit Service 310c69
Packit Service 310c69
  if (discardInfo != NULL) {
Packit Service 310c69
    if (discardInfo->writeStatus == WRITE_STATUS_DEFERRED) {
Packit Service 310c69
      discardInfo->writeStatus = WRITE_STATUS_NORMAL;
Packit Service 310c69
      launchPageSave(discardInfo);
Packit Service 310c69
    }
Packit Service 310c69
    // if there are excess requests for pages (that have not already started
Packit Service 310c69
    // discards) we need to discard some page (which may be this one)
Packit Service 310c69
    discardPageIfNeeded(cache);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**
Packit Service 310c69
 * Helper function to load a page as described by a VDO Page Completion.
Packit Service 310c69
 *
Packit Service 310c69
 * @param info          the page info representing where to load the page
Packit Service 310c69
 * @param vdoPageComp   the VDO Page Completion describing the page
Packit Service 310c69
 **/
Packit Service 310c69
static void loadPageForCompletion(PageInfo          *info,
Packit Service 310c69
                                  VDOPageCompletion *vdoPageComp)
Packit Service 310c69
{
Packit Service 310c69
  int result = enqueueWaiter(&info->waiting, &vdoPageComp->waiter);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    finishCompletion(&vdoPageComp->completion, result);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  result = launchPageLoad(info, vdoPageComp->pbn);
Packit Service 310c69
  if (result != VDO_SUCCESS) {
Packit Service 310c69
    distributeErrorOverQueue(result, &info->waiting);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void getVDOPageAsync(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCompletion *vdoPageComp = asVDOPageCompletion(completion);
Packit Service 310c69
  VDOPageCache      *cache       = vdoPageComp->cache;
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  if (vdoPageComp->writable && isReadOnly(cache->zone->readOnlyNotifier)) {
Packit Service 310c69
    finishCompletion(completion, VDO_READ_ONLY);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  if (vdoPageComp->writable) {
Packit Service 310c69
    relaxedAdd64(&cache->stats.writeCount, 1);
Packit Service 310c69
  } else {
Packit Service 310c69
    relaxedAdd64(&cache->stats.readCount, 1);
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  PageInfo *info = vpcFindPage(cache, vdoPageComp->pbn);
Packit Service 310c69
  if (info != NULL) {
Packit Service 310c69
    // The page is in the cache already.
Packit Service 310c69
    if ((info->writeStatus == WRITE_STATUS_DEFERRED) || isIncoming(info)
Packit Service 310c69
        || (isOutgoing(info) && vdoPageComp->writable)) {
Packit Service 310c69
      // The page is unusable until it has finished I/O.
Packit Service 310c69
      relaxedAdd64(&cache->stats.waitForPage, 1);
Packit Service 310c69
      int result = enqueueWaiter(&info->waiting, &vdoPageComp->waiter);
Packit Service 310c69
      if (result != VDO_SUCCESS) {
Packit Service 310c69
        finishCompletion(&vdoPageComp->completion, result);
Packit Service 310c69
      }
Packit Service 310c69
Packit Service 310c69
      return;
Packit Service 310c69
    }
Packit Service 310c69
Packit Service 310c69
    if (isValid(info)) {
Packit Service 310c69
      // The page is usable.
Packit Service 310c69
      relaxedAdd64(&cache->stats.foundInCache, 1);
Packit Service 310c69
      if (!isPresent(info)) {
Packit Service 310c69
        relaxedAdd64(&cache->stats.readOutgoing, 1);
Packit Service 310c69
      }
Packit Service 310c69
      updateLru(info);
Packit Service 310c69
      ++info->busy;
Packit Service 310c69
      completeWithPage(info, vdoPageComp);
Packit Service 310c69
      return;
Packit Service 310c69
    }
Packit Service 310c69
    // Something horrible has gone wrong.
Packit Service 310c69
    ASSERT_LOG_ONLY(false, "Info found in a usable state.");
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  // The page must be fetched.
Packit Service 310c69
  info = findFreePage(cache);
Packit Service 310c69
  if (info != NULL) {
Packit Service 310c69
    relaxedAdd64(&cache->stats.fetchRequired, 1);
Packit Service 310c69
    loadPageForCompletion(info, vdoPageComp);
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  // The page must wait for a page to be discarded.
Packit Service 310c69
  relaxedAdd64(&cache->stats.discardRequired, 1);
Packit Service 310c69
  discardPageForCompletion(vdoPageComp);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void markCompletedVDOPageDirty(VDOCompletion  *completion,
Packit Service 310c69
                               SequenceNumber  oldDirtyPeriod,
Packit Service 310c69
                               SequenceNumber  newDirtyPeriod)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCompletion *vdoPageComp = validateCompletedPage(completion, true);
Packit Service 310c69
  if (vdoPageComp == NULL) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  PageInfo *info = vdoPageComp->info;
Packit Service 310c69
  setInfoState(info, PS_DIRTY);
Packit Service 310c69
  addToDirtyLists(info->cache->dirtyLists, &info->listNode, oldDirtyPeriod,
Packit Service 310c69
                  newDirtyPeriod);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void requestVDOPageWrite(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCompletion *vdoPageComp = validateCompletedPage(completion, true);
Packit Service 310c69
  if (vdoPageComp == NULL) {
Packit Service 310c69
    return;
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  PageInfo *info = vdoPageComp->info;
Packit Service 310c69
  setInfoState(info, PS_DIRTY);
Packit Service 310c69
  launchPageSave(info);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
static void *dereferencePageCompletion(VDOPageCompletion  *completion)
Packit Service 310c69
{
Packit Service 310c69
  return ((completion != NULL) ? getPageBuffer(completion->info) : NULL);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
const void *dereferenceReadableVDOPage(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  return dereferencePageCompletion(validateCompletedPage(completion, false));
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void *dereferenceWritableVDOPage(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  return dereferencePageCompletion(validateCompletedPage(completion, true));
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void *getVDOPageCompletionContext(VDOCompletion *completion)
Packit Service 310c69
{
Packit Service 310c69
  VDOPageCompletion *pageCompletion = asVDOPageCompletion(completion);
Packit Service 310c69
  PageInfo *info = ((pageCompletion != NULL) ? pageCompletion->info : NULL);
Packit Service 310c69
  return (((info != NULL) && isValid(info)) ? info->context : NULL);
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
void drainVDOPageCache(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
  ASSERT_LOG_ONLY(isDraining(&cache->zone->state),
Packit Service 310c69
                  "drainVDOPageCache() called during block map drain");
Packit Service 310c69
Packit Service 310c69
  if (!isSuspending(&cache->zone->state)) {
Packit Service 310c69
    flushDirtyLists(cache->dirtyLists);
Packit Service 310c69
    savePages(cache);
Packit Service 310c69
  }
Packit Service 310c69
}
Packit Service 310c69
Packit Service 310c69
/**********************************************************************/
Packit Service 310c69
int invalidateVDOPageCache(VDOPageCache *cache)
Packit Service 310c69
{
Packit Service 310c69
  assertOnCacheThread(cache, __func__);
Packit Service 310c69
Packit Service 310c69
  // Make sure we don't throw away any dirty pages.
Packit Service 310c69
  PageInfo *info;
Packit Service 310c69
  for (info = cache->infos; info < cache->infos + cache->pageCount; info++) {
Packit Service 310c69
    int result = ASSERT(!isDirty(info), "cache must have no dirty pages");
Packit Service 310c69
    if (result != VDO_SUCCESS) {
Packit Service 310c69
      return result;
Packit Service 310c69
    }
Packit Service 310c69
  }
Packit Service 310c69
Packit Service 310c69
  // Reset the pageMap by re-allocating it.
Packit Service 310c69
  freeIntMap(&cache->pageMap);
Packit Service 310c69
  return makeIntMap(cache->pageCount, 0, &cache->pageMap);
Packit Service 310c69
}