|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Copyright (c) 2020 Red Hat, Inc.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is free software; you can redistribute it and/or
|
|
Packit Service |
310c69 |
* modify it under the terms of the GNU General Public License
|
|
Packit Service |
310c69 |
* as published by the Free Software Foundation; either version 2
|
|
Packit Service |
310c69 |
* of the License, or (at your option) any later version.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is distributed in the hope that it will be useful,
|
|
Packit Service |
310c69 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
310c69 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
Packit Service |
310c69 |
* GNU General Public License for more details.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* You should have received a copy of the GNU General Public License
|
|
Packit Service |
310c69 |
* along with this program; if not, write to the Free Software
|
|
Packit Service |
310c69 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
Packit Service |
310c69 |
* 02110-1301, USA.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/kvdoFlush.c#6 $
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "kvdoFlush.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "logger.h"
|
|
Packit Service |
310c69 |
#include "memoryAlloc.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "threadConfig.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "bio.h"
|
|
Packit Service |
310c69 |
#include "ioSubmitter.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* A specific (concrete) encapsulation of flush requests.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* We attempt to allocate a KVDOFlush objects for each incoming flush bio.
|
|
Packit Service |
310c69 |
* In case the allocate fails, a spare object is pre-allocated by and stored
|
|
Packit Service |
310c69 |
* in the kernel layer. The first time an allocation fails, the spare is used.
|
|
Packit Service |
310c69 |
* If another allocation fails while the spare is in use, it will merely be
|
|
Packit Service |
310c69 |
* queued for later processing.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* When a KVDOFlush is complete, it will either be freed, immediately
|
|
Packit Service |
310c69 |
* re-used for queued flushes, or stashed in the kernel layer as the new spare
|
|
Packit Service |
310c69 |
* object. This ensures that we will always make forward progress.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
struct kvdoFlush {
|
|
Packit Service |
310c69 |
KvdoWorkItem workItem;
|
|
Packit Service |
310c69 |
KernelLayer *layer;
|
|
Packit Service |
310c69 |
struct bio_list bios;
|
|
Packit Service |
310c69 |
Jiffies arrivalTime; // Time when earliest bio appeared
|
|
Packit Service |
310c69 |
VDOFlush vdoFlush;
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
int makeKVDOFlush(KVDOFlush **flushPtr)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return ALLOCATE(1, KVDOFlush, __func__, flushPtr);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
bool shouldProcessFlush(KernelLayer *layer)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return (getKVDOWritePolicy(&layer->kvdo) != WRITE_POLICY_SYNC);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Function call to handle an empty flush request from the request queue.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param item The work item representing the flush request
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void kvdoFlushWork(KvdoWorkItem *item)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
KVDOFlush *kvdoFlush = container_of(item, KVDOFlush, workItem);
|
|
Packit Service |
310c69 |
flush(kvdoFlush->layer->kvdo.vdo, &kvdoFlush->vdoFlush);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Initialize a KVDOFlush object, transferring all the bios in the kernel
|
|
Packit Service |
310c69 |
* layer's waitingFlushes list to it. The caller MUST already hold the layer's
|
|
Packit Service |
310c69 |
* flushLock.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param kvdoFlush The flush to initialize
|
|
Packit Service |
310c69 |
* @param layer The kernel layer on which the flushLock is held
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void initializeKVDOFlush(KVDOFlush *kvdoFlush, KernelLayer *layer)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
kvdoFlush->layer = layer;
|
|
Packit Service |
310c69 |
bio_list_init(&kvdoFlush->bios);
|
|
Packit Service |
310c69 |
bio_list_merge(&kvdoFlush->bios, &layer->waitingFlushes);
|
|
Packit Service |
310c69 |
bio_list_init(&layer->waitingFlushes);
|
|
Packit Service |
310c69 |
kvdoFlush->arrivalTime = layer->flushArrivalTime;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static void enqueueKVDOFlush(KVDOFlush *kvdoFlush)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
setupWorkItem(&kvdoFlush->workItem, kvdoFlushWork, NULL, REQ_Q_ACTION_FLUSH);
|
|
Packit Service |
310c69 |
KVDO *kvdo = &kvdoFlush->layer->kvdo;
|
|
Packit Service |
310c69 |
enqueueKVDOWork(kvdo, &kvdoFlush->workItem,
|
|
Packit Service |
310c69 |
getPackerZoneThread(getThreadConfig(kvdo->vdo)));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void launchKVDOFlush(KernelLayer *layer, BIO *bio)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
// Try to allocate a KVDOFlush to represent the flush request. If the
|
|
Packit Service |
310c69 |
// allocation fails, we'll deal with it later.
|
|
Packit Service |
310c69 |
KVDOFlush *kvdoFlush = ALLOCATE_NOWAIT(KVDOFlush, __func__);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
spin_lock(&layer->flushLock);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// We have a new bio to start. Add it to the list. If it becomes the
|
|
Packit Service |
310c69 |
// only entry on the list, record the time.
|
|
Packit Service |
310c69 |
if (bio_list_empty(&layer->waitingFlushes)) {
|
|
Packit Service |
310c69 |
layer->flushArrivalTime = jiffies;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
bio_list_add(&layer->waitingFlushes, bio);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (kvdoFlush == NULL) {
|
|
Packit Service |
310c69 |
// The KVDOFlush allocation failed. Try to use the spare KVDOFlush object.
|
|
Packit Service |
310c69 |
if (layer->spareKVDOFlush == NULL) {
|
|
Packit Service |
310c69 |
// The spare is already in use. This bio is on waitingFlushes and it
|
|
Packit Service |
310c69 |
// will be handled by a flush completion or by a bio that can allocate.
|
|
Packit Service |
310c69 |
spin_unlock(&layer->flushLock);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Take and use the spare KVDOFlush object.
|
|
Packit Service |
310c69 |
kvdoFlush = layer->spareKVDOFlush;
|
|
Packit Service |
310c69 |
layer->spareKVDOFlush = NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// We have flushes to start. Capture them in the KVDOFlush object.
|
|
Packit Service |
310c69 |
initializeKVDOFlush(kvdoFlush, layer);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
spin_unlock(&layer->flushLock);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Finish launching the flushes.
|
|
Packit Service |
310c69 |
enqueueKVDOFlush(kvdoFlush);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Release a KVDOFlush object that has completed its work. If there are any
|
|
Packit Service |
310c69 |
* pending flush requests whose KVDOFlush allocation failed, they will be
|
|
Packit Service |
310c69 |
* launched by immediately re-using the released KVDOFlush. If there is no
|
|
Packit Service |
310c69 |
* spare KVDOFlush, the released object will become the spare. Otherwise, the
|
|
Packit Service |
310c69 |
* KVDOFlush will be freed.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param kvdoFlush The completed flush object to re-use or free
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void releaseKVDOFlush(KVDOFlush *kvdoFlush)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
KernelLayer *layer = kvdoFlush->layer;
|
|
Packit Service |
310c69 |
bool relaunchFlush = false;
|
|
Packit Service |
310c69 |
bool freeFlush = false;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
spin_lock(&layer->flushLock);
|
|
Packit Service |
310c69 |
if (bio_list_empty(&layer->waitingFlushes)) {
|
|
Packit Service |
310c69 |
// Nothing needs to be started. Save one spare KVDOFlush object.
|
|
Packit Service |
310c69 |
if (layer->spareKVDOFlush == NULL) {
|
|
Packit Service |
310c69 |
// Make the new spare all zero, just like a newly allocated one.
|
|
Packit Service |
310c69 |
memset(kvdoFlush, 0, sizeof(*kvdoFlush));
|
|
Packit Service |
310c69 |
layer->spareKVDOFlush = kvdoFlush;
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
freeFlush = true;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
// We have flushes to start. Capture them in the KVDOFlush object.
|
|
Packit Service |
310c69 |
initializeKVDOFlush(kvdoFlush, layer);
|
|
Packit Service |
310c69 |
relaunchFlush = true;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
spin_unlock(&layer->flushLock);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (relaunchFlush) {
|
|
Packit Service |
310c69 |
// Finish launching the flushes.
|
|
Packit Service |
310c69 |
enqueueKVDOFlush(kvdoFlush);
|
|
Packit Service |
310c69 |
} else if (freeFlush) {
|
|
Packit Service |
310c69 |
FREE(kvdoFlush);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Function called to complete and free a flush request
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param item The flush-request work item
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void kvdoCompleteFlushWork(KvdoWorkItem *item)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
KVDOFlush *kvdoFlush = container_of(item, KVDOFlush, workItem);
|
|
Packit Service |
310c69 |
KernelLayer *layer = kvdoFlush->layer;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
BIO *bio;
|
|
Packit Service |
310c69 |
while ((bio = bio_list_pop(&kvdoFlush->bios)) != NULL) {
|
|
Packit Service |
310c69 |
// We're not acknowledging this bio now, but we'll never touch it
|
|
Packit Service |
310c69 |
// again, so this is the last chance to account for it.
|
|
Packit Service |
310c69 |
countBios(&layer->biosAcknowledged, bio);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Make sure the bio is a empty flush bio.
|
|
Packit Service |
310c69 |
prepareFlushBIO(bio, bio->bi_private, getKernelLayerBdev(layer),
|
|
Packit Service |
310c69 |
bio->bi_end_io);
|
|
Packit Service |
310c69 |
atomic64_inc(&layer->flushOut);
|
|
Packit Service |
310c69 |
generic_make_request(bio);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Release the KVDOFlush object, freeing it, re-using it as the spare, or
|
|
Packit Service |
310c69 |
// using it to launch any flushes that had to wait when allocations failed.
|
|
Packit Service |
310c69 |
releaseKVDOFlush(kvdoFlush);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void kvdoCompleteFlush(VDOFlush **kfp)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (*kfp != NULL) {
|
|
Packit Service |
310c69 |
KVDOFlush *kvdoFlush = container_of(*kfp, KVDOFlush, vdoFlush);
|
|
Packit Service |
310c69 |
setupWorkItem(&kvdoFlush->workItem, kvdoCompleteFlushWork, NULL,
|
|
Packit Service |
310c69 |
BIO_Q_ACTION_FLUSH);
|
|
Packit Service |
310c69 |
enqueueBioWorkItem(kvdoFlush->layer->ioSubmitter,
|
|
Packit Service |
310c69 |
&kvdoFlush->workItem);
|
|
Packit Service |
310c69 |
*kfp = NULL;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
int synchronousFlush(KernelLayer *layer)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
BIO bio;
|
|
Packit Service |
310c69 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
|
|
Packit Service |
310c69 |
bio_init(&bio, 0, 0);
|
|
Packit Service |
310c69 |
#else
|
|
Packit Service |
310c69 |
bio_init(&bio;;
|
|
Packit Service |
310c69 |
#endif
|
|
Packit Service |
310c69 |
int result = 0;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
prepareFlushBIO(&bio, layer, getKernelLayerBdev(layer), NULL);
|
|
Packit Service |
310c69 |
result = submitBioAndWait(&bio;;
|
|
Packit Service |
310c69 |
atomic64_inc(&layer->flushOut);
|
|
Packit Service |
310c69 |
if (result != 0) {
|
|
Packit Service |
310c69 |
logErrorWithStringError(result, "synchronous flush failed");
|
|
Packit Service |
310c69 |
result = -EIO;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
|
|
Packit Service |
310c69 |
bio_uninit(&bio;;
|
|
Packit Service |
310c69 |
#endif
|
|
Packit Service |
310c69 |
return result;
|
|
Packit Service |
310c69 |
}
|