|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Copyright (c) 2020 Red Hat, Inc.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is free software; you can redistribute it and/or
|
|
Packit Service |
310c69 |
* modify it under the terms of the GNU General Public License
|
|
Packit Service |
310c69 |
* as published by the Free Software Foundation; either version 2
|
|
Packit Service |
310c69 |
* of the License, or (at your option) any later version.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* This program is distributed in the hope that it will be useful,
|
|
Packit Service |
310c69 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
310c69 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
Packit Service |
310c69 |
* GNU General Public License for more details.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* You should have received a copy of the GNU General Public License
|
|
Packit Service |
310c69 |
* along with this program; if not, write to the Free Software
|
|
Packit Service |
310c69 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
Packit Service |
310c69 |
* 02110-1301, USA.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/base/vioWrite.c#9 $
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* This file contains almost all of the VDO write path, which begins with
|
|
Packit Service |
310c69 |
* writeExtent(). The progression through the callbacks which make up the
|
|
Packit Service |
310c69 |
* write path depends upon whether or not the write policy is synchronous or
|
|
Packit Service |
310c69 |
* asynchronous. The paths would proceed as outlined in the pseudo-code here
|
|
Packit Service |
310c69 |
* if this were normal, synchronous code without callbacks. Complications
|
|
Packit Service |
310c69 |
* involved in waiting on locks are not included.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* ######################################################################
|
|
Packit Service |
310c69 |
* writeExtentSynchronous(extent)
|
|
Packit Service |
310c69 |
* {
|
|
Packit Service |
310c69 |
* foreach (vio in extent) {
|
|
Packit Service |
310c69 |
* launchWriteVIO()
|
|
Packit Service |
310c69 |
* # allocateBlockForWrite()
|
|
Packit Service |
310c69 |
* if (!trim and !zero-block) {
|
|
Packit Service |
310c69 |
* allocate block
|
|
Packit Service |
310c69 |
* if (vio is compressed) {
|
|
Packit Service |
310c69 |
* completeCompressedBlockWrite()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* writeBlock()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* finishBlockWrite()
|
|
Packit Service |
310c69 |
* addJournalEntry() # Increment
|
|
Packit Service |
310c69 |
* if (vio->newMapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* journalIncrementForWrite()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* acknowledgeWriteCallback()
|
|
Packit Service |
310c69 |
* readOldBlockMapping()
|
|
Packit Service |
310c69 |
* journalUnmappingForWrite()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* journalDecrementForWrite()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForWrite()
|
|
Packit Service |
310c69 |
* if (trim || zero-block) {
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* prepareForDedupe()
|
|
Packit Service |
310c69 |
* hashData()
|
|
Packit Service |
310c69 |
* resolveHashZone()
|
|
Packit Service |
310c69 |
* acquireHashLock()
|
|
Packit Service |
310c69 |
* attemptDedupe() (query albireo)
|
|
Packit Service |
310c69 |
* if (isDuplicate) {
|
|
Packit Service |
310c69 |
* verifyAdvice() (read verify)
|
|
Packit Service |
310c69 |
* if (isDuplicate and canAddReference) {
|
|
Packit Service |
310c69 |
* shareBlock()
|
|
Packit Service |
310c69 |
* addJournalEntryForDedupe()
|
|
Packit Service |
310c69 |
* incrementForDedupe()
|
|
Packit Service |
310c69 |
* journalUnmappingForDedupe()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* decrementForDedupe()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForDedupe()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* if (not canAddReference) {
|
|
Packit Service |
310c69 |
* layer->updateAlbireo()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* # compressData()
|
|
Packit Service |
310c69 |
* if (compressing and not mooted and has no waiters) {
|
|
Packit Service |
310c69 |
* layer->compressVIO()
|
|
Packit Service |
310c69 |
* packCompressedData()
|
|
Packit Service |
310c69 |
* if (compressed) {
|
|
Packit Service |
310c69 |
* journalCompressedBlocks()
|
|
Packit Service |
310c69 |
* incrementForDedupe()
|
|
Packit Service |
310c69 |
* readOldBlockMappingForDedupe()
|
|
Packit Service |
310c69 |
* journalUnmappingForDedupe()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* decrementForDedupe()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForDedupe()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* ######################################################################
|
|
Packit Service |
310c69 |
* writeExtentAsynchronous(extent)
|
|
Packit Service |
310c69 |
* {
|
|
Packit Service |
310c69 |
* foreach (vio in extent) {
|
|
Packit Service |
310c69 |
* launchWriteVIO()
|
|
Packit Service |
310c69 |
* # allocateBlockForWrite()
|
|
Packit Service |
310c69 |
* if (trim || zero-block) {
|
|
Packit Service |
310c69 |
* acknowledgeWrite()
|
|
Packit Service |
310c69 |
* } else {
|
|
Packit Service |
310c69 |
* allocateAndLockBlock()
|
|
Packit Service |
310c69 |
* if (vio is compressed) {
|
|
Packit Service |
310c69 |
* writeBlock()
|
|
Packit Service |
310c69 |
* completeCompressedBlockWrite()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* acknowledgeWrite()
|
|
Packit Service |
310c69 |
* prepareForDedupe()
|
|
Packit Service |
310c69 |
* hashData()
|
|
Packit Service |
310c69 |
* resolveHashZone()
|
|
Packit Service |
310c69 |
* acquireHashLock()
|
|
Packit Service |
310c69 |
* attemptDedupe() (query albireo)
|
|
Packit Service |
310c69 |
* if (isDuplicate) {
|
|
Packit Service |
310c69 |
* verifyAdvice() (read verify)
|
|
Packit Service |
310c69 |
* if (isDuplicate and canAddReference) {
|
|
Packit Service |
310c69 |
* shareBlock()
|
|
Packit Service |
310c69 |
* addJournalEntryForDedupe()
|
|
Packit Service |
310c69 |
* incrementForDedupe()
|
|
Packit Service |
310c69 |
* readOldBlockMappingForDedupe()
|
|
Packit Service |
310c69 |
* journalUnmappingForDedupe()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* decrementForDedupe()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForDedupe()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* if (not canAddReference) {
|
|
Packit Service |
310c69 |
* layer->updateAlbireo()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* # compressData()
|
|
Packit Service |
310c69 |
* if (compressing and not mooted and has no waiters) {
|
|
Packit Service |
310c69 |
* layer->compressVIO()
|
|
Packit Service |
310c69 |
* packCompressedData()
|
|
Packit Service |
310c69 |
* if (compressed) {
|
|
Packit Service |
310c69 |
* journalCompressedBlocks()
|
|
Packit Service |
310c69 |
* journalIncrementForDedupe()
|
|
Packit Service |
310c69 |
* readOldBlockMappingForDedupe()
|
|
Packit Service |
310c69 |
* journalUnmappingForDedupe()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* decrementForDedupe()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForDedupe()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* return
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* writeBlock()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* finishBlockWrite()
|
|
Packit Service |
310c69 |
* addJournalEntry() # Increment
|
|
Packit Service |
310c69 |
* if (vio->newMapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* journalIncrementForWrite()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* readOldBlockMappingForWrite()
|
|
Packit Service |
310c69 |
* journalUnmappingForWrite()
|
|
Packit Service |
310c69 |
* if (vio->mapped is not ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
* journalDecrementForWrite()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* updateBlockMapForWrite()
|
|
Packit Service |
310c69 |
* finishVIO()
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
* }
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "vioWrite.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "logger.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
#include "allocatingVIO.h"
|
|
Packit Service |
310c69 |
#include "atomic.h"
|
|
Packit Service |
310c69 |
#include "blockMap.h"
|
|
Packit Service |
310c69 |
#include "compressionState.h"
|
|
Packit Service |
310c69 |
#include "dataVIO.h"
|
|
Packit Service |
310c69 |
#include "hashLock.h"
|
|
Packit Service |
310c69 |
#include "recoveryJournal.h"
|
|
Packit Service |
310c69 |
#include "referenceOperation.h"
|
|
Packit Service |
310c69 |
#include "slab.h"
|
|
Packit Service |
310c69 |
#include "slabDepot.h"
|
|
Packit Service |
310c69 |
#include "slabJournal.h"
|
|
Packit Service |
310c69 |
#include "vdoInternal.h"
|
|
Packit Service |
310c69 |
#include "vioRead.h"
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* The steps taken cleaning up a VIO, in the order they are performed.
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
typedef enum dataVIOCleanupStage {
|
|
Packit Service |
310c69 |
VIO_CLEANUP_START = 0,
|
|
Packit Service |
310c69 |
VIO_RELEASE_ALLOCATED = VIO_CLEANUP_START,
|
|
Packit Service |
310c69 |
VIO_RELEASE_RECOVERY_LOCKS,
|
|
Packit Service |
310c69 |
VIO_RELEASE_HASH_LOCK,
|
|
Packit Service |
310c69 |
VIO_RELEASE_LOGICAL,
|
|
Packit Service |
310c69 |
VIO_CLEANUP_DONE
|
|
Packit Service |
310c69 |
} DataVIOCleanupStage;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Actions to take on error used by abortOnError().
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
typedef enum {
|
|
Packit Service |
310c69 |
NOT_READ_ONLY,
|
|
Packit Service |
310c69 |
READ_ONLY_IF_ASYNC,
|
|
Packit Service |
310c69 |
READ_ONLY,
|
|
Packit Service |
310c69 |
} ReadOnlyAction;
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Forward declarations required because of circular function references.
|
|
Packit Service |
310c69 |
static void performCleanupStage(DataVIO *dataVIO, DataVIOCleanupStage stage);
|
|
Packit Service |
310c69 |
static void writeBlock(DataVIO *dataVIO);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Check whether we are in async mode.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO A DataVIO containing a pointer to the VDO whose write
|
|
Packit Service |
310c69 |
* policy we want to check
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return true if we are in async mode
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static inline bool isAsync(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return (getWritePolicy(getVDOFromDataVIO(dataVIO)) != WRITE_POLICY_SYNC);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Release the PBN lock and/or the reference on the allocated block at the
|
|
Packit Service |
310c69 |
* end of processing a DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void releaseAllocatedLock(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInAllocatedZone(dataVIO);
|
|
Packit Service |
310c69 |
releaseAllocationLock(dataVIOAsAllocatingVIO(dataVIO));
|
|
Packit Service |
310c69 |
performCleanupStage(dataVIO, VIO_RELEASE_RECOVERY_LOCKS);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Release the logical block lock and flush generation lock at the end of
|
|
Packit Service |
310c69 |
* processing a DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void releaseLogicalLock(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInLogicalZone(dataVIO);
|
|
Packit Service |
310c69 |
releaseLogicalBlockLock(dataVIO);
|
|
Packit Service |
310c69 |
releaseFlushGenerationLock(dataVIO);
|
|
Packit Service |
310c69 |
performCleanupStage(dataVIO, VIO_CLEANUP_DONE);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Release the hash lock at the end of processing a DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void cleanHashLock(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInHashZone(dataVIO);
|
|
Packit Service |
310c69 |
releaseHashLock(dataVIO);
|
|
Packit Service |
310c69 |
performCleanupStage(dataVIO, VIO_RELEASE_LOGICAL);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Make some assertions about a DataVIO which has finished cleaning up
|
|
Packit Service |
310c69 |
* and do its final callback.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO which has finished cleaning up
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void finishCleanup(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(dataVIOAsAllocatingVIO(dataVIO)->allocationLock == NULL,
|
|
Packit Service |
310c69 |
"complete DataVIO has no allocation lock");
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(dataVIO->hashLock == NULL,
|
|
Packit Service |
310c69 |
"complete DataVIO has no hash lock");
|
|
Packit Service |
310c69 |
vioDoneCallback(dataVIOAsCompletion(dataVIO));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Perform the next step in the process of cleaning up a DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO to clean up
|
|
Packit Service |
310c69 |
* @param stage The cleanup stage to perform
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void performCleanupStage(DataVIO *dataVIO, DataVIOCleanupStage stage)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
switch (stage) {
|
|
Packit Service |
310c69 |
case VIO_RELEASE_ALLOCATED:
|
|
Packit Service |
310c69 |
if (hasAllocation(dataVIO)) {
|
|
Packit Service |
310c69 |
launchAllocatedZoneCallback(dataVIO, releaseAllocatedLock,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=releaseAllocLock"));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// fall through
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case VIO_RELEASE_RECOVERY_LOCKS:
|
|
Packit Service |
310c69 |
if ((dataVIO->recoverySequenceNumber > 0)
|
|
Packit Service |
310c69 |
&& !isOrWillBeReadOnly(dataVIOAsVIO(dataVIO)->vdo->readOnlyNotifier)
|
|
Packit Service |
310c69 |
&& (dataVIOAsCompletion(dataVIO)->result != VDO_READ_ONLY)) {
|
|
Packit Service |
310c69 |
logWarning("VDO not read-only when cleaning DataVIO with RJ lock");
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// fall through
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case VIO_RELEASE_HASH_LOCK:
|
|
Packit Service |
310c69 |
if (dataVIO->hashLock != NULL) {
|
|
Packit Service |
310c69 |
launchHashZoneCallback(dataVIO, cleanHashLock,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=cleanHashLock"));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// fall through
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
case VIO_RELEASE_LOGICAL:
|
|
Packit Service |
310c69 |
if (!isCompressedWriteDataVIO(dataVIO)) {
|
|
Packit Service |
310c69 |
launchLogicalCallback(dataVIO, releaseLogicalLock,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=releaseLL"));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
// fall through
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
default:
|
|
Packit Service |
310c69 |
finishCleanup(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Return a DataVIO that encountered an error to its hash lock so it can
|
|
Packit Service |
310c69 |
* update the hash lock state accordingly. This continuation is registered in
|
|
Packit Service |
310c69 |
* abortOnError(), and must be called in the hash zone of the DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the DataVIO to return to its hash lock
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void finishWriteDataVIOWithError(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInHashZone(dataVIO);
|
|
Packit Service |
310c69 |
continueHashLockOnError(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Check whether a result is an error, and if so abort the DataVIO associated
|
|
Packit Service |
310c69 |
* with the error.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param result The result to check
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO
|
|
Packit Service |
310c69 |
* @param readOnlyAction The conditions under which the VDO should be put
|
|
Packit Service |
310c69 |
* into read-only mode if the result is an error
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @return true if the result is an error
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static bool abortOnError(int result,
|
|
Packit Service |
310c69 |
DataVIO *dataVIO,
|
|
Packit Service |
310c69 |
ReadOnlyAction readOnlyAction)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (result == VDO_SUCCESS) {
|
|
Packit Service |
310c69 |
return false;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if ((result == VDO_READ_ONLY)
|
|
Packit Service |
310c69 |
|| (readOnlyAction == READ_ONLY)
|
|
Packit Service |
310c69 |
|| ((readOnlyAction == READ_ONLY_IF_ASYNC) && isAsync(dataVIO))) {
|
|
Packit Service |
310c69 |
ReadOnlyNotifier *notifier = dataVIOAsVIO(dataVIO)->vdo->readOnlyNotifier;
|
|
Packit Service |
310c69 |
if (!isReadOnly(notifier)) {
|
|
Packit Service |
310c69 |
if (result != VDO_READ_ONLY) {
|
|
Packit Service |
310c69 |
logErrorWithStringError(result, "Preparing to enter read-only mode:"
|
|
Packit Service |
310c69 |
" DataVIO for LBN %llu (becoming mapped"
|
|
Packit Service |
310c69 |
" to %llu, previously mapped"
|
|
Packit Service |
310c69 |
" to %llu, allocated %llu) is"
|
|
Packit Service |
310c69 |
" completing with a fatal error after"
|
|
Packit Service |
310c69 |
" operation %s", dataVIO->logical.lbn,
|
|
Packit Service |
310c69 |
dataVIO->newMapped.pbn, dataVIO->mapped.pbn,
|
|
Packit Service |
310c69 |
getDataVIOAllocation(dataVIO),
|
|
Packit Service |
310c69 |
getOperationName(dataVIO));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
enterReadOnlyMode(notifier, result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->hashLock != NULL) {
|
|
Packit Service |
310c69 |
launchHashZoneCallback(dataVIO, finishWriteDataVIOWithError,
|
|
Packit Service |
310c69 |
THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
finishDataVIO(dataVIO, result);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
return true;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Return a DataVIO that finished writing, compressing, or deduplicating to
|
|
Packit Service |
310c69 |
* its hash lock so it can share the result with any DataVIOs waiting in the
|
|
Packit Service |
310c69 |
* hash lock, or update albireo, or simply release its share of the lock. This
|
|
Packit Service |
310c69 |
* continuation is registered in updateBlockMapForWrite(),
|
|
Packit Service |
310c69 |
* updateBlockMapForDedupe(), and abortDeduplication(), and must be called in
|
|
Packit Service |
310c69 |
* the hash zone of the DataVIO.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the DataVIO to return to its hash lock
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void finishWriteDataVIO(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInHashZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
continueHashLock(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Abort the data optimization process.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO which does not deduplicate or compress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void abortDeduplication(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (!hasAllocation(dataVIO)) {
|
|
Packit Service |
310c69 |
// There was no space to write this block and we failed to deduplicate
|
|
Packit Service |
310c69 |
// or compress it.
|
|
Packit Service |
310c69 |
finishDataVIO(dataVIO, VDO_NO_SPACE);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (isAsync(dataVIO)) {
|
|
Packit Service |
310c69 |
// We failed to deduplicate or compress an async DataVIO, so now we need
|
|
Packit Service |
310c69 |
// to actually write the data.
|
|
Packit Service |
310c69 |
writeBlock(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->hashLock == NULL) {
|
|
Packit Service |
310c69 |
// We failed to compress a synchronous DataVIO that is a hash collision,
|
|
Packit Service |
310c69 |
// which means it can't dedpe or be used for dedupe, so it's done now.
|
|
Packit Service |
310c69 |
finishDataVIO(dataVIO, VDO_SUCCESS);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* This synchronous DataVIO failed to compress and so is finished, but must
|
|
Packit Service |
310c69 |
* now return to its hash lock so other DataVIOs with the same data can
|
|
Packit Service |
310c69 |
* deduplicate against the uncompressed block it wrote.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
launchHashZoneCallback(dataVIO, finishWriteDataVIO, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Update the block map now that we've added an entry in the recovery journal
|
|
Packit Service |
310c69 |
* for a block we have just shared. This is the callback registered in
|
|
Packit Service |
310c69 |
* decrementForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void updateBlockMapForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInLogicalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->hashLock != NULL) {
|
|
Packit Service |
310c69 |
setHashZoneCallback(dataVIO, finishWriteDataVIO, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
completion->callback = completeDataVIO;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = PUT_MAPPED_BLOCK_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
putMappedBlockAsync(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Make a recovery journal increment.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO
|
|
Packit Service |
310c69 |
* @param lock The PBNLock on the block being incremented
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void journalIncrement(DataVIO *dataVIO, PBNLock *lock)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
setUpReferenceOperationWithLock(DATA_INCREMENT, dataVIO->newMapped.pbn,
|
|
Packit Service |
310c69 |
dataVIO->newMapped.state, lock,
|
|
Packit Service |
310c69 |
&dataVIO->operation);
|
|
Packit Service |
310c69 |
addRecoveryJournalEntry(getVDOFromDataVIO(dataVIO)->recoveryJournal,
|
|
Packit Service |
310c69 |
dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Make a recovery journal decrement entry.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void journalDecrement(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
setUpReferenceOperationWithZone(DATA_DECREMENT, dataVIO->mapped.pbn,
|
|
Packit Service |
310c69 |
dataVIO->mapped.state, dataVIO->mapped.zone,
|
|
Packit Service |
310c69 |
&dataVIO->operation);
|
|
Packit Service |
310c69 |
addRecoveryJournalEntry(getVDOFromDataVIO(dataVIO)->recoveryJournal,
|
|
Packit Service |
310c69 |
dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Make a reference count change.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void updateReferenceCount(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
SlabDepot *depot = getVDOFromDataVIO(dataVIO)->depot;
|
|
Packit Service |
310c69 |
PhysicalBlockNumber pbn = dataVIO->operation.pbn;
|
|
Packit Service |
310c69 |
int result = ASSERT(isPhysicalDataBlock(depot, pbn),
|
|
Packit Service |
310c69 |
"Adding slab journal entry for impossible PBN %" PRIu64
|
|
Packit Service |
310c69 |
"for LBN %llu", pbn, dataVIO->logical.lbn);
|
|
Packit Service |
310c69 |
if (abortOnError(result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
addSlabJournalEntry(getSlabJournal(depot, pbn), dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Do the decref after a successful dedupe or compression. This is the callback
|
|
Packit Service |
310c69 |
* registered by journalUnmappingForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void decrementForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInMappedZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
AllocatingVIO *allocatingVIO = dataVIOAsAllocatingVIO(dataVIO);
|
|
Packit Service |
310c69 |
if (allocatingVIO->allocation == dataVIO->mapped.pbn) {
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* If we are about to release the reference on the allocated block,
|
|
Packit Service |
310c69 |
* we must release the PBN lock on it first so that the allocator will
|
|
Packit Service |
310c69 |
* not allocate a write-locked block.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
releaseAllocationLock(allocatingVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, updateBlockMapForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;js=dec"));
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_DECREMENT_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
updateReferenceCount(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Write the appropriate journal entry for removing the mapping of logical to
|
|
Packit Service |
310c69 |
* mapped, for dedupe or compression. This is the callback registered in
|
|
Packit Service |
310c69 |
* readOldBlockMappingForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void journalUnmappingForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInJournalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->mapped.pbn == ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, updateBlockMapForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;j=dedupe;js=unmap;cb=updateBM"));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
setMappedZoneCallback(dataVIO, decrementForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;j=dedupe;js=unmap;cb=decDedupe"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_UNMAPPING_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
journalDecrement(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Get the previous PBN mapped to this LBN from the block map, so as to make
|
|
Packit Service |
310c69 |
* an appropriate journal entry referencing the removal of this LBN->PBN
|
|
Packit Service |
310c69 |
* mapping, for dedupe or compression. This callback is registered in
|
|
Packit Service |
310c69 |
* incrementForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void readOldBlockMappingForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInLogicalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = GET_MAPPED_BLOCK_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, journalUnmappingForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=journalUnmapDedupe"));
|
|
Packit Service |
310c69 |
getMappedBlockAsync(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Do the incref after compression. This is the callback registered by
|
|
Packit Service |
310c69 |
* addRecoveryJournalEntryForCompression().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void incrementForCompression(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInNewMappedZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(isCompressed(dataVIO->newMapped.state),
|
|
Packit Service |
310c69 |
"Impossible attempt to update reference counts for a block "
|
|
Packit Service |
310c69 |
"which was not compressed (logical block %llu)",
|
|
Packit Service |
310c69 |
dataVIO->logical.lbn);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* If we are synchronous and allocated a block, we know the one we
|
|
Packit Service |
310c69 |
* allocated is the block we need to decrement, so there is no need
|
|
Packit Service |
310c69 |
* to look in the block map.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
if (isAsync(dataVIO) || !hasAllocation(dataVIO)) {
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, readOldBlockMappingForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=readOldBlockMappingForDedupe"));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, journalUnmappingForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=journalUnmappingForDedupe"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_INCREMENT_FOR_COMPRESSION;
|
|
Packit Service |
310c69 |
updateReferenceCount(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Add a recovery journal entry for the increment resulting from compression.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO which has been compressed
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void addRecoveryJournalEntryForCompression(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInJournalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!isCompressed(dataVIO->newMapped.state)) {
|
|
Packit Service |
310c69 |
abortDeduplication(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setNewMappedZoneCallback(dataVIO, incrementForCompression,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F($dup);js=map/$dup;"
|
|
Packit Service |
310c69 |
"cb=incCompress($dup)"));
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_MAPPING_FOR_COMPRESSION;
|
|
Packit Service |
310c69 |
journalIncrement(dataVIO, getDuplicateLock(dataVIO));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Attempt to pack the compressed DataVIO into a block. This is the callback
|
|
Packit Service |
310c69 |
* registered in compressData().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of a compressed DataVIO
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void packCompressedData(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInPackerZone(dataVIO);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// XXX this is a callback, so there should probably be an error check here
|
|
Packit Service |
310c69 |
// even if we think compression can't currently return one.
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!mayPackDataVIO(dataVIO)) {
|
|
Packit Service |
310c69 |
abortDeduplication(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, addRecoveryJournalEntryForCompression,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=update(compress)"));
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = PACK_COMPRESSED_BLOCK;
|
|
Packit Service |
310c69 |
attemptPacking(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void compressData(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(!dataVIO->isDuplicate,
|
|
Packit Service |
310c69 |
"compressing a non-duplicate block");
|
|
Packit Service |
310c69 |
if (!mayCompressDataVIO(dataVIO)) {
|
|
Packit Service |
310c69 |
abortDeduplication(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = COMPRESS_DATA;
|
|
Packit Service |
310c69 |
setPackerCallback(dataVIO, packCompressedData, THIS_LOCATION("$F;cb=pack"));
|
|
Packit Service |
310c69 |
dataVIOAsCompletion(dataVIO)->layer->compressDataVIO(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Do the incref after deduplication. This is the callback registered by
|
|
Packit Service |
310c69 |
* addRecoveryJournalEntryForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void incrementForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInNewMappedZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(dataVIO->isDuplicate,
|
|
Packit Service |
310c69 |
"Impossible attempt to update reference counts for a block "
|
|
Packit Service |
310c69 |
"which was not a duplicate (logical block %llu)",
|
|
Packit Service |
310c69 |
dataVIO->logical.lbn);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* If we are synchronous and allocated a block, we know the one we
|
|
Packit Service |
310c69 |
* allocated is the block we need to decrement, so there is no need
|
|
Packit Service |
310c69 |
* to look in the block map.
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
if (isAsync(dataVIO) || !hasAllocation(dataVIO)) {
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, readOldBlockMappingForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=readOldBlockMappingForDedupe"));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, journalUnmappingForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=journalUnmappingForDedupe"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_INCREMENT_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
updateReferenceCount(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Add a recovery journal entry for the increment resulting from deduplication.
|
|
Packit Service |
310c69 |
* This callback is registered in shareBlock().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO which has been deduplicated
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void addRecoveryJournalEntryForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInJournalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setNewMappedZoneCallback(dataVIO, incrementForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F($dup);js=map/$dup;"
|
|
Packit Service |
310c69 |
"cb=incDedupe($dup)"));
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_MAPPING_FOR_DEDUPE;
|
|
Packit Service |
310c69 |
journalIncrement(dataVIO, getDuplicateLock(dataVIO));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Share a block in the block map if it is a duplicate. This is the lock
|
|
Packit Service |
310c69 |
* callback registered in acquirePBNReadLock(). This is only public so
|
|
Packit Service |
310c69 |
* test code can compare the function to the current callback in a completion.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
void shareBlock(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInDuplicateZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!dataVIO->isDuplicate) {
|
|
Packit Service |
310c69 |
compressData(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->newMapped = dataVIO->duplicate;
|
|
Packit Service |
310c69 |
launchJournalCallback(dataVIO, addRecoveryJournalEntryForDedupe,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=addJournalEntryDup"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Route the DataVIO to the HashZone responsible for the chunk name to acquire
|
|
Packit Service |
310c69 |
* a hash lock on that name, or join with a existing hash lock managing
|
|
Packit Service |
310c69 |
* concurrent dedupe for that name. This is the callback registered in
|
|
Packit Service |
310c69 |
* resolveHashZone().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO to lock
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void lockHashInZone(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInHashZone(dataVIO);
|
|
Packit Service |
310c69 |
// Shouldn't have had any errors since all we did was switch threads.
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
int result = acquireHashLock(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->hashLock == NULL) {
|
|
Packit Service |
310c69 |
// It's extremely unlikely, but in the case of a hash collision, the
|
|
Packit Service |
310c69 |
// DataVIO will not obtain a reference to the lock and cannot deduplicate.
|
|
Packit Service |
310c69 |
compressData(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
enterHashLock(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Set the hash zone (and flag the chunk name as set) while still on the
|
|
Packit Service |
310c69 |
* thread that just hashed the data to set the chunk name. This is the
|
|
Packit Service |
310c69 |
* callback registered by prepareForDedupe().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO whose chunk name was just generated, as a
|
|
Packit Service |
310c69 |
* completion
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void resolveHashZone(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
// We don't care what thread we are on.
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(!dataVIO->isZeroBlock, "zero blocks should not be hashed");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->hashZone
|
|
Packit Service |
310c69 |
= selectHashZone(getVDOFromDataVIO(dataVIO), &dataVIO->chunkName);
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = ACQUIRE_HASH_LOCK;
|
|
Packit Service |
310c69 |
launchHashZoneCallback(dataVIO, lockHashInZone, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Prepare for the dedupe path after a synchronous write or an asynchronous
|
|
Packit Service |
310c69 |
* allocation. This callback is registered in updateBlockMapForWrite() for
|
|
Packit Service |
310c69 |
* sync, and continueWriteAfterAllocation() (via acknowledgeWrite()) for
|
|
Packit Service |
310c69 |
* async. It is also called directly from the latter when allocation fails.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void prepareForDedupe(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
// We don't care what thread we are on
|
|
Packit Service |
310c69 |
dataVIOAddTraceRecord(dataVIO, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!isAsync(dataVIO)) {
|
|
Packit Service |
310c69 |
// Remember which block we wrote so we will decrement the reference to it
|
|
Packit Service |
310c69 |
// if we deduplicate. This avoids having to look it up in the block map.
|
|
Packit Service |
310c69 |
dataVIO->mapped = dataVIO->newMapped;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(!dataVIO->isZeroBlock,
|
|
Packit Service |
310c69 |
"must not prepare to dedupe zero blocks");
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Before we can dedupe, we need to know the chunk name, so the first step
|
|
Packit Service |
310c69 |
// is to hash the block data.
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = HASH_DATA;
|
|
Packit Service |
310c69 |
// XXX this is the wrong thread to run this callback, but we don't yet have
|
|
Packit Service |
310c69 |
// a mechanism for running it on the CPU thread immediately after hashing.
|
|
Packit Service |
310c69 |
setAllocatedZoneCallback(dataVIO, resolveHashZone, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
completion->layer->hashData(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Update the block map after a data write (or directly for a ZERO_BLOCK write
|
|
Packit Service |
310c69 |
* or trim). This callback is registered in decrementForWrite() and
|
|
Packit Service |
310c69 |
* journalUnmappingForWrite().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void updateBlockMapForWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInLogicalZone(dataVIO);
|
|
Packit Service |
310c69 |
dataVIOAddTraceRecord(dataVIO, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->isZeroBlock || isTrimDataVIO(dataVIO)) {
|
|
Packit Service |
310c69 |
completion->callback = completeDataVIO;
|
|
Packit Service |
310c69 |
} else if (!isAsync(dataVIO)) {
|
|
Packit Service |
310c69 |
// Synchronous DataVIOs branch off to the hash/dedupe path after finishing
|
|
Packit Service |
310c69 |
// the uncompressed write of their data.
|
|
Packit Service |
310c69 |
completion->callback = prepareForDedupe;
|
|
Packit Service |
310c69 |
} else if (dataVIO->hashLock != NULL) {
|
|
Packit Service |
310c69 |
// Async writes will be finished, but must return to the hash lock to
|
|
Packit Service |
310c69 |
// allow other DataVIOs with the same data to dedupe against the write.
|
|
Packit Service |
310c69 |
setHashZoneCallback(dataVIO, finishWriteDataVIO, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
// Async writes without a hash lock (hash collisions) will be finished.
|
|
Packit Service |
310c69 |
completion->callback = completeDataVIO;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = PUT_MAPPED_BLOCK;
|
|
Packit Service |
310c69 |
putMappedBlockAsync(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Do the decref after a successful block write. This is the callback
|
|
Packit Service |
310c69 |
* by journalUnmappingForWrite() if the old mapping was not the zero block.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void decrementForWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInMappedZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_DECREMENT_FOR_WRITE;
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, updateBlockMapForWrite, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
updateReferenceCount(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Write the appropriate journal entry for unmapping logical to mapped for a
|
|
Packit Service |
310c69 |
* write. This is the callback registered in readOldBlockMappingForWrite().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void journalUnmappingForWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInJournalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->mapped.pbn == ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, updateBlockMapForWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;js=unmap;cb=updateBMwrite"));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
setMappedZoneCallback(dataVIO, decrementForWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;js=unmap;cb=decWrite"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_UNMAPPING_FOR_WRITE;
|
|
Packit Service |
310c69 |
journalDecrement(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Get the previous PBN mapped to this LBN from the block map for a write, so
|
|
Packit Service |
310c69 |
* as to make an appropriate journal entry referencing the removal of this
|
|
Packit Service |
310c69 |
* LBN->PBN mapping. This callback is registered in finishBlockWrite() in the
|
|
Packit Service |
310c69 |
* async path, and is registered in acknowledgeWrite() in the sync path.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void readOldBlockMappingForWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInLogicalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, journalUnmappingForWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=journalUnmapWrite"));
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = GET_MAPPED_BLOCK_FOR_WRITE;
|
|
Packit Service |
310c69 |
getMappedBlockAsync(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Acknowledge a write to the requestor.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO being acknowledged
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void acknowledgeWrite(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
ASSERT_LOG_ONLY(dataVIO->hasFlushGenerationLock,
|
|
Packit Service |
310c69 |
"write VIO to be acknowledged has a flush generation lock");
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = ACKNOWLEDGE_WRITE;
|
|
Packit Service |
310c69 |
dataVIOAsCompletion(dataVIO)->layer->acknowledgeDataVIO(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Acknowledge a write now that we have made an entry in the recovery
|
|
Packit Service |
310c69 |
* journal. This is the callback registered in finishBlockWrite() in
|
|
Packit Service |
310c69 |
* synchronous mode.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void acknowledgeWriteCallback(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, readOldBlockMappingForWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
acknowledgeWrite(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
static VDOAction *getWriteIncrementCallback(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
return (isAsync(dataVIO)
|
|
Packit Service |
310c69 |
? readOldBlockMappingForWrite : acknowledgeWriteCallback);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Do the incref after a successful block write. This is the callback
|
|
Packit Service |
310c69 |
* registered by finishBlockWrite().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void incrementForWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInAllocatedZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/*
|
|
Packit Service |
310c69 |
* Now that the data has been written, it's safe to deduplicate against the
|
|
Packit Service |
310c69 |
* block. Downgrade the allocation lock to a read lock so it can be used
|
|
Packit Service |
310c69 |
* later by the hash lock (which we don't have yet in sync mode).
|
|
Packit Service |
310c69 |
*/
|
|
Packit Service |
310c69 |
downgradePBNWriteLock(dataVIOAsAllocatingVIO(dataVIO)->allocationLock);
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_INCREMENT_FOR_WRITE;
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, getWriteIncrementCallback(dataVIO),
|
|
Packit Service |
310c69 |
THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
updateReferenceCount(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Add an entry in the recovery journal after a successful block write. This is
|
|
Packit Service |
310c69 |
* the callback registered by writeBlock(). It is also registered in
|
|
Packit Service |
310c69 |
* allocateBlockForWrite().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The completion of the write in progress
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void finishBlockWrite(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
assertInJournalZone(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, READ_ONLY_IF_ASYNC)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->newMapped.pbn == ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
setLogicalCallback(dataVIO, getWriteIncrementCallback(dataVIO),
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;js=writeZero"));
|
|
Packit Service |
310c69 |
} else {
|
|
Packit Service |
310c69 |
setAllocatedZoneCallback(dataVIO, incrementForWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;js=mapWrite"));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = JOURNAL_MAPPING_FOR_WRITE;
|
|
Packit Service |
310c69 |
journalIncrement(dataVIO, dataVIOAsAllocatingVIO(dataVIO)->allocationLock);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Write data to the underlying storage.
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param dataVIO The DataVIO to write
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void writeBlock(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = WRITE_DATA;
|
|
Packit Service |
310c69 |
setJournalCallback(dataVIO, finishBlockWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F(data);cb=finishWrite"));
|
|
Packit Service |
310c69 |
dataVIOAsCompletion(dataVIO)->layer->writeData(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Continue the write path for a DataVIO now that block allocation is complete
|
|
Packit Service |
310c69 |
* (the DataVIO may or may not have actually received an allocation). This
|
|
Packit Service |
310c69 |
* callback is registered in continueWriteWithBlockMapSlot().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param allocatingVIO The DataVIO which has finished the allocation process
|
|
Packit Service |
310c69 |
* (as an AllocatingVIO)
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void continueWriteAfterAllocation(AllocatingVIO *allocatingVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = allocatingVIOAsDataVIO(allocatingVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(dataVIOAsCompletion(dataVIO)->result, dataVIO,
|
|
Packit Service |
310c69 |
NOT_READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!hasAllocation(dataVIO)) {
|
|
Packit Service |
310c69 |
prepareForDedupe(dataVIOAsCompletion(dataVIO));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
atomicStoreBool(&dataVIO->hasAllocation, true);
|
|
Packit Service |
310c69 |
dataVIO->newMapped = (ZonedPBN) {
|
|
Packit Service |
310c69 |
.zone = allocatingVIO->zone,
|
|
Packit Service |
310c69 |
.pbn = allocatingVIO->allocation,
|
|
Packit Service |
310c69 |
.state = MAPPING_STATE_UNCOMPRESSED,
|
|
Packit Service |
310c69 |
};
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (!isAsync(dataVIO)) {
|
|
Packit Service |
310c69 |
writeBlock(dataVIO);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// XXX prepareForDedupe can run from any thread, so this is a place where
|
|
Packit Service |
310c69 |
// running the callback on the kernel thread would save a thread switch.
|
|
Packit Service |
310c69 |
setAllocatedZoneCallback(dataVIO, prepareForDedupe, THIS_LOCATION(NULL));
|
|
Packit Service |
310c69 |
if (vioRequiresFlushAfter(allocatingVIOAsVIO(allocatingVIO))) {
|
|
Packit Service |
310c69 |
invokeCallback(dataVIOAsCompletion(dataVIO));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
acknowledgeWrite(dataVIO);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**
|
|
Packit Service |
310c69 |
* Continue the write path for a VIO now that block map slot resolution is
|
|
Packit Service |
310c69 |
* complete. This callback is registered in launchWriteDataVIO().
|
|
Packit Service |
310c69 |
*
|
|
Packit Service |
310c69 |
* @param completion The DataVIO to write
|
|
Packit Service |
310c69 |
**/
|
|
Packit Service |
310c69 |
static void continueWriteWithBlockMapSlot(VDOCompletion *completion)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
DataVIO *dataVIO = asDataVIO(completion);
|
|
Packit Service |
310c69 |
// We don't care what thread we're on.
|
|
Packit Service |
310c69 |
if (abortOnError(completion->result, dataVIO, NOT_READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->treeLock.treeSlots[0].blockMapSlot.pbn == ZERO_BLOCK) {
|
|
Packit Service |
310c69 |
int result = ASSERT(isTrimDataVIO(dataVIO),
|
|
Packit Service |
310c69 |
"dataVIO with no block map page is a trim");
|
|
Packit Service |
310c69 |
if (abortOnError(result, dataVIO, READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// This is a trim for a block on a block map page which has not been
|
|
Packit Service |
310c69 |
// allocated, so there's nothing more we need to do.
|
|
Packit Service |
310c69 |
finishDataVIO(dataVIO, VDO_SUCCESS);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
if (dataVIO->isZeroBlock || isTrimDataVIO(dataVIO)) {
|
|
Packit Service |
310c69 |
// We don't need to write any data, so skip allocation and just update
|
|
Packit Service |
310c69 |
// the block map and reference counts (via the journal).
|
|
Packit Service |
310c69 |
dataVIO->newMapped.pbn = ZERO_BLOCK;
|
|
Packit Service |
310c69 |
launchJournalCallback(dataVIO, finishBlockWrite,
|
|
Packit Service |
310c69 |
THIS_LOCATION("$F;cb=finishWrite"));
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
allocateDataBlock(dataVIOAsAllocatingVIO(dataVIO),
|
|
Packit Service |
310c69 |
getAllocationSelector(dataVIO->logical.zone),
|
|
Packit Service |
310c69 |
VIO_WRITE_LOCK, continueWriteAfterAllocation);
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void launchWriteDataVIO(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
if (isReadOnly(dataVIOAsVIO(dataVIO)->vdo->readOnlyNotifier)) {
|
|
Packit Service |
310c69 |
finishDataVIO(dataVIO, VDO_READ_ONLY);
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Write requests join the current flush generation.
|
|
Packit Service |
310c69 |
int result = acquireFlushGenerationLock(dataVIO);
|
|
Packit Service |
310c69 |
if (abortOnError(result, dataVIO, NOT_READ_ONLY)) {
|
|
Packit Service |
310c69 |
return;
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
// Go find the block map slot for the LBN mapping.
|
|
Packit Service |
310c69 |
dataVIO->lastAsyncOperation = FIND_BLOCK_MAP_SLOT;
|
|
Packit Service |
310c69 |
findBlockMapSlotAsync(dataVIO, continueWriteWithBlockMapSlot,
|
|
Packit Service |
310c69 |
getLogicalZoneThreadID(dataVIO->logical.zone));
|
|
Packit Service |
310c69 |
}
|
|
Packit Service |
310c69 |
|
|
Packit Service |
310c69 |
/**********************************************************************/
|
|
Packit Service |
310c69 |
void cleanupWriteDataVIO(DataVIO *dataVIO)
|
|
Packit Service |
310c69 |
{
|
|
Packit Service |
310c69 |
performCleanupStage(dataVIO, VIO_CLEANUP_START);
|
|
Packit Service |
310c69 |
}
|