/*
* Copyright (c) 2020 Red Hat, Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*
* $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/bio.c#8 $
*/
#include "bio.h"
#include "logger.h"
#include "memoryAlloc.h"
#include "numeric.h"
#include "flush.h"
#include "recoveryJournal.h"
#include "bioIterator.h"
#include "ioSubmitter.h"
/**
* Gets the raw buffer from a biovec.
*
* @param biovec The biovec in question
*
* @return the buffer
**/
static char *getBufferForBiovec(struct bio_vec *biovec)
{
return (page_address(biovec->bv_page) + biovec->bv_offset);
}
/**********************************************************************/
void bioCopyDataIn(BIO *bio, char *dataPtr)
{
struct bio_vec *biovec;
for (BioIterator iter = createBioIterator(bio);
(biovec = getNextBiovec(&iter)) != NULL;
advanceBioIterator(&iter)) {
memcpy(dataPtr, getBufferForBiovec(biovec), biovec->bv_len);
dataPtr += biovec->bv_len;
}
}
/**********************************************************************/
void bioCopyDataOut(BIO *bio, char *dataPtr)
{
struct bio_vec *biovec;
for (BioIterator iter = createBioIterator(bio);
(biovec = getNextBiovec(&iter)) != NULL;
advanceBioIterator(&iter)) {
memcpy(getBufferForBiovec(biovec), dataPtr, biovec->bv_len);
flush_dcache_page(biovec->bv_page);
dataPtr += biovec->bv_len;
}
}
/**********************************************************************/
void setBioOperation(BIO *bio, unsigned int operation)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
bio->bi_opf &= ~REQ_OP_MASK;
bio->bi_opf |= operation;
#else
unsigned int OPERATION_MASK = WRITE | REQ_DISCARD | REQ_FLUSH;
// Clear the relevant bits
bio->bi_rw &= ~OPERATION_MASK;
// Set the operation we care about
bio->bi_rw |= operation;
#endif
}
/**********************************************************************/
void freeBio(BIO *bio, KernelLayer *layer)
{
bio_put(bio);
}
/**********************************************************************/
void countBios(AtomicBioStats *bioStats, BIO *bio)
{
if (isWriteBio(bio)) {
atomic64_inc(&bioStats->write);
} else {
atomic64_inc(&bioStats->read);
}
if (isDiscardBio(bio)) {
atomic64_inc(&bioStats->discard);
}
if (isFlushBio(bio)) {
atomic64_inc(&bioStats->flush);
}
if (isFUABio(bio)) {
atomic64_inc(&bioStats->fua);
}
}
/**
* The function determines whether a buffer contains all zeroes.
*
* @param buffer The buffer to check
* @param length The length of the buffer
*
* @return true is all zeroes, false otherwise
**/
static inline bool isAllZeros(const char *buffer, unsigned int length)
{
/*
* Handle expected common case of even the first word being nonzero,
* without getting into the more expensive (for one iteration) loop
* below.
*/
if (likely(length >= sizeof(uint64_t))) {
if (GET_UNALIGNED(uint64_t, buffer) != 0) {
return false;
}
unsigned int wordCount = length / sizeof(uint64_t);
// Unroll to process 64 bytes at a time
unsigned int chunkCount = wordCount / 8;
while (chunkCount-- > 0) {
uint64_t word0 = GET_UNALIGNED(uint64_t, buffer);
uint64_t word1 = GET_UNALIGNED(uint64_t, buffer + 1 * sizeof(uint64_t));
uint64_t word2 = GET_UNALIGNED(uint64_t, buffer + 2 * sizeof(uint64_t));
uint64_t word3 = GET_UNALIGNED(uint64_t, buffer + 3 * sizeof(uint64_t));
uint64_t word4 = GET_UNALIGNED(uint64_t, buffer + 4 * sizeof(uint64_t));
uint64_t word5 = GET_UNALIGNED(uint64_t, buffer + 5 * sizeof(uint64_t));
uint64_t word6 = GET_UNALIGNED(uint64_t, buffer + 6 * sizeof(uint64_t));
uint64_t word7 = GET_UNALIGNED(uint64_t, buffer + 7 * sizeof(uint64_t));
uint64_t or = (word0 | word1 | word2 | word3
| word4 | word5 | word6 | word7);
// Prevent compiler from using 8*(cmp;jne).
__asm__ __volatile__ ("" : : "g" (or));
if (or != 0) {
return false;
}
buffer += 8 * sizeof(uint64_t);
}
wordCount %= 8;
// Unroll to process 8 bytes at a time.
// (Is this still worthwhile?)
while (wordCount-- > 0) {
if (GET_UNALIGNED(uint64_t, buffer) != 0) {
return false;
}
buffer += sizeof(uint64_t);
}
length %= sizeof(uint64_t);
// Fall through to finish up anything left over.
}
while (length-- > 0) {
if (*buffer++ != 0) {
return false;
}
}
return true;
}
/**********************************************************************/
bool bioIsZeroData(BIO *bio)
{
struct bio_vec *biovec;
for (BioIterator iter = createBioIterator(bio);
(biovec = getNextBiovec(&iter)) != NULL;
advanceBioIterator(&iter)) {
if (!isAllZeros(getBufferForBiovec(biovec), biovec->bv_len)) {
return false;
}
}
return true;
}
/**********************************************************************/
void bioZeroData(BIO *bio)
{
zero_fill_bio(bio);
}
/**********************************************************************/
static void setBioSize(BIO *bio, BlockSize bioSize)
{
#ifdef USE_BI_ITER
bio->bi_iter.bi_size = bioSize;
#else
bio->bi_size = bioSize;
#endif
}
/**
* Initialize a bio.
*
* @param bio The bio to initialize
* @param layer The layer to which it belongs.
**/
static void initializeBio(BIO *bio, KernelLayer *layer)
{
// Save off important info so it can be set back later
unsigned short vcnt = bio->bi_vcnt;
void *pvt = bio->bi_private;
bio_reset(bio); // Memsets large portion of bio. Reset all needed fields.
bio->bi_private = pvt;
bio->bi_vcnt = vcnt;
bio->bi_end_io = completeAsyncBio;
setBioSector(bio, (sector_t) -1); // Sector will be set later on.
setBioBlockDevice(bio, getKernelLayerBdev(layer));
}
/**********************************************************************/
void resetBio(BIO *bio, KernelLayer *layer)
{
initializeBio(bio, layer);
setBioSize(bio, VDO_BLOCK_SIZE);
}
/**********************************************************************/
int allocateBio(KernelLayer *layer, unsigned int bvecCount, BIO **bioPtr)
{
BIO *bio = bio_alloc_bioset(GFP_NOIO, bvecCount, layer->bioset);
if (IS_ERR(bio)) {
logError("bio allocation failure %ld", PTR_ERR(bio));
return PTR_ERR(bio);
}
initializeBio(bio, layer);
*bioPtr = bio;
return VDO_SUCCESS;
}
/**********************************************************************/
int createBio(KernelLayer *layer, char *data, BIO **bioPtr)
{
BIO *bio = NULL;
if (data == NULL) {
int result = allocateBio(layer, 0, &bio);
if (result != VDO_SUCCESS) {
return result;
}
*bioPtr = bio;
return VDO_SUCCESS;
}
unsigned int len = VDO_BLOCK_SIZE;
unsigned long kaddr = (unsigned long) data;
unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
unsigned long start = kaddr >> PAGE_SHIFT;
const int bvecCount = end - start;
int result = allocateBio(layer, bvecCount, &bio);
if (result != VDO_SUCCESS) {
return result;
}
int offset = offset_in_page(kaddr);
for (unsigned int i = 0; (i < bvecCount) && (len > 0); i++) {
unsigned int bytes = PAGE_SIZE - offset;
if (bytes > len) {
bytes = len;
}
struct page *page
= is_vmalloc_addr(data) ? vmalloc_to_page(data) : virt_to_page(data);
int bytesAdded = bio_add_page(bio, page, bytes, offset);
if (bytesAdded != bytes) {
freeBio(bio, layer);
return logErrorWithStringError(VDO_BIO_CREATION_FAILED,
"Could only add %i bytes to bio",
bytesAdded);
}
data += bytes;
len -= bytes;
offset = 0;
}
*bioPtr = bio;
return VDO_SUCCESS;
}
/**********************************************************************/
void prepareFlushBIO(BIO *bio,
void *context,
struct block_device *device,
bio_end_io_t *endIOCallback)
{
clearBioOperationAndFlags(bio);
/*
* One would think we could use REQ_OP_FLUSH on new kernels, but some
* layers of the stack don't recognize that as a flush. So do it
* like blkdev_issue_flush() and make it a write+flush.
*/
setBioOperationWrite(bio);
setBioOperationFlagPreflush(bio);
bio->bi_end_io = endIOCallback;
bio->bi_private = context;
bio->bi_vcnt = 0;
setBioBlockDevice(bio, device);
setBioSize(bio, 0);
setBioSector(bio, 0);
}