/*

 * Copyright (c) 2020 Red Hat, Inc.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 * 

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 * 

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

 * 02110-1301, USA. 

 *

 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/kernel/bio.c#8 $

 */

#include "bio.h"

#include "logger.h"

#include "memoryAlloc.h"

#include "numeric.h"

#include "flush.h"

#include "recoveryJournal.h"

#include "bioIterator.h"

#include "ioSubmitter.h"

/**

 * Gets the raw buffer from a biovec.

 *

 * @param biovec  The biovec in question

 *

 * @return the buffer

 **/

static char *getBufferForBiovec(struct bio_vec *biovec)

{

  return (page_address(biovec->bv_page) + biovec->bv_offset);

}

/**********************************************************************/

void bioCopyDataIn(BIO *bio, char *dataPtr)

{

  struct bio_vec *biovec;

  for (BioIterator iter = createBioIterator(bio);

       (biovec = getNextBiovec(&iter)) != NULL;

       advanceBioIterator(&iter)) {

    memcpy(dataPtr, getBufferForBiovec(biovec), biovec->bv_len);

    dataPtr += biovec->bv_len;

  }

}

/**********************************************************************/

void bioCopyDataOut(BIO *bio, char *dataPtr)

{

  struct bio_vec *biovec;

  for (BioIterator iter = createBioIterator(bio);

       (biovec = getNextBiovec(&iter)) != NULL;

       advanceBioIterator(&iter)) {

    memcpy(getBufferForBiovec(biovec), dataPtr, biovec->bv_len);

    flush_dcache_page(biovec->bv_page);

    dataPtr += biovec->bv_len;

  }

}

/**********************************************************************/

void setBioOperation(BIO *bio, unsigned int operation)

{

#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)

  bio->bi_opf &= ~REQ_OP_MASK;

  bio->bi_opf |= operation;

#else

  unsigned int OPERATION_MASK = WRITE | REQ_DISCARD | REQ_FLUSH;

  // Clear the relevant bits

  bio->bi_rw &= ~OPERATION_MASK;

  // Set the operation we care about

  bio->bi_rw |= operation;

#endif

}

/**********************************************************************/

void freeBio(BIO *bio, KernelLayer *layer)

{

  bio_put(bio);

}

/**********************************************************************/

void countBios(AtomicBioStats *bioStats, BIO *bio)

{

  if (isWriteBio(bio)) {

    atomic64_inc(&bioStats->write);

  } else {

    atomic64_inc(&bioStats->read);

  }

  if (isDiscardBio(bio)) {

    atomic64_inc(&bioStats->discard);

  }

  if (isFlushBio(bio)) {

    atomic64_inc(&bioStats->flush);

  }

  if (isFUABio(bio)) {

    atomic64_inc(&bioStats->fua);

  }

}

/**

 * The function determines whether a buffer contains all zeroes.

 *

 * @param buffer  The buffer to check

 * @param length  The length of the buffer

 *

 * @return true is all zeroes, false otherwise

 **/

static inline bool isAllZeros(const char *buffer, unsigned int length)

{

  /*

   * Handle expected common case of even the first word being nonzero,

   * without getting into the more expensive (for one iteration) loop

   * below.

   */

  if (likely(length >= sizeof(uint64_t))) {

    if (GET_UNALIGNED(uint64_t, buffer) != 0) {

      return false;

    }

    unsigned int wordCount = length / sizeof(uint64_t);

    // Unroll to process 64 bytes at a time

    unsigned int chunkCount = wordCount / 8;

    while (chunkCount-- > 0) {

      uint64_t word0 = GET_UNALIGNED(uint64_t, buffer);

      uint64_t word1 = GET_UNALIGNED(uint64_t, buffer + 1 * sizeof(uint64_t));

      uint64_t word2 = GET_UNALIGNED(uint64_t, buffer + 2 * sizeof(uint64_t));

      uint64_t word3 = GET_UNALIGNED(uint64_t, buffer + 3 * sizeof(uint64_t));

      uint64_t word4 = GET_UNALIGNED(uint64_t, buffer + 4 * sizeof(uint64_t));

      uint64_t word5 = GET_UNALIGNED(uint64_t, buffer + 5 * sizeof(uint64_t));

      uint64_t word6 = GET_UNALIGNED(uint64_t, buffer + 6 * sizeof(uint64_t));

      uint64_t word7 = GET_UNALIGNED(uint64_t, buffer + 7 * sizeof(uint64_t));

      uint64_t or = (word0 | word1 | word2 | word3

                     | word4 | word5 | word6 | word7);

      // Prevent compiler from using 8*(cmp;jne).

      __asm__ __volatile__ ("" : : "g" (or));

      if (or != 0) {

        return false;

      }

      buffer += 8 * sizeof(uint64_t);

    }

    wordCount %= 8;

    // Unroll to process 8 bytes at a time.

    // (Is this still worthwhile?)

    while (wordCount-- > 0) {

      if (GET_UNALIGNED(uint64_t, buffer) != 0) {

        return false;

      }

      buffer += sizeof(uint64_t);

    }

    length %= sizeof(uint64_t);

    // Fall through to finish up anything left over.

  }

  while (length-- > 0) {

    if (*buffer++ != 0) {

      return false;

    }

  }

  return true;

}

/**********************************************************************/

bool bioIsZeroData(BIO *bio)

{

  struct bio_vec *biovec;

  for (BioIterator iter = createBioIterator(bio);

       (biovec = getNextBiovec(&iter)) != NULL;

       advanceBioIterator(&iter)) {

    if (!isAllZeros(getBufferForBiovec(biovec), biovec->bv_len)) {

      return false;

    }

  }

  return true;

}

/**********************************************************************/

void bioZeroData(BIO *bio)

{

  zero_fill_bio(bio);

}

/**********************************************************************/

static void setBioSize(BIO *bio, BlockSize bioSize)

{

#ifdef USE_BI_ITER

  bio->bi_iter.bi_size = bioSize;

#else

  bio->bi_size = bioSize;

#endif

}

/**

 * Initialize a bio.

 *

 * @param bio    The bio to initialize

 * @param layer  The layer to which it belongs.

 **/

static void initializeBio(BIO *bio, KernelLayer *layer)

{

  // Save off important info so it can be set back later

  unsigned short  vcnt = bio->bi_vcnt;

  void           *pvt  = bio->bi_private;

  bio_reset(bio);     // Memsets large portion of bio. Reset all needed fields.

  bio->bi_private      = pvt;

  bio->bi_vcnt         = vcnt;

  bio->bi_end_io       = completeAsyncBio;

  setBioSector(bio, (sector_t) -1);  // Sector will be set later on.

  setBioBlockDevice(bio, getKernelLayerBdev(layer));

}

/**********************************************************************/

void resetBio(BIO *bio, KernelLayer *layer)

{

  initializeBio(bio, layer);

  setBioSize(bio, VDO_BLOCK_SIZE);

}

/**********************************************************************/

int allocateBio(KernelLayer *layer, unsigned int bvecCount, BIO **bioPtr)

{

  BIO *bio = bio_alloc_bioset(GFP_NOIO, bvecCount, layer->bioset);

  if (IS_ERR(bio)) {

    logError("bio allocation failure %ld", PTR_ERR(bio));

    return PTR_ERR(bio);

  }

  initializeBio(bio, layer);

  *bioPtr = bio;

  return VDO_SUCCESS;

}

/**********************************************************************/

int createBio(KernelLayer *layer, char *data, BIO **bioPtr)

{

  BIO *bio = NULL;

  if (data == NULL) {

    int result = allocateBio(layer, 0, &bio;;

    if (result != VDO_SUCCESS) {

      return result;

    }

    *bioPtr = bio;

    return VDO_SUCCESS;

  }

  unsigned int  len       = VDO_BLOCK_SIZE;

  unsigned long kaddr     = (unsigned long) data;

  unsigned long end       = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

  unsigned long start     = kaddr >> PAGE_SHIFT;

  const int     bvecCount = end - start;

  int result = allocateBio(layer, bvecCount, &bio;;

  if (result != VDO_SUCCESS) {

    return result;

  }

  int offset = offset_in_page(kaddr);

  for (unsigned int i = 0; (i < bvecCount) && (len > 0); i++) {

    unsigned int bytes = PAGE_SIZE - offset;

    if (bytes > len) {

      bytes = len;

    }

  struct page *page

    = is_vmalloc_addr(data) ? vmalloc_to_page(data) : virt_to_page(data);

  int bytesAdded = bio_add_page(bio, page, bytes, offset);

  if (bytesAdded != bytes) {

    freeBio(bio, layer);

    return logErrorWithStringError(VDO_BIO_CREATION_FAILED,

                                   "Could only add %i bytes to bio",

                                   bytesAdded);

    }

    data   += bytes;

    len    -= bytes;

    offset  = 0;

  }

  *bioPtr = bio;

  return VDO_SUCCESS;

}

/**********************************************************************/

void prepareFlushBIO(BIO                 *bio,

                     void                *context,

                     struct block_device *device,

                     bio_end_io_t        *endIOCallback)

{

  clearBioOperationAndFlags(bio);

  /*

   * One would think we could use REQ_OP_FLUSH on new kernels, but some

   * layers of the stack don't recognize that as a flush. So do it

   * like blkdev_issue_flush() and make it a write+flush.

   */

  setBioOperationWrite(bio);

  setBioOperationFlagPreflush(bio);

  bio->bi_end_io  = endIOCallback;

  bio->bi_private = context;

  bio->bi_vcnt    = 0;

  setBioBlockDevice(bio, device);

  setBioSize(bio, 0);

  setBioSector(bio, 0);

}