/*
 * Copyright (c) 2020 Red Hat, Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA. 
 *
 * $Id: //eng/vdo-releases/aluminum/src/c++/vdo/base/atomic.h#2 $
 */

#ifndef ATOMIC_H
#define ATOMIC_H

#include "atomicDefs.h"
#include "compiler.h"
#include "typeDefs.h"

#define ATOMIC_INITIALIZER(value) { (value) }

typedef struct {
  atomic_t value;
} __attribute__((aligned(4))) Atomic32;

typedef struct {
  atomic64_t value;
} __attribute__((aligned(8))) Atomic64;

typedef struct {
  Atomic32 value;
} __attribute__((aligned(4))) AtomicBool;

/**
 * Memory load operations that precede this fence will be prevented from
 * changing order with any that follow this fence, by either the compiler or
 * the CPU. This can be used to ensure that the load operations accessing
 * the fields of a structure are not re-ordered so they actually take effect
 * before a pointer to the structure is resolved.
 **/
static INLINE void loadFence(void)
{
  smp_rmb();
}

/**
 * Memory store operations that precede this fence will be prevented from
 * changing order with any that follow this fence, by either the compiler or
 * the CPU. This can be used to ensure that the store operations initializing
 * the fields of a structure are not re-ordered so they actually take effect
 * after a pointer to the structure is published.
 **/
static INLINE void storeFence(void)
{
  smp_wmb();
}

/**
 * Generate a full memory fence for the compiler and CPU. Load and store
 * operations issued before the fence will not be re-ordered with operations
 * issued after the fence.
 **/
static INLINE void memoryFence(void)
{
  smp_mb();
}

/**
 * Access the value of a 32-bit atomic variable, ensuring that the load is not
 * re-ordered by the compiler or CPU with any subsequent load operations.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE uint32_t atomicLoad32(const Atomic32 *atom)
{
  uint32_t value = atomic_read(&atom->value);
  loadFence();
  return value;
}

/**
 * Access the value of a 64-bit atomic variable, ensuring that the memory load
 * is not re-ordered by the compiler or CPU with any subsequent load
 * operations.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE uint64_t atomicLoad64(const Atomic64 *atom)
{
  uint64_t value = atomic64_read(&atom->value);
  loadFence();
  return value;
}

/**
 * Access the value of a boolean atomic variable, ensuring that the load is not
 * re-ordered by the compiler or CPU with any subsequent load operations.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE bool atomicLoadBool(const AtomicBool *atom)
{
  return (atomicLoad32(&atom->value) > 0);
}

/**
 * Set the value of a 32-bit atomic variable, ensuring that the memory store
 * operation is not re-ordered by the compiler or CPU with any preceding store
 * operations.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void atomicStore32(Atomic32 *atom, uint32_t newValue)
{
  storeFence();
  atomic_set(&atom->value, newValue);
}

/**
 * Set the value of a 64-bit atomic variable, ensuring that the memory store
 * operation is not re-ordered by the compiler or CPU with any preceding store
 * operations.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void atomicStore64(Atomic64 *atom, uint64_t newValue)
{
  storeFence();
  atomic64_set(&atom->value, newValue);
}

/**
 * Set the value of a boolean atomic variable, ensuring that the memory store
 * operation is not re-ordered by the compiler or CPU with any preceding store
 * operations.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void atomicStoreBool(AtomicBool *atom, bool newValue)
{
  atomicStore32(&atom->value, (newValue ? 1 : 0));
}

/**
 * Add a 32-bit signed delta to a 32-bit atomic variable.
 *
 * @param atom   a pointer to the atomic variable
 * @param delta  the value to be added (or subtracted) from the variable
 *
 * @return       the new value of the atom after the add operation
 **/
static INLINE uint32_t atomicAdd32(Atomic32 *atom, int32_t delta)
{
  return atomic_add_return(delta, &atom->value);
}

/**
 * Add a 64-bit signed delta to a 64-bit atomic variable.
 *
 * @param atom   a pointer to the atomic variable
 * @param delta  the value to be added (or subtracted) from the variable
 *
 * @return       the new value of the atom after the add operation
 **/
static INLINE uint64_t atomicAdd64(Atomic64 *atom, int64_t delta)
{
  return atomic64_add_return(delta, &atom->value);
}

/**
 * Atomic 32-bit compare-and-swap. If the atom is identical to a required
 * value, atomically replace it with the new value and return true, otherwise
 * do nothing and return false.
 *
 * @param atom           a pointer to the atomic variable
 * @param requiredValue  the value that must be present to perform the swap
 * @param newValue       the value to be swapped for the required value
 *
 * @return               true if the atom was changed, false otherwise
 **/
static INLINE bool compareAndSwap32(Atomic32 *atom,
                                    uint32_t  requiredValue,
                                    uint32_t  newValue)
{
  /*
   * Our initial implementation, for x86, effectively got a full
   * memory barrier because of how "lock cmpxchg" operates. The
   * atomic_cmpxchg interface provides for a full barrier *if* the
   * exchange is done, but not necessarily if it is not.
   *
   * Do we need the full barrier always? We need to investigate that,
   * as part of (eventually) converting to using that API directly.
   * For now, play it safe, and ensure the same behavior on other
   * architectures too.
   */
#ifndef __x86_64__
  smp_mb();
#endif
  int oldValue = atomic_cmpxchg(&atom->value, requiredValue, newValue);
#ifndef __x86_64__
  smp_mb();
#endif
  return requiredValue == (uint32_t) oldValue;
}

/**
 * Atomic 64-bit compare-and-swap. If the atom is identical to a required
 * value, atomically replace it with the new value and return true, otherwise
 * do nothing and return false.
 *
 * @param atom           a pointer to the atomic variable
 * @param requiredValue  the value that must be present to perform the swap
 * @param newValue       the value to be swapped for the required value
 *
 * @return               true if the atom was changed, false otherwise
 **/
static INLINE bool compareAndSwap64(Atomic64 *atom,
                                    uint64_t  requiredValue,
                                    uint64_t  newValue)
{
#ifndef __x86_64__
  smp_mb();
#endif
  long oldValue = atomic64_cmpxchg(&atom->value, requiredValue, newValue);
#ifndef __x86_64__
  smp_mb();
#endif
  return requiredValue == (uint64_t) oldValue;
}

/**
 * Atomic boolean compare-and-swap. If the atom is identical to a required
 * value, atomically replace it with the new value and return true, otherwise
 * do nothing and return false.
 *
 * @param atom           a pointer to the atomic variable
 * @param requiredValue  the value that must be present to perform the swap
 * @param newValue       the value to be swapped for the required value
 *
 * @return               true if the atom was changed, false otherwise
 **/
static INLINE bool compareAndSwapBool(AtomicBool *atom,
                                      bool        requiredValue,
                                      bool        newValue)
{
  return compareAndSwap32(&atom->value, (requiredValue ? 1 : 0),
                          (newValue ? 1 : 0));
}

/**
 * Access the value of a 32-bit atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE uint32_t relaxedLoad32(const Atomic32 *atom)
{
  return atomic_read(&atom->value);
}

/**
 * Access the value of a 64-bit atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE uint64_t relaxedLoad64(const Atomic64 *atom)
{
  return atomic64_read(&atom->value);
}

/**
 * Access the value of a boolean atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom  a pointer to the atomic variable to access
 *
 * @return the value that was in the atom at the moment it was accessed
 **/
static INLINE bool relaxedLoadBool(const AtomicBool *atom)
{
  return (relaxedLoad32(&atom->value) > 0);
}

/**
 * Set the value of a 32-bit atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void relaxedStore32(Atomic32 *atom, uint32_t newValue)
{
  atomic_set(&atom->value, newValue);
}

/**
 * Set the value of a 64-bit atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void relaxedStore64(Atomic64 *atom, uint64_t newValue)
{
  atomic64_set(&atom->value, newValue);
}

/**
 * Set the value of a boolean atomic variable using relaxed memory order,
 * without any compiler or CPU fences.
 *
 * @param atom      a pointer to the atomic variable to modify
 * @param newValue  the value to assign to the atomic variable
 **/
static INLINE void relaxedStoreBool(AtomicBool *atom, bool newValue)
{
  relaxedStore32(&atom->value, (newValue ? 1 : 0));
}

/**
 * Non-atomically add a 32-bit signed delta to a 32-bit atomic variable,
 * without any compiler or CPU fences.
 *
 * @param atom   a pointer to the atomic variable
 * @param delta  the value to be added (or subtracted) from the variable
 *
 * @return       the new value of the atom after the add operation
 **/
static INLINE uint32_t relaxedAdd32(Atomic32 *atom, int32_t delta)
{
  uint32_t newValue = (relaxedLoad32(atom) + delta);
  relaxedStore32(atom, newValue);
  return newValue;
}

/**
 * Non-atomically add a 64-bit signed delta to a 64-bit atomic variable,
 * without any compiler or CPU fences.
 *
 * @param atom   a pointer to the atomic variable
 * @param delta  the value to be added (or subtracted) from the variable
 *
 * @return       the new value of the atom after the add operation
 **/
static INLINE uint64_t relaxedAdd64(Atomic64 *atom, int64_t delta)
{
  uint64_t newValue = (relaxedLoad64(atom) + delta);
  relaxedStore64(atom, newValue);
  return newValue;
}

#endif /* ATOMIC_H */