/*

 * Copyright (C) 2015 - 2019 Intel Corporation.

 * All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions are met:

 * 1. Redistributions of source code must retain the above copyright notice(s),

 *    this list of conditions and the following disclaimer.

 * 2. Redistributions in binary form must reproduce the above copyright notice(s),

 *    this list of conditions and the following disclaimer in the documentation

 *    and/or other materials provided with the distribution.

 *

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS

 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF

 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO

 * EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,

 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF

 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE

 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF

 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

///////////////////////////////////////////////////////////////////////////

// File   : autohbw.c

// Purpose: Library to automatically allocate HBW (MCDRAM)

// Author : Ruchira Sasanka (ruchira DOT sasanka AT intel DOT com)

// Date   : Jan 30, 2015

///////////////////////////////////////////////////////////////////////////

#include <memkind.h>

#include <assert.h>

#include <ctype.h>

#include <errno.h>

#include <stdbool.h>

#include <stddef.h>

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#define AUTOHBW_EXPORT __attribute__((visibility("default")))

#define AUTOHBW_INIT __attribute__((constructor))

//-2 = nothing is printed

//-1 = critical messages are printed

// 0 = no log messages for allocations are printed but INFO messages are printed

// 1 = a log message is printed for each allocation (Default)

// 2 = a log message is printed for each allocation with a backtrace

enum {

    ALWAYS = -1,

    INFO,

    ALLOC,

    VERBOSE

};

// Default is to print allocations

static int LogLevel = ALLOC;

// Allocations of size greater than low limit promoted to HBW memory.

// If there is a high limit specified, allocations larger than this limit

// will not be allocated in HBW.

static size_t HBWLowLimit = 1 * 1024 * 1024;

static size_t HBWHighLimit = -1ull;

// Whether we have initialized HBW arena of memkind library -- by making

// a dummy call to it. HBW arena (and hence any memkind_* call with kind

// HBW) must NOT be used until this flag is set true.

static bool MemkindInitDone = false;

// Following is the type of HBW memory that is allocated using memkind.

// By changing this type, this library can be used to allocate other

// types of memory types (e.g., MEMKIND_HUGETLB, MEMKIND_GBTLB,

// MEMKIND_HBW_HUGETLB etc.)

static memkind_t hbw_kind;

// API control for HBW allocations.

static bool isAutoHBWEnabled = true;

#define LOG(level, ...)                                                                            \

    do {                                                                                           \

        if (LogLevel >= level) {                                                                   \

            fprintf(stderr, __VA_ARGS__);                                                          \

        }                                                                                          \

    } while (0)

static bool isAllocInHBW(size_t size)

{

    if (!MemkindInitDone)

        return false;

    if (!isAutoHBWEnabled)

        return false;

    if (size < HBWLowLimit)

        return false;

    if (size > HBWHighLimit)

        return false;

    return true;

}

// Returns the limit in bytes using a limit value and a multiplier

// character like K, M, G

static size_t getLimit(size_t limit, char lchar)

{

    // Now read the trailing character (e.g., K, M, G)

    // Based on the character, determine the multiplier

    if ((limit > 0) && isalpha(lchar)) {

        long mult = 1;

        switch (toupper(lchar)) {

            case 'G':

                mult *= 1024;

            case 'M':

                mult *= 1024;

            case 'K':

                mult *= 1024;

        }

        // check for overflow, saturate at max

        if (limit >= -1ull / mult)

            return -1ull;

        return limit * mult;

    }

    return limit;

}

// Once HBWLowLimit (and HBWHighLimit) are set, call this method to

// inform the user about the size range of arrays that will be allocated

// in HBW

static void printLimits()

{

    // Inform according to the limits set

    if ((HBWLowLimit > 0) && (HBWHighLimit < -1ull)) {

        // if both high and low limits are specified, we use a range

        LOG(INFO, "INFO: Allocations between %ldK - %ldK will be allocated in "

            "HBW. Set AUTO_HBW_SIZE=X:Y to change this limit.\n",

            HBWLowLimit / 1024, HBWHighLimit / 1024);

    } else if (HBWLowLimit > 0) {

        // if only a low limit is provided, use that

        LOG(INFO, "INFO: Allocations greater than %ldK will be allocated in HBW."

            " Set AUTO_HBW_SIZE=X:Y to change this limit.\n",

            HBWLowLimit / 1024);

    } else if (HBWHighLimit < -1ull) {

        // if only a high limit is provided, use that

        LOG(INFO, "INFO: Allocations smaller than %ldK will be allocated in HBW. "

            "Set AUTO_HBW_SIZE=X:Y to change this limit.\n",

            HBWHighLimit / 1024);

    } else {

        // none of limits is set to non-edge value, everything goes to HBW

        LOG(INFO, "INFO: All allocation will be done in HBW.");

    }

}

struct kind_name_t {

    memkind_t *kind;

    const char *name;

};

static struct kind_name_t named_kinds[] = {

    { &MEMKIND_DEFAULT, "memkind_default" },

    { &MEMKIND_HUGETLB, "memkind_hugetlb" },

    { &MEMKIND_INTERLEAVE, "memkind_interleave" },

    { &MEMKIND_HBW, "memkind_hbw" },

    { &MEMKIND_HBW_PREFERRED, "memkind_hbw_preferred" },

    { &MEMKIND_HBW_HUGETLB, "memkind_hbw_hugetlb" },

    { &MEMKIND_HBW_PREFERRED_HUGETLB, "memkind_hbw_preferred_hugetlb" },

    { &MEMKIND_HBW_GBTLB, "memkind_hbw_gbtlb" },

    { &MEMKIND_HBW_PREFERRED_GBTLB, "memkind_hbw_preferred_gbtlb" },

    { &MEMKIND_GBTLB, "memkind_gbtlb" },

    { &MEMKIND_HBW_INTERLEAVE, "memkind_hbw_interleave" },

};

static memkind_t get_kind_by_name(const char *name)

{

    int i;

    for (i = 0; i < sizeof(named_kinds) / sizeof(named_kinds[0]); ++i)

        if (strcasecmp(named_kinds[i].name, name) == 0)

            return *named_kinds[i].kind;

    return 0;

}

// Read from the environment and sets global variables

// Env variables are:

//   AUTO_HBW_SIZE = gives the size for auto HBW allocation

//   AUTO_HBW_LOG = gives logging level

static void setEnvValues()

{

    // STEP: Read the log level from the env variable. Do this early because

    //       printing depends on this

    char *log_str = secure_getenv("AUTO_HBW_LOG");

    if (log_str && strlen(log_str)) {

        int level = atoi(log_str);

        LogLevel = level;

        LOG(ALWAYS, "INFO: Setting log level to %d\n", LogLevel);

    }

    if (LogLevel == INFO) {

        LOG(INFO, "INFO: HBW allocation stats will not be printed. "

            "Set AUTO_HBW_LOG to enable.\n");

    } else if (LogLevel == ALLOC) {

        LOG(INFO, "INFO: Only HBW allocations will be printed. "

            "Set AUTO_HBW_LOG to disable/enable.\n");

    } else if (LogLevel == VERBOSE) {

        LOG(INFO, "INFO: HBW allocation with backtrace info will be printed. "

            "Set AUTO_HBW_LOG to disable.\n");

    }

    // Set the memory type allocated by this library. By default, it is

    // MEMKIND_HBW, but we can use this library to allocate other memory

    // types

    const char *memtype_str = secure_getenv("AUTO_HBW_MEM_TYPE");

    if (memtype_str && strlen(memtype_str)) {

        // Find the memkind_t using the name the user has provided in the env variable

        memkind_t mty = get_kind_by_name(memtype_str);

        if (mty != 0) {

            hbw_kind = mty;

            LOG(INFO, "INFO: Setting HBW memory type to %s\n", memtype_str);

        } else {

            LOG(ALWAYS, "WARN: Memory type %s not recognized. Using default type\n",

                memtype_str);

        }

    }

    // STEP: Set the size limits (thresholds) for HBW allocation

    //

    // Reads the environment variable

    const char *size_str = secure_getenv("AUTO_HBW_SIZE");

    if (size_str) {

        size_t lowlim = HBWLowLimit / 1024;

        size_t highlim = HBWHighLimit / 1024;

        char lowC = 'K', highC = 'K';

        if (size_str) {

            char *ptr = (char *)size_str;

            lowlim = strtoll(ptr, &ptr, 10);

            if (*ptr != 0 && *ptr != ':')

                lowC = *ptr++;

            else

                lowC = ' ';

            if (*ptr++ == ':') {

                highlim = strtoll(ptr, &ptr, 10);

                if (*ptr)

                    highC = *ptr;

                else

                    highC = ' ';

            }

            LOG(INFO, "INFO: lowlim=%zu(%c), highlim=%zu(%c)\n", lowlim, lowC, highlim,

                highC);

        }

        HBWLowLimit = getLimit(lowlim, lowC);

        HBWHighLimit = getLimit(highlim, highC);

        if (HBWLowLimit >= HBWHighLimit) {

            LOG(ALWAYS, "WARN: In AUTO_HBW_SIZE=X:Y, X cannot be greater or equal to Y. "

                "None of allocations will use HBW memory.\n");

        }

    } else {

        // if the user did not specify any limits, inform that we are using

        // default limits

        LOG(INFO, "INFO: Using default values for array size thresholds. "

            "Set AUTO_HBW_SIZE=X:Y to change.\n");

    }

    // inform the user about limits

    printLimits();

}

// This function is executed at library load time.

// Initialize HBW arena by making a dummy allocation/free at library load

// time. Until HBW initialization is complete, we must not call any

// allocation routines with HBW as kind.

static void AUTOHBW_INIT autohbw_load(void)

{

    // First set the default memory type this library allocates. This can

    // be overridden by env variable

    // Note: 'memkind_hbw_preferred' will allow falling back to DDR but

    //       'memkind_hbw will not'

    // Note: If HBM is not installed on a system, memkind_hbw_preferred call

    //       would fail. Therefore, we need to check for availability first.

    if (memkind_check_available(MEMKIND_HBW) != 0) {

        LOG(ALWAYS, "WARN: *** No HBM found in system. Will use default (DDR) "

            "OR user specified type ***\n");

        hbw_kind = MEMKIND_DEFAULT;

    } else {

        hbw_kind = MEMKIND_HBW_PREFERRED;

    }

    // Read any env variables. This has to be done first because DbgLevel

    // is set using env variables and debug printing is used below

    setEnvValues(); // read any env variables

    LOG(INFO, "INFO: autohbw.so loaded!\n");

    // dummy HBW call to initialize HBW arena

    void *pp = memkind_malloc(hbw_kind, 16);

    if (pp == 0) {

        LOG(ALWAYS, "\t-HBW init call FAILED. "

            "Is required memory type present on your system?\n");

        abort();

    }

    LOG(ALWAYS, "\t-HBW int call succeeded\n");

    memkind_free(hbw_kind, pp);

    MemkindInitDone = true; // enable HBW allocation

}

static void *MemkindMalloc(size_t size)

{

    LOG(VERBOSE, "In my memkind malloc sz:%ld ... ", size);

    bool useHbw = isAllocInHBW(size);

    memkind_t kind = useHbw ? hbw_kind : MEMKIND_DEFAULT;

    if (useHbw)

        LOG(VERBOSE, "\tHBW");

    void *ptr = memkind_malloc(kind, size);

    LOG(VERBOSE, "\tptr:%p\n", ptr);

    return ptr;

}

static void *MemkindCalloc(size_t nmemb, size_t size)

{

    LOG(VERBOSE, "In my memkind calloc sz:%ld ..", size * nmemb);

    bool useHbw = isAllocInHBW(size);

    memkind_t kind = useHbw ? hbw_kind : MEMKIND_DEFAULT;

    if (useHbw)

        LOG(VERBOSE, "\tHBW");

    void *ptr = memkind_calloc(kind, nmemb, size);

    LOG(VERBOSE, "\tptr:%p\n", ptr);

    return ptr;

}

static void *MemkindRealloc(void *ptr, size_t size)

{

    LOG(VERBOSE, "In my memkind realloc sz:%ld, p1:%p ..", size, ptr);

    bool useHbw = isAllocInHBW(size);

    memkind_t kind = useHbw ? hbw_kind : MEMKIND_DEFAULT;

    if (useHbw)

        LOG(VERBOSE, "\tHBW");

    void *nptr = memkind_realloc(kind, ptr, size);

    LOG(VERBOSE, "\tptr=%p\n", nptr);

    return nptr;

}

static int MemkindAlign(void **memptr, size_t alignment, size_t size)

{

    LOG(VERBOSE, "In my memkind align sz:%ld .. ", size);

    bool useHbw = isAllocInHBW(size);

    memkind_t kind = useHbw ? hbw_kind : MEMKIND_DEFAULT;

    if (useHbw)

        LOG(VERBOSE, "\tHBW");

    int ret = memkind_posix_memalign(kind, memptr, alignment, size);

    LOG(VERBOSE, "\tptr:%p\n", *memptr);

    return ret;

}

// memkind_free does not need the exact kind, if kind is 0. Then

// the library can figure out the proper kind itself.

static void MemkindFree(void *ptr)

{

    // avoid to many useless logs

    if (ptr)

        LOG(VERBOSE, "In my memkind free, ptr:%p\n", ptr);

    memkind_free(0, ptr);

}

//--------------------------------------------------------------------------

// ------------------ Public API of autohbw           ----------------------

//--------------------------------------------------------------------------

AUTOHBW_EXPORT void enableAutoHBW()

{

    isAutoHBWEnabled = true;

    LOG(INFO, "INFO: HBW allocations enabled by application (for this rank)\n");

}

AUTOHBW_EXPORT void disableAutoHBW()

{

    isAutoHBWEnabled = false;

    LOG(INFO, "INFO: HBW allocations disabled by application (for this rank)\n");

}

AUTOHBW_EXPORT void *malloc(size_t size)

{

    return MemkindMalloc(size);

}

AUTOHBW_EXPORT void *calloc(size_t nmemb, size_t size)

{

    return MemkindCalloc(nmemb, size);

}

AUTOHBW_EXPORT void *realloc(void *ptr, size_t size)

{

    return MemkindRealloc(ptr, size);

}

AUTOHBW_EXPORT int posix_memalign(void **memptr, size_t alignment, size_t size)

{

    return MemkindAlign(memptr, alignment, size);

}

// Warn about deprecated function usage.

AUTOHBW_EXPORT void *valloc(size_t size)

{

    LOG(ALWAYS, "use of deprecated valloc. Use posix_memalign instead\n");

    void *memptr = 0;

    size_t boundary = sysconf(_SC_PAGESIZE);

    int status = MemkindAlign(&memptr, boundary, size);

    if (status == 0 && memptr != 0)

        return memptr;

    return 0;

}

// Warn about deprecated function usage.

AUTOHBW_EXPORT void *memalign(size_t boundary, size_t size)

{

    LOG(ALWAYS, "use of deprecated memalign. Use posix_memalign instead\n");

    void *memptr = 0;

    int status = MemkindAlign(&memptr, boundary, size);

    if (status == 0 && memptr != 0)

        return memptr;

    return 0;

}

AUTOHBW_EXPORT void free(void *ptr)

{

    return MemkindFree(ptr);

}