/** * Copyright (C) Mellanox Technologies Ltd. 2001-2018. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCS_REG_CACHE_H_ #define UCS_REG_CACHE_H_ /* * Memory registration cache - holds registered memory regions, takes care of * memory invalidation (if it's unmapped), merging of regions, protection flags. * This data structure is thread safe. */ #include #include #include #include #include #include #define UCS_RCACHE_PROT_FMT "%c%c" #define UCS_RCACHE_PROT_ARG(_prot) \ ((_prot) & PROT_READ) ? 'r' : '-', \ ((_prot) & PROT_WRITE) ? 'w' : '-' typedef struct ucs_rcache ucs_rcache_t; typedef struct ucs_rcache_ops ucs_rcache_ops_t; typedef struct ucs_rcache_params ucs_rcache_params_t; typedef struct ucs_rcache_region ucs_rcache_region_t; /* * Memory region flags. */ enum { UCS_RCACHE_REGION_FLAG_REGISTERED = UCS_BIT(0), /**< Memory registered */ UCS_RCACHE_REGION_FLAG_PGTABLE = UCS_BIT(1) /**< In the page table */ }; /* * Memory registration flags. */ enum { UCS_RCACHE_MEM_REG_HIDE_ERRORS = UCS_BIT(0) /**< Hide errors on memory registration */ }; /* * Registration cache operations. */ struct ucs_rcache_ops { /** * Register a memory region. * * @param [in] context User context, as passed to @ref ucs_rcache_create * @param [in] rcache Pointer to the registration cache. * @param [in] arg Custom argument passed to @ref ucs_rcache_get(). * @param [in] region Memory region to register. This may point to a larger * user-defined structure, as specified by the field * `region_struct_size' in @ref ucs_rcache_params. * This function may store relevant information (such * as memory keys) inside the larger structure. * @param [in] flags Memory registration flags. * * @return UCS_OK if registration is successful, error otherwise. * * @note This function should be able to handle inaccessible memory addresses * and return error status in this case, without any destructive consequences * such as error messages or fatal failure. */ ucs_status_t (*mem_reg)(void *context, ucs_rcache_t *rcache, void *arg, ucs_rcache_region_t *region, uint16_t flags); /** * Deregister a memory region. * * @param [in] context User context, as passed to @ref ucs_rcache_create * @param [in] rcache Pointer to the registration cache. * @param [in] region Memory region to deregister. */ void (*mem_dereg)(void *context, ucs_rcache_t *rcache, ucs_rcache_region_t *region); /** * Dump memory region information to a string buffer. * (Only the user-defined part of the memory regoin should be dumped) * * @param [in] context User context, as passed to @ref ucs_rcache_create * @param [in] rcache Pointer to the registration cache. * @param [in] region Memory region to dump. * @param [in] buf String buffer to dump to. * @param [in] max Maximal length of the string buffer. */ void (*dump_region)(void *context, ucs_rcache_t *rcache, ucs_rcache_region_t *region, char *buf, size_t max); }; struct ucs_rcache_params { size_t region_struct_size; /**< Size of memory region structure, must be at least the size of @ref ucs_rcache_region_t */ size_t alignment; /**< Force-align regions to this size. Must be smaller or equal to system page size. */ size_t max_alignment; /**< Maximum alignment */ int ucm_events; /**< UCM events to register. Currently UCM_EVENT_VM_UNMAPPED and UCM_EVENT_MEM_TYPE_FREE are supported */ int ucm_event_priority; /**< Priority of memory events */ const ucs_rcache_ops_t *ops; /**< Memory operations functions */ void *context; /**< User-defined context that will be passed to mem_reg/mem_dereg */ }; struct ucs_rcache_region { ucs_pgt_region_t super; /**< Base class - page table region */ ucs_list_link_t list; /**< List element */ volatile uint32_t refcount; /**< Reference count, including +1 if it's in the page table */ ucs_status_t status; /**< Current status code */ uint8_t prot; /**< Protection bits */ uint16_t flags; /**< Status flags. Protected by page table lock. */ uint64_t priv; /**< Used internally */ }; /** * Create a memory registration cache. * * @param [in] params Registration cache parameters. * @param [in] name Registration cache name, for debugging. * @param [in] stats_parent Pointer to statistics parent node. * @param [out] rcache_p Filled with a pointer to the registration cache. */ ucs_status_t ucs_rcache_create(const ucs_rcache_params_t *params, const char *name, ucs_stats_node_t *stats_parent, ucs_rcache_t **rcache_p); /** * Destroy a memory registration cache. * * @param [in] rcache Registration cache to destroy. */ void ucs_rcache_destroy(ucs_rcache_t *rcache); /** * Resolve buffer in the registration cache, or register it if not found. * TODO register after N usages. * * @param [in] rcache Memory registration cache. * @param [in] address Address to register or resolve. * @param [in] length Length of buffer to register or resolve. * @param [in] prot Requested access flags, PROT_xx (same as passed to mmap). * @param [in] arg Custom argument passed down to memory registration * callback, if a memory registration happens during * this call. * @param [out] region_p On success, filled with a pointer to the memory * region. The user could put more data in the region * structure in mem_reg() function. * * On success succeeds, the memory region reference count is incremented by 1. * * @return Error code. */ ucs_status_t ucs_rcache_get(ucs_rcache_t *rcache, void *address, size_t length, int prot, void *arg, ucs_rcache_region_t **region_p); /** * Increment memory region reference count. * * @param [in] rcache Memory registration cache. * @param [in] region Memory region whose reference count to increment. */ void ucs_rcache_region_hold(ucs_rcache_t *rcache, ucs_rcache_region_t *region); /** * Decrement memory region reference count and possibly destroy it. * * @param [in] rcache Memory registration cache. * @param [in] region Memory region to release. */ void ucs_rcache_region_put(ucs_rcache_t *rcache, ucs_rcache_region_t *region); #endif