// Copyright(c) 2017-2020, Intel Corporation
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
// * Neither the name of Intel Corporation nor the names of its contributors
// may be used to endorse or promote products derived from this software
// without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
/**
* @file hello_fpga.c
* @brief A code sample illustrates the basic usage of the OPAE C API.
*
* The sample is a host application that demonstrates the basic steps of
* interacting with FPGA using the OPAE library. These steps include:
*
* - FPGA enumeration
* - Resource acquiring and releasing
* - Managing shared memory buffer
* - MMIO read and write
*
* The sample also demonstrates OPAE's object model, such as tokens, handles,
* and properties.
*
* The sample requires a native loopback mode (NLB) test image to be loaded on
* the FPGA. Refer to
* <a href="https://opae.github.io/docs/fpga_api/quick_start/readme.html">Quick
* Start Guide</a> for full instructions on building, configuring, and running
* this code sample.
*
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif // HAVE_CONFIG_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <getopt.h>
#include <unistd.h>
#include <uuid/uuid.h>
#include <opae/fpga.h>
int usleep(unsigned);
#ifndef TEST_TIMEOUT
#define TEST_TIMEOUT 30000
#endif // TEST_TIMEOUT
#ifndef CL
# define CL(x) ((x) * 64)
#endif // CL
#ifndef LOG2_CL
# define LOG2_CL 6
#endif // LOG2_CL
#ifndef MB
# define MB(x) ((x) * 1024 * 1024)
#endif // MB
#define CACHELINE_ALIGNED_ADDR(p) ((p) >> LOG2_CL)
#define LPBK1_BUFFER_SIZE MB(1)
#define LPBK1_BUFFER_ALLOCATION_SIZE MB(2)
#define LPBK1_DSM_SIZE MB(2)
#define CSR_SRC_ADDR 0x0120
#define CSR_DST_ADDR 0x0128
#define CSR_CTL 0x0138
#define CSR_STATUS1 0x0168
#define CSR_CFG 0x0140
#define CSR_NUM_LINES 0x0130
#define DSM_STATUS_TEST_COMPLETE 0x40
#define CSR_AFU_DSM_BASEL 0x0110
/* NLB0 AFU_ID */
#define NLB0_AFUID "D8424DC4-A4A3-C413-F89E-433683F9040B"
/*
* macro to check return codes, print error message, and goto cleanup label
* NOTE: this changes the program flow (uses goto)!
*/
#define ON_ERR_GOTO(res, label, desc) \
do { \
if ((res) != FPGA_OK) { \
print_err((desc), (res)); \
goto label; \
} \
} while (0)
/* Type definitions */
typedef struct {
uint32_t uint[16];
} cache_line;
void print_err(const char *s, fpga_result res)
{
fprintf(stderr, "Error %s: %s\n", s, fpgaErrStr(res));
}
/*
* Global configuration of bus, set during parse_args()
* */
struct config {
struct target {
int bus;
} target;
int open_flags;
}
config = {
.target = {
.bus = -1,
},
.open_flags = 0
};
#define GETOPT_STRING "B:sv"
fpga_result parse_args(int argc, char *argv[])
{
struct option longopts[] = {
{ "bus", required_argument, NULL, 'B' },
{ "shared", no_argument, NULL, 's' },
{ "version", no_argument, NULL, 'v' },
{ NULL, 0, NULL, 0 }
};
int getopt_ret;
int option_index;
char *endptr = NULL;
char version[32];
char build[32];
while (-1 != (getopt_ret = getopt_long(argc, argv, GETOPT_STRING,
longopts, &option_index))) {
const char *tmp_optarg = optarg;
/* Checks to see if optarg is null and if not it goes to value of optarg */
if ((optarg) && ('=' == *tmp_optarg)) {
++tmp_optarg;
}
switch (getopt_ret) {
case 'B': /* bus */
if (NULL == tmp_optarg) {
return FPGA_EXCEPTION;
}
endptr = NULL;
config.target.bus = (int) strtoul(tmp_optarg, &endptr, 0);
if (endptr != tmp_optarg + strnlen(tmp_optarg, 100)) {
fprintf(stderr, "invalid bus: %s\n", tmp_optarg);
return FPGA_EXCEPTION;
}
break;
case 's':
config.open_flags |= FPGA_OPEN_SHARED;
break;
case 'v':
fpgaGetOPAECVersionString(version, sizeof(version));
fpgaGetOPAECBuildString(build, sizeof(build));
printf("hello_fpga %s %s\n",
version, build);
return -1;
default: /* invalid option */
fprintf(stderr, "Invalid cmdline option \n");
return FPGA_EXCEPTION;
}
}
return FPGA_OK;
}
fpga_result find_fpga(fpga_guid afu_guid,
fpga_token *accelerator_token,
uint32_t *num_matches_accelerators)
{
fpga_properties filter = NULL;
fpga_result res1;
fpga_result res2 = FPGA_OK;
res1 = fpgaGetProperties(NULL, &filter);
ON_ERR_GOTO(res1, out, "creating properties object");
res1 = fpgaPropertiesSetObjectType(filter, FPGA_ACCELERATOR);
ON_ERR_GOTO(res1, out_destroy, "setting object type");
res1 = fpgaPropertiesSetGUID(filter, afu_guid);
ON_ERR_GOTO(res1, out_destroy, "setting GUID");
if (-1 != config.target.bus) {
res1 = fpgaPropertiesSetBus(filter, config.target.bus);
ON_ERR_GOTO(res1, out_destroy, "setting bus");
}
res1 = fpgaEnumerate(&filter, 1, accelerator_token, 1, num_matches_accelerators);
ON_ERR_GOTO(res1, out_destroy, "enumerating accelerators");
out_destroy:
res2 = fpgaDestroyProperties(&filter);
ON_ERR_GOTO(res2, out, "destroying properties object");
out:
return res1 != FPGA_OK ? res1 : res2;
}
/* function to get the bus number when there are multiple accelerators */
fpga_result get_bus(fpga_token tok, uint8_t *bus)
{
fpga_result res1;
fpga_result res2 = FPGA_OK;
fpga_properties props = NULL;
res1 = fpgaGetProperties(tok, &props);
ON_ERR_GOTO(res1, out, "reading properties from Token");
res1 = fpgaPropertiesGetBus(props, bus);
ON_ERR_GOTO(res1, out_destroy, "Reading bus from properties");
out_destroy:
res2 = fpgaDestroyProperties(&props);
ON_ERR_GOTO(res2, out, "fpgaDestroyProps");
out:
return res1 != FPGA_OK ? res1 : res2;
}
/* Is the FPGA simulated with ASE? */
bool probe_for_ase(void)
{
fpga_result r = FPGA_OK;
uint16_t device_id = 0;
fpga_properties filter = NULL;
uint32_t num_matches = 1;
fpga_token fme_token;
/* Connect to the FPGA management engine */
fpgaGetProperties(NULL, &filter);
fpgaPropertiesSetObjectType(filter, FPGA_DEVICE);
/* Connecting to one is sufficient to find ASE */
fpgaEnumerate(&filter, 1, &fme_token, 1, &num_matches);
if (0 != num_matches) {
/* Retrieve the device ID of the FME */
fpgaDestroyProperties(&filter);
fpgaGetProperties(fme_token, &filter);
r = fpgaPropertiesGetDeviceID(filter, &device_id);
fpgaDestroyToken(&fme_token);
}
fpgaDestroyProperties(&filter);
/* ASE's device ID is 0xa5e */
return ((FPGA_OK == r) && (0xa5e == device_id));
}
int main(int argc, char *argv[])
{
fpga_token accelerator_token;
fpga_handle accelerator_handle;
fpga_guid guid;
uint32_t num_matches_accelerators = 0;
uint32_t use_ase;
volatile uint64_t *dsm_ptr = NULL;
volatile uint64_t *status_ptr = NULL;
volatile uint64_t *input_ptr = NULL;
volatile uint64_t *output_ptr = NULL;
uint64_t dsm_wsid;
uint64_t input_wsid;
uint64_t output_wsid;
uint8_t bus = 0xff;
uint32_t i;
uint32_t timeout;
fpga_result res1 = FPGA_OK;
fpga_result res2 = FPGA_OK;
res1 = parse_args(argc, argv);
if ((int)res1 < 0)
goto out_exit;
ON_ERR_GOTO(res1, out_exit, "parsing arguments");
if (uuid_parse(NLB0_AFUID, guid) < 0) {
res1 = FPGA_EXCEPTION;
}
ON_ERR_GOTO(res1, out_exit, "parsing guid");
use_ase = probe_for_ase();
if (use_ase) {
printf("Running in ASE mode\n");
}
/* Look for accelerator with NLB0_AFUID */
res1 = find_fpga(guid, &accelerator_token, &num_matches_accelerators);
ON_ERR_GOTO(res1, out_exit, "finding FPGA accelerator");
if (num_matches_accelerators <= 0) {
res1 = FPGA_NOT_FOUND;
}
ON_ERR_GOTO(res1, out_exit, "no matching accelerator");
if (num_matches_accelerators > 1) {
printf("Found more than one suitable accelerator. ");
res1 = get_bus(accelerator_token, &bus);
ON_ERR_GOTO(res1, out_exit, "getting bus num");
printf("Running on bus 0x%02x.\n", bus);
}
/* Open accelerator and map MMIO */
res1 = fpgaOpen(accelerator_token, &accelerator_handle, config.open_flags);
ON_ERR_GOTO(res1, out_destroy_tok, "opening accelerator");
res1 = fpgaMapMMIO(accelerator_handle, 0, NULL);
ON_ERR_GOTO(res1, out_close, "mapping MMIO space");
/* Allocate buffers */
res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_DSM_SIZE,
(void **)&dsm_ptr, &dsm_wsid, 0);
ON_ERR_GOTO(res1, out_close, "allocating DSM buffer");
res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,
(void **)&input_ptr, &input_wsid, 0);
ON_ERR_GOTO(res1, out_free_dsm, "allocating input buffer");
res1 = fpgaPrepareBuffer(accelerator_handle, LPBK1_BUFFER_ALLOCATION_SIZE,
(void **)&output_ptr, &output_wsid, 0);
ON_ERR_GOTO(res1, out_free_input, "allocating output buffer");
printf("Running Test\n");
bus = 0xff;
res1 = get_bus(accelerator_token, &bus);
ON_ERR_GOTO(res1, out_free_output, "getting bus num");
printf("Running on bus 0x%02x.\n", bus);
/* Initialize buffers */
memset((void *)dsm_ptr, 0, LPBK1_DSM_SIZE);
memset((void *)input_ptr, 0xAF, LPBK1_BUFFER_SIZE);
memset((void *)output_ptr, 0xBE, LPBK1_BUFFER_SIZE);
cache_line *cl_ptr = (cache_line *)input_ptr;
for (i = 0; i < LPBK1_BUFFER_SIZE / CL(1); ++i) {
cl_ptr[i].uint[15] = i+1; /* set the last uint in every cacheline */
}
/* Reset accelerator */
res1 = fpgaReset(accelerator_handle);
ON_ERR_GOTO(res1, out_free_output, "resetting accelerator");
/* Program DMA addresses */
uint64_t iova = 0;
res1 = fpgaGetIOAddress(accelerator_handle, dsm_wsid, &iova);
ON_ERR_GOTO(res1, out_free_output, "getting DSM IOVA");
res1 = fpgaWriteMMIO64(accelerator_handle, 0, CSR_AFU_DSM_BASEL, iova);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_AFU_DSM_BASEL");
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_CTL, 0);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_CFG");
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_CTL, 1);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_CFG");
res1 = fpgaGetIOAddress(accelerator_handle, input_wsid, &iova);
ON_ERR_GOTO(res1, out_free_output, "getting input IOVA");
res1 = fpgaWriteMMIO64(accelerator_handle, 0, CSR_SRC_ADDR, CACHELINE_ALIGNED_ADDR(iova));
ON_ERR_GOTO(res1, out_free_output, "writing CSR_SRC_ADDR");
res1 = fpgaGetIOAddress(accelerator_handle, output_wsid, &iova);
ON_ERR_GOTO(res1, out_free_output, "getting output IOVA");
res1 = fpgaWriteMMIO64(accelerator_handle, 0, CSR_DST_ADDR, CACHELINE_ALIGNED_ADDR(iova));
ON_ERR_GOTO(res1, out_free_output, "writing CSR_DST_ADDR");
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_NUM_LINES, LPBK1_BUFFER_SIZE / CL(1));
ON_ERR_GOTO(res1, out_free_output, "writing CSR_NUM_LINES");
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_CFG, 0x42000);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_CFG");
status_ptr = dsm_ptr + DSM_STATUS_TEST_COMPLETE/sizeof(uint64_t);
/* Start the test */
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_CTL, 3);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_CFG");
/* Wait for test completion */
timeout = TEST_TIMEOUT;
while (0 == ((*status_ptr) & 0x1)) {
usleep(100);
if (!use_ase && (--timeout == 0)) {
res1 = FPGA_EXCEPTION;
ON_ERR_GOTO(res1, out_free_output, "test timed out");
}
}
/* Stop the device */
res1 = fpgaWriteMMIO32(accelerator_handle, 0, CSR_CTL, 7);
ON_ERR_GOTO(res1, out_free_output, "writing CSR_CFG");
/* Wait for the AFU's read/write traffic to complete */
uint32_t afu_traffic_trips = 0;
while (afu_traffic_trips < 100) {
/*
* CSR_STATUS1 holds two 32 bit values: num pending reads and writes.
* Wait for it to be 0.
*/
uint64_t s1;
res1 = fpgaReadMMIO64(accelerator_handle, 0, CSR_STATUS1, &s1);
ON_ERR_GOTO(res1, out_free_output, "reading CSR_STATUS1");
if (s1 == 0) {
break;
}
afu_traffic_trips += 1;
usleep(1000);
}
/* Check output buffer contents */
for (i = 0; i < LPBK1_BUFFER_SIZE; i++) {
if (((uint8_t *)output_ptr)[i] != ((uint8_t *)input_ptr)[i]) {
fprintf(stderr, "Output does NOT match input "
"at offset %i!\n", i);
break;
}
}
printf("Done Running Test\n");
/* Release buffers */
out_free_output:
res2 = fpgaReleaseBuffer(accelerator_handle, output_wsid);
ON_ERR_GOTO(res2, out_free_input, "releasing output buffer");
out_free_input:
res2 = fpgaReleaseBuffer(accelerator_handle, input_wsid);
ON_ERR_GOTO(res2, out_free_dsm, "releasing input buffer");
out_free_dsm:
res2 = fpgaReleaseBuffer(accelerator_handle, dsm_wsid);
ON_ERR_GOTO(res2, out_unmap, "releasing DSM buffer");
/* Unmap MMIO space */
out_unmap:
res2 = fpgaUnmapMMIO(accelerator_handle, 0);
ON_ERR_GOTO(res2, out_close, "unmapping MMIO space");
/* Release accelerator */
out_close:
res2 = fpgaClose(accelerator_handle);
ON_ERR_GOTO(res2, out_destroy_tok, "closing accelerator");
/* Destroy token */
out_destroy_tok:
res2 = fpgaDestroyToken(&accelerator_token);
ON_ERR_GOTO(res2, out_exit, "destroying token");
out_exit:
return res1 != FPGA_OK ? res1 : res2;
}