|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This file is provided under a dual BSD/GPLv2 license. When using or
|
|
Packit Service |
155747 |
redistributing this file, you may do so under either license.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
GPL LICENSE SUMMARY
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is free software; you can redistribute it and/or modify
|
|
Packit Service |
155747 |
it under the terms of version 2 of the GNU General Public License as
|
|
Packit Service |
155747 |
published by the Free Software Foundation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is distributed in the hope that it will be useful, but
|
|
Packit Service |
155747 |
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
155747 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit Service |
155747 |
General Public License for more details.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Contact Information:
|
|
Packit Service |
155747 |
Intel Corporation, www.intel.com
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
BSD LICENSE
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Redistribution and use in source and binary forms, with or without
|
|
Packit Service |
155747 |
modification, are permitted provided that the following conditions
|
|
Packit Service |
155747 |
are met:
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
* Redistributions of source code must retain the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer.
|
|
Packit Service |
155747 |
* Redistributions in binary form must reproduce the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer in
|
|
Packit Service |
155747 |
the documentation and/or other materials provided with the
|
|
Packit Service |
155747 |
distribution.
|
|
Packit Service |
155747 |
* Neither the name of Intel Corporation nor the names of its
|
|
Packit Service |
155747 |
contributors may be used to endorse or promote products derived
|
|
Packit Service |
155747 |
from this software without specific prior written permission.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
Packit Service |
155747 |
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
Packit Service |
155747 |
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
Packit Service |
155747 |
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
Packit Service |
155747 |
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
Packit Service |
155747 |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
Packit Service |
155747 |
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
Packit Service |
155747 |
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
Packit Service |
155747 |
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Copyright (c) 2003-2014 Intel Corporation. All rights reserved. */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* This file contains the initialization functions used by the low
|
|
Packit Service |
155747 |
level hfi protocol code. */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include <sys/poll.h>
|
|
Packit Service |
155747 |
#include <sys/types.h>
|
|
Packit Service |
155747 |
#include <stdint.h>
|
|
Packit Service |
155747 |
#include <stdlib.h>
|
|
Packit Service |
155747 |
#include <stddef.h>
|
|
Packit Service |
155747 |
#include <unistd.h>
|
|
Packit Service |
155747 |
#include <errno.h>
|
|
Packit Service |
155747 |
#include <string.h>
|
|
Packit Service |
155747 |
#include <stdio.h>
|
|
Packit Service |
155747 |
#include <fcntl.h>
|
|
Packit Service |
155747 |
#include <malloc.h>
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include "hfi_user.h"
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* These pio copy routines are here so they can be used by test code, as well
|
|
Packit Service |
155747 |
* as by MPI, and can change independently of MPI
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* for processors that may not write store buffers in the order filled,
|
|
Packit Service |
155747 |
* and when the store buffer is not completely filled (partial at end, or
|
|
Packit Service |
155747 |
* interrupted and flushed) may write the partial buffer in
|
|
Packit Service |
155747 |
* "random" order. requires additional serialization
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
void hfi_write_pio_force_order(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm, void *hdr,
|
|
Packit Service |
155747 |
void *bdata)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
union hfi_pbc buf = {.qword = 0 };
|
|
Packit Service |
155747 |
uint32_t cksum_len = pioparm->cksum_is_valid ?
|
|
Packit Service |
155747 |
HFI_CRC_SIZE_IN_BYTES : 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
buf.length =
|
|
Packit Service |
155747 |
__cpu_to_le16(((HFI_MESSAGE_HDR_SIZE + cksum_len +
|
|
Packit Service |
155747 |
pioparm->length) >> 2) + 1);
|
|
Packit Service |
155747 |
if (pioparm->port > 1)
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32(pioparm->vl << __PBC_VLSHIFT |
|
|
Packit Service |
155747 |
__PBC_IBPORT | pioparm->rate);
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32(pioparm->vl << __PBC_VLSHIFT |
|
|
Packit Service |
155747 |
pioparm->rate);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*piob++ = buf.dword;
|
|
Packit Service |
155747 |
/* 32 bit programs require fence after first 32 bits of pbc write */
|
|
Packit Service |
155747 |
/* Can't do as uint64_t store, or compiler could reorder */
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
*piob++ = buf.pbcflags;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (!pioparm->length) {
|
|
Packit Service |
155747 |
uint32_t *dhdr, dcpywords;
|
|
Packit Service |
155747 |
dcpywords = (HFI_MESSAGE_HDR_SIZE >> 2) - 1;
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, hdr, dcpywords);
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
dhdr = hdr;
|
|
Packit Service |
155747 |
piob += dcpywords;
|
|
Packit Service |
155747 |
dhdr += dcpywords;
|
|
Packit Service |
155747 |
*piob++ = *dhdr;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
uint32_t *pay2 = bdata, j;
|
|
Packit Service |
155747 |
uint32_t len = pioparm->length;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, hdr, HFI_MESSAGE_HDR_SIZE >> 2);
|
|
Packit Service |
155747 |
piob += HFI_MESSAGE_HDR_SIZE >> 2;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
len >>= 2;
|
|
Packit Service |
155747 |
if (len > 16) {
|
|
Packit Service |
155747 |
uint32_t pay_words = 16 * ((len - 1) / 16);
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, pay2, pay_words);
|
|
Packit Service |
155747 |
piob += pay_words;
|
|
Packit Service |
155747 |
pay2 += pay_words;
|
|
Packit Service |
155747 |
len -= pay_words;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
/* now write the final chunk a word at a time, fence before trigger */
|
|
Packit Service |
155747 |
for (j = 0; j < (len - 1); j++)
|
|
Packit Service |
155747 |
*piob++ = *pay2++;
|
|
Packit Service |
155747 |
ips_wmb(); /* flush the buffer out now, so */
|
|
Packit Service |
155747 |
*piob++ = *pay2;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* If checksum is enabled insert CRC at end of packet */
|
|
Packit Service |
155747 |
if_pf(pioparm->cksum_is_valid) {
|
|
Packit Service |
155747 |
int nCRCopies = HFI_CRC_SIZE_IN_BYTES >> 2;
|
|
Packit Service |
155747 |
int nCRC = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
while (nCRC < (nCRCopies - 1)) {
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
piob++;
|
|
Packit Service |
155747 |
nCRC++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* send it on it's way, now, rather than waiting for processor to
|
|
Packit Service |
155747 |
* get around to flushing it */
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* for processors that always write store buffers in the order filled,
|
|
Packit Service |
155747 |
* and if store buffer not completely filled (partial at end, or
|
|
Packit Service |
155747 |
* interrupted and flushed) always write the partial buffer in
|
|
Packit Service |
155747 |
* address order. Avoids serializing and flush instructions
|
|
Packit Service |
155747 |
* where possible.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
void hfi_write_pio(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm, void *hdr, void *bdata)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
union hfi_pbc buf = { 0 };
|
|
Packit Service |
155747 |
uint32_t cksum_len = pioparm->cksum_is_valid ?
|
|
Packit Service |
155747 |
HFI_CRC_SIZE_IN_BYTES : 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
buf.length =
|
|
Packit Service |
155747 |
__cpu_to_le16(((HFI_MESSAGE_HDR_SIZE + cksum_len +
|
|
Packit Service |
155747 |
pioparm->length) >> 2) + 1);
|
|
Packit Service |
155747 |
if (pioparm->port > 1)
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32((pioparm->vl << __PBC_VLSHIFT) |
|
|
Packit Service |
155747 |
__PBC_IBPORT | pioparm->rate);
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32(pioparm->vl << __PBC_VLSHIFT |
|
|
Packit Service |
155747 |
pioparm->rate);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*piob++ = buf.dword;
|
|
Packit Service |
155747 |
/* 32 bit programs needs compiler fence to prevent compiler reordering
|
|
Packit Service |
155747 |
the two 32 bit stores in a uint64_t, but on inorder wc systems, does
|
|
Packit Service |
155747 |
not need a memory fence. */
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
*piob++ = buf.pbcflags;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, hdr, HFI_MESSAGE_HDR_SIZE >> 2);
|
|
Packit Service |
155747 |
piob += HFI_MESSAGE_HDR_SIZE >> 2;
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (pioparm->length)
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, (uint32_t *) bdata,
|
|
Packit Service |
155747 |
pioparm->length >> 2);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* If checksum is enabled insert CRC at end of packet */
|
|
Packit Service |
155747 |
if_pf(pioparm->cksum_is_valid) {
|
|
Packit Service |
155747 |
int nCRCopies = HFI_CRC_SIZE_IN_BYTES >> 2;
|
|
Packit Service |
155747 |
int nCRC = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
piob += pioparm->length >> 2;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
while (nCRC < (nCRCopies - 1)) {
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
piob++;
|
|
Packit Service |
155747 |
nCRC++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* send it on it's way, now, rather than waiting for processor to
|
|
Packit Service |
155747 |
* get around to flushing it */
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* for processors that always write store buffers in the order filled,
|
|
Packit Service |
155747 |
* and if store buffer not completely filled (partial at end, or
|
|
Packit Service |
155747 |
* interrupted and flushed) always write the partial buffer in
|
|
Packit Service |
155747 |
* address order. Avoids serializing and flush instructions
|
|
Packit Service |
155747 |
* where possible.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
static void hfi_write_pio_special_trigger(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm,
|
|
Packit Service |
155747 |
void *hdr, void *bdata,
|
|
Packit Service |
155747 |
unsigned offset)
|
|
Packit Service |
155747 |
__attribute__ ((always_inline));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static void hfi_write_pio_special_trigger(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm,
|
|
Packit Service |
155747 |
void *hdr, void *bdata,
|
|
Packit Service |
155747 |
unsigned offset)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
union hfi_pbc buf = { 0 };
|
|
Packit Service |
155747 |
volatile uint32_t *piobs = piob;
|
|
Packit Service |
155747 |
uint32_t cksum_len = pioparm->cksum_is_valid ?
|
|
Packit Service |
155747 |
HFI_CRC_SIZE_IN_BYTES : 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
buf.length =
|
|
Packit Service |
155747 |
__cpu_to_le16(((HFI_MESSAGE_HDR_SIZE + cksum_len +
|
|
Packit Service |
155747 |
pioparm->length) >> 2) + 1);
|
|
Packit Service |
155747 |
if (pioparm->port > 1)
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32((pioparm->vl << __PBC_VLSHIFT) |
|
|
Packit Service |
155747 |
__PBC_IBPORT | pioparm->rate);
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
buf.pbcflags = __cpu_to_le32(pioparm->vl << __PBC_VLSHIFT |
|
|
Packit Service |
155747 |
pioparm->rate);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*piob++ = buf.dword;
|
|
Packit Service |
155747 |
/* 32 bit programs needs compiler fence to prevent compiler reordering
|
|
Packit Service |
155747 |
the two 32 bit stores in a uint64_t, but on inorder wc systems, does
|
|
Packit Service |
155747 |
not need a memory fence. */
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
*piob++ = buf.pbcflags;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, hdr, HFI_MESSAGE_HDR_SIZE >> 2);
|
|
Packit Service |
155747 |
piob += HFI_MESSAGE_HDR_SIZE >> 2;
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (pioparm->length)
|
|
Packit Service |
155747 |
hfi_dwordcpy_safe(piob, (uint32_t *) bdata,
|
|
Packit Service |
155747 |
pioparm->length >> 2);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* If checksum is enabled insert CRC at end of packet */
|
|
Packit Service |
155747 |
if_pf(pioparm->cksum_is_valid) {
|
|
Packit Service |
155747 |
int nCRCopies = HFI_CRC_SIZE_IN_BYTES >> 2;
|
|
Packit Service |
155747 |
int nCRC = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
piob += pioparm->length >> 2;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
while (nCRC < (nCRCopies - 1)) {
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
piob++;
|
|
Packit Service |
155747 |
nCRC++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
asm volatile ("" : : : "memory");
|
|
Packit Service |
155747 |
*piob = pioparm->cksum;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* send it on it's way, now, rather than waiting for processor to
|
|
Packit Service |
155747 |
* get around to flushing it */
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
*(piobs + offset) = HFI_SPECIAL_TRIGGER_MAGIC;
|
|
Packit Service |
155747 |
ips_wmb();
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void hfi_write_pio_special_trigger2k(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm,
|
|
Packit Service |
155747 |
void *hdr, void *bdata)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
hfi_write_pio_special_trigger(piob, pioparm, hdr, bdata, 1023);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void hfi_write_pio_special_trigger4k(volatile uint32_t *piob,
|
|
Packit Service |
155747 |
const struct hfi_pio_params *pioparm,
|
|
Packit Service |
155747 |
void *hdr, void *bdata)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
hfi_write_pio_special_trigger(piob, pioparm, hdr, bdata, 2047);
|
|
Packit Service |
155747 |
}
|