|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This file is provided under a dual BSD/GPLv2 license. When using or
|
|
Packit Service |
155747 |
redistributing this file, you may do so under either license.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
GPL LICENSE SUMMARY
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is free software; you can redistribute it and/or modify
|
|
Packit Service |
155747 |
it under the terms of version 2 of the GNU General Public License as
|
|
Packit Service |
155747 |
published by the Free Software Foundation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is distributed in the hope that it will be useful, but
|
|
Packit Service |
155747 |
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
155747 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit Service |
155747 |
General Public License for more details.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Contact Information:
|
|
Packit Service |
155747 |
Intel Corporation, www.intel.com
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
BSD LICENSE
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Redistribution and use in source and binary forms, with or without
|
|
Packit Service |
155747 |
modification, are permitted provided that the following conditions
|
|
Packit Service |
155747 |
are met:
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
* Redistributions of source code must retain the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer.
|
|
Packit Service |
155747 |
* Redistributions in binary form must reproduce the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer in
|
|
Packit Service |
155747 |
the documentation and/or other materials provided with the
|
|
Packit Service |
155747 |
distribution.
|
|
Packit Service |
155747 |
* Neither the name of Intel Corporation nor the names of its
|
|
Packit Service |
155747 |
contributors may be used to endorse or promote products derived
|
|
Packit Service |
155747 |
from this software without specific prior written permission.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
Packit Service |
155747 |
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
Packit Service |
155747 |
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
Packit Service |
155747 |
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
Packit Service |
155747 |
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
Packit Service |
155747 |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
Packit Service |
155747 |
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
Packit Service |
155747 |
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
Packit Service |
155747 |
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Copyright (c) 2003-2014 Intel Corporation. All rights reserved. */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include <sys/types.h>
|
|
Packit Service |
155747 |
#include <sys/uio.h>
|
|
Packit Service |
155747 |
#include <sys/syscall.h>
|
|
Packit Service |
155747 |
#include <unistd.h>
|
|
Packit Service |
155747 |
#include <errno.h>
|
|
Packit Service |
155747 |
#include <string.h>
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include "psm_user.h"
|
|
Packit Service |
155747 |
#include "cmarw.h"
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* An iovec looks like this:
|
|
Packit Service |
155747 |
* struct iovec {
|
|
Packit Service |
155747 |
* void *iov_base; // Starting address
|
|
Packit Service |
155747 |
* size_t iov_len; // Number of bytes to transfer
|
|
Packit Service |
155747 |
* };
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#if 0
|
|
Packit Service |
155747 |
#define __NR_process_vm_readv 310
|
|
Packit Service |
155747 |
#define __NR_process_vm_writev 311
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define process_vm_readv(pid, local_iov, liovcnt, remote_iov, riovcnt, flags) \
|
|
Packit Service |
155747 |
syscall(__NR_process_vm_readv, \
|
|
Packit Service |
155747 |
pid, local_iov, liovcnt, remote_iov, riovcnt, flags)
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define process_vm_writev(pid, local_iov, liovcnt, remote_iov, riovcnt, flags) \
|
|
Packit Service |
155747 |
syscall(__NR_process_vm_writev, \
|
|
Packit Service |
155747 |
pid, local_iov, liovcnt, remote_iov, riovcnt, flags)
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*CMA syscall wrappers were added in glibc 2.15. For anything older than that,
|
|
Packit Service |
155747 |
we need to define our own wrappers. Apparently older (and maybe newer?)
|
|
Packit Service |
155747 |
(2.12 from RHEL6.3 definitely has this bug) glibcs only pass up to 5
|
|
Packit Service |
155747 |
arguments via the generic syscall() function. These CMA functions, however,
|
|
Packit Service |
155747 |
have 6 arguments. So for now, we hack our way around it by generating ASM
|
|
Packit Service |
155747 |
code for doing a syscall directly.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#if defined(__GLIBC__) && ((__GLIBC__ == 2) && (__GLIBC_MINOR__ < 15))
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#ifdef __x86_64__
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define __NR_process_vm_readv 310
|
|
Packit Service |
155747 |
#define __NR_process_vm_writev 311
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static inline ssize_t __x86_64_syscall6(int syscall,
|
|
Packit Service |
155747 |
pid_t pid,
|
|
Packit Service |
155747 |
const struct iovec *local_iov,
|
|
Packit Service |
155747 |
unsigned long liovcnt,
|
|
Packit Service |
155747 |
const struct iovec *remote_iov,
|
|
Packit Service |
155747 |
unsigned long riovcnt,
|
|
Packit Service |
155747 |
unsigned long flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
/*GCC inline ASM is annoying -- can't specify all the x86_64 registers
|
|
Packit Service |
155747 |
directly, so declare register-specific variables and use them. */
|
|
Packit Service |
155747 |
register int64_t rax asm("rax") = syscall;
|
|
Packit Service |
155747 |
register int64_t rdi asm("rdi") = pid;
|
|
Packit Service |
155747 |
register int64_t rsi asm("rsi") = (intptr_t) local_iov;
|
|
Packit Service |
155747 |
register int64_t rdx asm("rdx") = liovcnt;
|
|
Packit Service |
155747 |
register int64_t r10 asm("r10") = (intptr_t) remote_iov;
|
|
Packit Service |
155747 |
register int64_t r8 asm("r8") = riovcnt;
|
|
Packit Service |
155747 |
register int64_t r9 asm("r9") = flags;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
asm volatile ("syscall\n" : "=a" (rax)
|
|
Packit Service |
155747 |
: "r"(rax), "r"(rdi), "r"(rsi), "r"(rdx), "r"(r10),
|
|
Packit Service |
155747 |
"r"(r8), "r"(r9)
|
|
Packit Service |
155747 |
: "%rcx", "%r11", "cc", "memory");
|
|
Packit Service |
155747 |
return rax;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define process_vm_readv(pid, local_iov, liovcnt, remote_iov, riovcnt, flags) \
|
|
Packit Service |
155747 |
__x86_64_syscall6(__NR_process_vm_readv, \
|
|
Packit Service |
155747 |
pid, local_iov, liovcnt, remote_iov, riovcnt, flags)
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define process_vm_writev(pid, local_iov, liovcnt, remote_iov, riovcnt, flags) \
|
|
Packit Service |
155747 |
__x86_64_syscall6(__NR_process_vm_writev, \
|
|
Packit Service |
155747 |
pid, local_iov, liovcnt, remote_iov, riovcnt, flags)
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#else /* ndef __x86_64__ */
|
|
Packit Service |
155747 |
#error "Can't compile CMA support for this architecture."
|
|
Packit Service |
155747 |
#endif /* __x86_64__ */
|
|
Packit Service |
155747 |
#endif /* __GLIBC__ < 2.15 */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int64_t cma_get(pid_t pid, const void *src, void *dst, int64_t n)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int64_t nr, sum;
|
|
Packit Service |
155747 |
struct iovec local = {
|
|
Packit Service |
155747 |
.iov_base = dst,
|
|
Packit Service |
155747 |
.iov_len = n
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
struct iovec remote = {
|
|
Packit Service |
155747 |
.iov_base = (void *)src,
|
|
Packit Service |
155747 |
.iov_len = n
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
nr = sum = 0;
|
|
Packit Service |
155747 |
while (sum != n) {
|
|
Packit Service |
155747 |
nr = process_vm_readv(pid, &local, 1, &remote, 1, 0);
|
|
Packit Service |
155747 |
if (nr == -1) {
|
|
Packit Service |
155747 |
return -1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
sum += nr;
|
|
Packit Service |
155747 |
local.iov_base += nr;
|
|
Packit Service |
155747 |
local.iov_len -= nr;
|
|
Packit Service |
155747 |
remote.iov_base += nr;
|
|
Packit Service |
155747 |
remote.iov_len -= nr;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return sum;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int64_t cma_put(const void *src, pid_t pid, void *dst, int64_t n)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int64_t nr, sum;
|
|
Packit Service |
155747 |
struct iovec local = {
|
|
Packit Service |
155747 |
.iov_base = (void *)src,
|
|
Packit Service |
155747 |
.iov_len = n
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
struct iovec remote = {
|
|
Packit Service |
155747 |
.iov_base = dst,
|
|
Packit Service |
155747 |
.iov_len = n
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
nr = sum = 0;
|
|
Packit Service |
155747 |
while (sum != n) {
|
|
Packit Service |
155747 |
nr = process_vm_writev(pid, &local, 1, &remote, 1, 0);
|
|
Packit Service |
155747 |
if (nr == -1) {
|
|
Packit Service |
155747 |
return -1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
sum += nr;
|
|
Packit Service |
155747 |
local.iov_base += nr;
|
|
Packit Service |
155747 |
local.iov_len -= nr;
|
|
Packit Service |
155747 |
remote.iov_base += nr;
|
|
Packit Service |
155747 |
remote.iov_len -= nr;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return sum;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Test if CMA is available by trying a no-op call. */
|
|
Packit Service |
155747 |
int cma_available(void)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Make a no-op CMA syscall. If CMA is present, 0 (bytes transferred)
|
|
Packit Service |
155747 |
* should be returned. If not present, expect -ENOSYS. */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int ret = process_vm_readv(getpid(), NULL, 0, NULL, 0, 0);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ret == 0) {
|
|
Packit Service |
155747 |
/* CMA is available! */
|
|
Packit Service |
155747 |
return 1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return 0;
|
|
Packit Service |
155747 |
}
|