|
Packit |
6c4009 |
/* sem_waitcommon -- wait on a semaphore, shared code.
|
|
Packit |
6c4009 |
Copyright (C) 2003-2018 Free Software Foundation, Inc.
|
|
Packit |
6c4009 |
This file is part of the GNU C Library.
|
|
Packit |
6c4009 |
Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is free software; you can redistribute it and/or
|
|
Packit |
6c4009 |
modify it under the terms of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License as published by the Free Software Foundation; either
|
|
Packit |
6c4009 |
version 2.1 of the License, or (at your option) any later version.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit |
6c4009 |
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
6c4009 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
6c4009 |
Lesser General Public License for more details.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
You should have received a copy of the GNU Lesser General Public
|
|
Packit |
6c4009 |
License along with the GNU C Library; if not, see
|
|
Packit |
6c4009 |
<http://www.gnu.org/licenses/>. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <kernel-features.h>
|
|
Packit |
6c4009 |
#include <errno.h>
|
|
Packit |
6c4009 |
#include <sysdep.h>
|
|
Packit |
6c4009 |
#include <futex-internal.h>
|
|
Packit |
6c4009 |
#include <internaltypes.h>
|
|
Packit |
6c4009 |
#include <semaphore.h>
|
|
Packit |
6c4009 |
#include <sys/time.h>
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#include <pthreadP.h>
|
|
Packit |
6c4009 |
#include <shlib-compat.h>
|
|
Packit |
6c4009 |
#include <atomic.h>
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* The semaphore provides two main operations: sem_post adds a token to the
|
|
Packit |
6c4009 |
semaphore; sem_wait grabs a token from the semaphore, potentially waiting
|
|
Packit |
6c4009 |
until there is a token available. A sem_wait needs to synchronize with
|
|
Packit |
6c4009 |
the sem_post that provided the token, so that whatever lead to the sem_post
|
|
Packit |
6c4009 |
happens before the code after sem_wait.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
Conceptually, available tokens can simply be counted; let's call that the
|
|
Packit |
6c4009 |
value of the semaphore. However, we also want to know whether there might
|
|
Packit |
6c4009 |
be a sem_wait that is blocked on the value because it was zero (using a
|
|
Packit |
6c4009 |
futex with the value being the futex variable); if there is no blocked
|
|
Packit |
6c4009 |
sem_wait, sem_post does not need to execute a futex_wake call. Therefore,
|
|
Packit |
6c4009 |
we also need to count the number of potentially blocked sem_wait calls
|
|
Packit |
6c4009 |
(which we call nwaiters).
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
What makes this tricky is that POSIX requires that a semaphore can be
|
|
Packit |
6c4009 |
destroyed as soon as the last remaining sem_wait has returned, and no
|
|
Packit |
6c4009 |
other sem_wait or sem_post calls are executing concurrently. However, the
|
|
Packit |
6c4009 |
sem_post call whose token was consumed by the last sem_wait is considered
|
|
Packit |
6c4009 |
to have finished once it provided the token to the sem_wait.
|
|
Packit |
6c4009 |
Thus, sem_post must not access the semaphore struct anymore after it has
|
|
Packit |
6c4009 |
made a token available; IOW, it needs to be able to atomically provide
|
|
Packit |
6c4009 |
a token and check whether any blocked sem_wait calls might exist.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
This is straightforward to do if the architecture provides 64b atomics
|
|
Packit |
6c4009 |
because we can just put both the value and nwaiters into one variable that
|
|
Packit |
6c4009 |
we access atomically: This is the data field, the value is in the
|
|
Packit |
6c4009 |
least-significant 32 bits, and nwaiters in the other bits. When sem_post
|
|
Packit |
6c4009 |
makes a value available, it can atomically check nwaiters.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
If we have only 32b atomics available, we cannot put both nwaiters and
|
|
Packit |
6c4009 |
value into one 32b value because then we might have too few bits for both
|
|
Packit |
6c4009 |
of those counters. Therefore, we need to use two distinct fields.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
To allow sem_post to atomically make a token available and check for
|
|
Packit |
6c4009 |
blocked sem_wait calls, we use one bit in value to indicate whether
|
|
Packit |
6c4009 |
nwaiters is nonzero. That allows sem_post to use basically the same
|
|
Packit |
6c4009 |
algorithm as with 64b atomics, but requires sem_wait to update the bit; it
|
|
Packit |
6c4009 |
can't do this atomically with another access to nwaiters, but it can compute
|
|
Packit |
6c4009 |
a conservative value for the bit because it's benign if the bit is set
|
|
Packit |
6c4009 |
even if nwaiters is zero (all we get is an unnecessary futex wake call by
|
|
Packit |
6c4009 |
sem_post).
|
|
Packit |
6c4009 |
Specifically, sem_wait will unset the bit speculatively if it believes that
|
|
Packit |
6c4009 |
there is no other concurrently executing sem_wait. If it misspeculated,
|
|
Packit |
6c4009 |
it will have to clean up by waking any other sem_wait call (i.e., what
|
|
Packit |
6c4009 |
sem_post would do otherwise). This does not conflict with the destruction
|
|
Packit |
6c4009 |
requirement because the semaphore must not be destructed while any sem_wait
|
|
Packit |
6c4009 |
is still executing. */
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if !__HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
static void
|
|
Packit |
6c4009 |
__sem_wait_32_finish (struct new_sem *sem);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
static void
|
|
Packit |
6c4009 |
__sem_wait_cleanup (void *arg)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
struct new_sem *sem = (struct new_sem *) arg;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if __HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
/* Stop being registered as a waiter. See below for MO. */
|
|
Packit |
6c4009 |
atomic_fetch_add_relaxed (&sem->data, -((uint64_t) 1 << SEM_NWAITERS_SHIFT));
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
__sem_wait_32_finish (sem);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Wait until at least one token is available, possibly with a timeout.
|
|
Packit |
6c4009 |
This is in a separate function in order to make sure gcc
|
|
Packit |
6c4009 |
puts the call site into an exception region, and thus the
|
|
Packit |
6c4009 |
cleanups get properly run. TODO still necessary? Other futex_wait
|
|
Packit |
6c4009 |
users don't seem to need it. */
|
|
Packit |
6c4009 |
static int
|
|
Packit |
6c4009 |
__attribute__ ((noinline))
|
|
Packit |
6c4009 |
do_futex_wait (struct new_sem *sem, const struct timespec *abstime)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
int err;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if __HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
err = futex_abstimed_wait_cancelable (
|
|
Packit |
6c4009 |
(unsigned int *) &sem->data + SEM_VALUE_OFFSET, 0, abstime,
|
|
Packit |
6c4009 |
sem->private);
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
err = futex_abstimed_wait_cancelable (&sem->value, SEM_NWAITERS_MASK,
|
|
Packit |
6c4009 |
abstime, sem->private);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return err;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Fast path: Try to grab a token without blocking. */
|
|
Packit |
6c4009 |
static int
|
|
Packit |
6c4009 |
__new_sem_wait_fast (struct new_sem *sem, int definitive_result)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* We need acquire MO if we actually grab a token, so that this
|
|
Packit |
6c4009 |
synchronizes with all token providers (i.e., the RMW operation we read
|
|
Packit |
6c4009 |
from or all those before it in modification order; also see sem_post).
|
|
Packit |
6c4009 |
We do not need to guarantee any ordering if we observed that there is
|
|
Packit |
6c4009 |
no token (POSIX leaves it unspecified whether functions that fail
|
|
Packit |
6c4009 |
synchronize memory); thus, relaxed MO is sufficient for the initial load
|
|
Packit |
6c4009 |
and the failure path of the CAS. If the weak CAS fails and we need a
|
|
Packit |
6c4009 |
definitive result, retry. */
|
|
Packit |
6c4009 |
#if __HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
uint64_t d = atomic_load_relaxed (&sem->data);
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if ((d & SEM_VALUE_MASK) == 0)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
if (atomic_compare_exchange_weak_acquire (&sem->data, &d, d - 1))
|
|
Packit |
6c4009 |
return 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (definitive_result);
|
|
Packit |
6c4009 |
return -1;
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
unsigned int v = atomic_load_relaxed (&sem->value);
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if ((v >> SEM_VALUE_SHIFT) == 0)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
if (atomic_compare_exchange_weak_acquire (&sem->value,
|
|
Packit |
6c4009 |
&v, v - (1 << SEM_VALUE_SHIFT)))
|
|
Packit |
6c4009 |
return 0;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (definitive_result);
|
|
Packit |
6c4009 |
return -1;
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Slow path that blocks. */
|
|
Packit |
6c4009 |
static int
|
|
Packit |
6c4009 |
__attribute__ ((noinline))
|
|
Packit |
6c4009 |
__new_sem_wait_slow (struct new_sem *sem, const struct timespec *abstime)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
int err = 0;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
#if __HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
/* Add a waiter. Relaxed MO is sufficient because we can rely on the
|
|
Packit |
6c4009 |
ordering provided by the RMW operations we use. */
|
|
Packit |
6c4009 |
uint64_t d = atomic_fetch_add_relaxed (&sem->data,
|
|
Packit |
6c4009 |
(uint64_t) 1 << SEM_NWAITERS_SHIFT);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
pthread_cleanup_push (__sem_wait_cleanup, sem);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Wait for a token to be available. Retry until we can grab one. */
|
|
Packit |
6c4009 |
for (;;)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* If there is no token available, sleep until there is. */
|
|
Packit |
6c4009 |
if ((d & SEM_VALUE_MASK) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
err = do_futex_wait (sem, abstime);
|
|
Packit |
6c4009 |
/* A futex return value of 0 or EAGAIN is due to a real or spurious
|
|
Packit |
6c4009 |
wake-up, or due to a change in the number of tokens. We retry in
|
|
Packit |
6c4009 |
these cases.
|
|
Packit |
6c4009 |
If we timed out, forward this to the caller.
|
|
Packit |
6c4009 |
EINTR is returned if we are interrupted by a signal; we
|
|
Packit |
6c4009 |
forward this to the caller. (See futex_wait and related
|
|
Packit |
6c4009 |
documentation. Before Linux 2.6.22, EINTR was also returned on
|
|
Packit |
6c4009 |
spurious wake-ups; we only support more recent Linux versions,
|
|
Packit |
6c4009 |
so do not need to consider this here.) */
|
|
Packit |
6c4009 |
if (err == ETIMEDOUT || err == EINTR)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
__set_errno (err);
|
|
Packit |
6c4009 |
err = -1;
|
|
Packit |
6c4009 |
/* Stop being registered as a waiter. */
|
|
Packit |
6c4009 |
atomic_fetch_add_relaxed (&sem->data,
|
|
Packit |
6c4009 |
-((uint64_t) 1 << SEM_NWAITERS_SHIFT));
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
/* Relaxed MO is sufficient; see below. */
|
|
Packit |
6c4009 |
d = atomic_load_relaxed (&sem->data);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
else
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* Try to grab both a token and stop being a waiter. We need
|
|
Packit |
6c4009 |
acquire MO so this synchronizes with all token providers (i.e.,
|
|
Packit |
6c4009 |
the RMW operation we read from or all those before it in
|
|
Packit |
6c4009 |
modification order; also see sem_post). On the failure path,
|
|
Packit |
6c4009 |
relaxed MO is sufficient because we only eventually need the
|
|
Packit |
6c4009 |
up-to-date value; the futex_wait or the CAS perform the real
|
|
Packit |
6c4009 |
work. */
|
|
Packit |
6c4009 |
if (atomic_compare_exchange_weak_acquire (&sem->data,
|
|
Packit |
6c4009 |
&d, d - 1 - ((uint64_t) 1 << SEM_NWAITERS_SHIFT)))
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
err = 0;
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
pthread_cleanup_pop (0);
|
|
Packit |
6c4009 |
#else
|
|
Packit |
6c4009 |
/* The main difference to the 64b-atomics implementation is that we need to
|
|
Packit |
6c4009 |
access value and nwaiters in separate steps, and that the nwaiters bit
|
|
Packit |
6c4009 |
in the value can temporarily not be set even if nwaiters is nonzero.
|
|
Packit |
6c4009 |
We work around incorrectly unsetting the nwaiters bit by letting sem_wait
|
|
Packit |
6c4009 |
set the bit again and waking the number of waiters that could grab a
|
|
Packit |
6c4009 |
token. There are two additional properties we need to ensure:
|
|
Packit |
6c4009 |
(1) We make sure that whenever unsetting the bit, we see the increment of
|
|
Packit |
6c4009 |
nwaiters by the other thread that set the bit. IOW, we will notice if
|
|
Packit |
6c4009 |
we make a mistake.
|
|
Packit |
6c4009 |
(2) When setting the nwaiters bit, we make sure that we see the unsetting
|
|
Packit |
6c4009 |
of the bit by another waiter that happened before us. This avoids having
|
|
Packit |
6c4009 |
to blindly set the bit whenever we need to block on it. We set/unset
|
|
Packit |
6c4009 |
the bit while having incremented nwaiters (i.e., are a registered
|
|
Packit |
6c4009 |
waiter), and the problematic case only happens when one waiter indeed
|
|
Packit |
6c4009 |
followed another (i.e., nwaiters was never larger than 1); thus, this
|
|
Packit |
6c4009 |
works similarly as with a critical section using nwaiters (see the MOs
|
|
Packit |
6c4009 |
and related comments below).
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
An alternative approach would be to unset the bit after decrementing
|
|
Packit |
6c4009 |
nwaiters; however, that would result in needing Dekker-like
|
|
Packit |
6c4009 |
synchronization and thus full memory barriers. We also would not be able
|
|
Packit |
6c4009 |
to prevent misspeculation, so this alternative scheme does not seem
|
|
Packit |
6c4009 |
beneficial. */
|
|
Packit |
6c4009 |
unsigned int v;
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Add a waiter. We need acquire MO so this synchronizes with the release
|
|
Packit |
6c4009 |
MO we use when decrementing nwaiters below; it ensures that if another
|
|
Packit |
6c4009 |
waiter unset the bit before us, we see that and set it again. Also see
|
|
Packit |
6c4009 |
property (2) above. */
|
|
Packit |
6c4009 |
atomic_fetch_add_acquire (&sem->nwaiters, 1);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
pthread_cleanup_push (__sem_wait_cleanup, sem);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Wait for a token to be available. Retry until we can grab one. */
|
|
Packit |
6c4009 |
/* We do not need any ordering wrt. to this load's reads-from, so relaxed
|
|
Packit |
6c4009 |
MO is sufficient. The acquire MO above ensures that in the problematic
|
|
Packit |
6c4009 |
case, we do see the unsetting of the bit by another waiter. */
|
|
Packit |
6c4009 |
v = atomic_load_relaxed (&sem->value);
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* We are about to block, so make sure that the nwaiters bit is
|
|
Packit |
6c4009 |
set. We need release MO on the CAS to ensure that when another
|
|
Packit |
6c4009 |
waiter unsets the nwaiters bit, it will also observe that we
|
|
Packit |
6c4009 |
incremented nwaiters in the meantime (also see the unsetting of
|
|
Packit |
6c4009 |
the bit below). Relaxed MO on CAS failure is sufficient (see
|
|
Packit |
6c4009 |
above). */
|
|
Packit |
6c4009 |
do
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
if ((v & SEM_NWAITERS_MASK) != 0)
|
|
Packit |
6c4009 |
break;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
while (!atomic_compare_exchange_weak_release (&sem->value,
|
|
Packit |
6c4009 |
&v, v | SEM_NWAITERS_MASK));
|
|
Packit |
6c4009 |
/* If there is no token, wait. */
|
|
Packit |
6c4009 |
if ((v >> SEM_VALUE_SHIFT) == 0)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* See __HAVE_64B_ATOMICS variant. */
|
|
Packit |
6c4009 |
err = do_futex_wait(sem, abstime);
|
|
Packit |
6c4009 |
if (err == ETIMEDOUT || err == EINTR)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
__set_errno (err);
|
|
Packit |
6c4009 |
err = -1;
|
|
Packit |
6c4009 |
goto error;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
err = 0;
|
|
Packit |
6c4009 |
/* We blocked, so there might be a token now. Relaxed MO is
|
|
Packit |
6c4009 |
sufficient (see above). */
|
|
Packit |
6c4009 |
v = atomic_load_relaxed (&sem->value);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
/* If there is no token, we must not try to grab one. */
|
|
Packit |
6c4009 |
while ((v >> SEM_VALUE_SHIFT) == 0);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
/* Try to grab a token. We need acquire MO so this synchronizes with
|
|
Packit |
6c4009 |
all token providers (i.e., the RMW operation we read from or all those
|
|
Packit |
6c4009 |
before it in modification order; also see sem_post). */
|
|
Packit |
6c4009 |
while (!atomic_compare_exchange_weak_acquire (&sem->value,
|
|
Packit |
6c4009 |
&v, v - (1 << SEM_VALUE_SHIFT)));
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
error:
|
|
Packit |
6c4009 |
pthread_cleanup_pop (0);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
__sem_wait_32_finish (sem);
|
|
Packit |
6c4009 |
#endif
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
return err;
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Stop being a registered waiter (non-64b-atomics code only). */
|
|
Packit |
6c4009 |
#if !__HAVE_64B_ATOMICS
|
|
Packit |
6c4009 |
static void
|
|
Packit |
6c4009 |
__sem_wait_32_finish (struct new_sem *sem)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* The nwaiters bit is still set, try to unset it now if this seems
|
|
Packit |
6c4009 |
necessary. We do this before decrementing nwaiters so that the unsetting
|
|
Packit |
6c4009 |
is visible to other waiters entering after us. Relaxed MO is sufficient
|
|
Packit |
6c4009 |
because we are just speculating here; a stronger MO would not prevent
|
|
Packit |
6c4009 |
misspeculation. */
|
|
Packit |
6c4009 |
unsigned int wguess = atomic_load_relaxed (&sem->nwaiters);
|
|
Packit |
6c4009 |
if (wguess == 1)
|
|
Packit |
6c4009 |
/* We might be the last waiter, so unset. This needs acquire MO so that
|
|
Packit |
6c4009 |
it syncronizes with the release MO when setting the bit above; if we
|
|
Packit |
6c4009 |
overwrite someone else that set the bit, we'll read in the following
|
|
Packit |
6c4009 |
decrement of nwaiters at least from that release sequence, so we'll
|
|
Packit |
6c4009 |
see if the other waiter is still active or if another writer entered
|
|
Packit |
6c4009 |
in the meantime (i.e., using the check below). */
|
|
Packit |
6c4009 |
atomic_fetch_and_acquire (&sem->value, ~SEM_NWAITERS_MASK);
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
/* Now stop being a waiter, and see whether our guess was correct.
|
|
Packit |
6c4009 |
This needs release MO so that it synchronizes with the acquire MO when
|
|
Packit |
6c4009 |
a waiter increments nwaiters; this makes sure that newer writers see that
|
|
Packit |
6c4009 |
we reset the waiters_present bit. */
|
|
Packit |
6c4009 |
unsigned int wfinal = atomic_fetch_add_release (&sem->nwaiters, -1);
|
|
Packit |
6c4009 |
if (wfinal > 1 && wguess == 1)
|
|
Packit |
6c4009 |
{
|
|
Packit |
6c4009 |
/* We guessed wrong, and so need to clean up after the mistake and
|
|
Packit |
6c4009 |
unblock any waiters that could have not been woken. There is no
|
|
Packit |
6c4009 |
additional ordering that we need to set up, so relaxed MO is
|
|
Packit |
6c4009 |
sufficient. */
|
|
Packit |
6c4009 |
unsigned int v = atomic_fetch_or_relaxed (&sem->value,
|
|
Packit |
6c4009 |
SEM_NWAITERS_MASK);
|
|
Packit |
6c4009 |
/* If there are available tokens, then wake as many waiters. If there
|
|
Packit |
6c4009 |
aren't any, then there is no need to wake anyone because there is
|
|
Packit |
6c4009 |
none to grab for another waiter. If tokens become available
|
|
Packit |
6c4009 |
subsequently, then the respective sem_post calls will do the wake-up
|
|
Packit |
6c4009 |
due to us having set the nwaiters bit again. */
|
|
Packit |
6c4009 |
v >>= SEM_VALUE_SHIFT;
|
|
Packit |
6c4009 |
if (v > 0)
|
|
Packit |
6c4009 |
futex_wake (&sem->value, v, sem->private);
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
#endif
|