/* Copyright (c) 2006, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* ---
* Author: Sanjay Ghemawat
*/
#include <config.h>
#include "base/spinlock.h"
#include "base/synchronization_profiling.h"
#include "base/spinlock_internal.h"
#include "base/cycleclock.h"
#include "base/sysinfo.h" /* for NumCPUs() */
// NOTE on the Lock-state values:
//
// kSpinLockFree represents the unlocked state
// kSpinLockHeld represents the locked state with no waiters
//
// Values greater than kSpinLockHeld represent the locked state with waiters,
// where the value is the time the current lock holder had to
// wait before obtaining the lock. The kSpinLockSleeper state is a special
// "locked with waiters" state that indicates that a sleeper needs to
// be woken, but the thread that just released the lock didn't wait.
static int adaptive_spin_count = 0;
const base::LinkerInitialized SpinLock::LINKER_INITIALIZED =
base::LINKER_INITIALIZED;
namespace {
struct SpinLock_InitHelper {
SpinLock_InitHelper() {
// On multi-cpu machines, spin for longer before yielding
// the processor or sleeping. Reduces idle time significantly.
if (NumCPUs() > 1) {
adaptive_spin_count = 1000;
}
}
};
// Hook into global constructor execution:
// We do not do adaptive spinning before that,
// but nothing lock-intensive should be going on at that time.
static SpinLock_InitHelper init_helper;
} // unnamed namespace
// Monitor the lock to see if its value changes within some time period
// (adaptive_spin_count loop iterations). A timestamp indicating
// when the thread initially started waiting for the lock is passed in via
// the initial_wait_timestamp value. The total wait time in cycles for the
// lock is returned in the wait_cycles parameter. The last value read
// from the lock is returned from the method.
Atomic32 SpinLock::SpinLoop(int64 initial_wait_timestamp,
Atomic32* wait_cycles) {
int c = adaptive_spin_count;
while (base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree && --c > 0) {
}
Atomic32 spin_loop_wait_cycles = CalculateWaitCycles(initial_wait_timestamp);
Atomic32 lock_value =
base::subtle::Acquire_CompareAndSwap(&lockword_, kSpinLockFree,
spin_loop_wait_cycles);
*wait_cycles = spin_loop_wait_cycles;
return lock_value;
}
void SpinLock::SlowLock() {
// The lock was not obtained initially, so this thread needs to wait for
// it. Record the current timestamp in the local variable wait_start_time
// so the total wait time can be stored in the lockword once this thread
// obtains the lock.
int64 wait_start_time = CycleClock::Now();
Atomic32 wait_cycles;
Atomic32 lock_value = SpinLoop(wait_start_time, &wait_cycles);
int lock_wait_call_count = 0;
while (lock_value != kSpinLockFree) {
// If the lock is currently held, but not marked as having a sleeper, mark
// it as having a sleeper.
if (lock_value == kSpinLockHeld) {
// Here, just "mark" that the thread is going to sleep. Don't store the
// lock wait time in the lock as that will cause the current lock
// owner to think it experienced contention.
lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_,
kSpinLockHeld,
kSpinLockSleeper);
if (lock_value == kSpinLockHeld) {
// Successfully transitioned to kSpinLockSleeper. Pass
// kSpinLockSleeper to the SpinLockWait routine to properly indicate
// the last lock_value observed.
lock_value = kSpinLockSleeper;
} else if (lock_value == kSpinLockFree) {
// Lock is free again, so try and aquire it before sleeping. The
// new lock state will be the number of cycles this thread waited if
// this thread obtains the lock.
lock_value = base::subtle::Acquire_CompareAndSwap(&lockword_,
kSpinLockFree,
wait_cycles);
continue; // skip the delay at the end of the loop
}
}
// Wait for an OS specific delay.
base::internal::SpinLockDelay(&lockword_, lock_value,
++lock_wait_call_count);
// Spin again after returning from the wait routine to give this thread
// some chance of obtaining the lock.
lock_value = SpinLoop(wait_start_time, &wait_cycles);
}
}
// The wait time for contentionz lock profiling must fit into 32 bits.
// However, the lower 32-bits of the cycle counter wrap around too quickly
// with high frequency processors, so a right-shift by 7 is performed to
// quickly divide the cycles by 128. Using these 32 bits, reduces the
// granularity of time measurement to 128 cycles, and loses track
// of wait time for waits greater than 109 seconds on a 5 GHz machine
// [(2^32 cycles/5 Ghz)*128 = 109.95 seconds]. Waits this long should be
// very rare and the reduced granularity should not be an issue given
// processors in the Google fleet operate at a minimum of one billion
// cycles/sec.
enum { PROFILE_TIMESTAMP_SHIFT = 7 };
void SpinLock::SlowUnlock(uint64 wait_cycles) {
base::internal::SpinLockWake(&lockword_, false); // wake waiter if necessary
// Collect contentionz profile info, expanding the wait_cycles back out to
// the full value. If wait_cycles is <= kSpinLockSleeper, then no wait
// was actually performed, so don't record the wait time. Note, that the
// CalculateWaitCycles method adds in kSpinLockSleeper cycles
// unconditionally to guarantee the wait time is not kSpinLockFree or
// kSpinLockHeld. The adding in of these small number of cycles may
// overestimate the contention by a slight amount 50% of the time. However,
// if this code tried to correct for that addition by subtracting out the
// kSpinLockSleeper amount that would underestimate the contention slightly
// 50% of the time. Both ways get the wrong answer, so the code
// overestimates to be more conservative. Overestimating also makes the code
// a little simpler.
//
if (wait_cycles > kSpinLockSleeper) {
base::SubmitSpinLockProfileData(this,
wait_cycles << PROFILE_TIMESTAMP_SHIFT);
}
}
inline int32 SpinLock::CalculateWaitCycles(int64 wait_start_time) {
int32 wait_cycles = ((CycleClock::Now() - wait_start_time) >>
PROFILE_TIMESTAMP_SHIFT);
// The number of cycles waiting for the lock is used as both the
// wait_cycles and lock value, so it can't be kSpinLockFree or
// kSpinLockHeld. Make sure the value returned is at least
// kSpinLockSleeper.
wait_cycles |= kSpinLockSleeper;
return wait_cycles;
}