diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 57966d2c528..b2ee94d772d 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc @@ -14,8 +14,11 @@ #include "absl/base/internal/spinlock.h" +#include #include +#include #include +#include #include "absl/base/attributes.h" #include "absl/base/call_once.h" @@ -110,6 +113,246 @@ static void ReportLockWait(SpinLock& lock, int64_t wait_start_time, ABSL_TSAN_MUTEX_POST_DIVERT(&lock, 0); } +namespace { + +// Spin duration target: +// +// T = min(kSpinLimitMaxNs, +// kSpinLimitNs * kSpinLimitReferenceCapacity / (g * N)) +// +// where N is NumCPUs() and g is the estimated speed of one logical CPU in +// compute units (~1.0 for a baseline server core), so g * N is the machine's +// compute capacity. T is a duration rather than a fixed iteration count because +// spin-loop iteration time varies by 2-4x across CPUs, and scales inversely +// with capacity because job schedulers pack work in proportion to capacity. +// Each lock primitive converts T into an iteration count by timing its own +// spin loop's per-iteration cost. +constexpr int64_t kSpinLimitNs = 1400; +constexpr int kSpinLimitReferenceCapacity = 250; +constexpr int64_t kSpinLimitMaxNs = 5000; + +// Empirical power-law fit `g = 1.65 * (1000ns / probe_ns)^1.2` mapping the +// 1,024-step latency probe duration (`kProbeReadings` trials per calibration) +// to per-logical-CPU compute units. The same step and trial counts are used to +// time the caller's spin loop (~5-9us total per calibration). +constexpr int kProbeReadings = 4; +constexpr int kProbeSteps = 1024; +constexpr uint64_t kProbeMixMultiplier = 0xbf58476d1ce4e5b9ULL; // SplitMix64. +constexpr int64_t kProbeReferenceNs = 1000; +constexpr double kProbeReferenceCompute = 1.65; +constexpr double kProbeExponent = 1.2; + +// Refresh schedule: starts at 10ms and doubles after each calibration up to +// 10s (`10ms * 2^10 = 10,240ms >= 10,000ms`). +constexpr int64_t kInitialUpdateIntervalMs = 10; +constexpr int64_t kMaxUpdateIntervalMs = 10000; +constexpr int kMaxUpdateBackoffShift = 10; + +// Sentinel deadlines for `next_spin_limit_update_cycles`: `min()` forces the +// next `MaybeUpdateSpinLimitCycles()` call to calibrate immediately, and +// `max()` disables future updates (for tests, single-CPU hosts, or +// uncalibrated clocks). +constexpr int64_t kUpdateImmediately = std::numeric_limits::min(); +constexpr int64_t kNeverUpdate = std::numeric_limits::max(); + +// Minimum usable CycleClock rate (1 kHz). Real CycleClock implementations run +// at >= 1 MHz; smaller values indicate an uncalibrated fallback (e.g. +// NominalCPUFrequency() returning 1.0 when OS frequency detection is +// unavailable). +constexpr double kMinCycleClockFrequency = 1e3; + +// Process-wide spin limit calibration state. `spin_limit_cycles` is 0 until the +// first calibration (and stays 0 on single-CPU or clockless platforms). +// `spin_limit_updating` serializes target calibration (and `min_probe_cycles` +// updates) so concurrent callers skip rather than wait, and +// `spin_limit_generation` increments after each target update so SpinLock and +// Mutex refresh their iteration counts independently. +ABSL_CONST_INIT std::atomic spin_limit_cycles{0}; +ABSL_CONST_INIT std::atomic spin_limit_generation{0}; +ABSL_CONST_INIT std::atomic next_spin_limit_update_cycles{ + kUpdateImmediately}; +ABSL_CONST_INIT std::atomic spin_limit_update_count{0}; +ABSL_CONST_INIT std::atomic spin_limit_updating{false}; +ABSL_CONST_INIT int64_t min_probe_cycles = + std::numeric_limits::max(); + +ABSL_CONST_INIT std::atomic spinlock_spin_generation{0}; +ABSL_CONST_INIT std::atomic min_spinlock_loop_cycles{ + std::numeric_limits::max()}; + +// Measures single-core integer execution latency via a `kProbeSteps` serial +// dependency chain (xorshift + 64-bit multiply). Seeded and consumed through +// `volatile` so the compiler cannot constant-fold, eliminate, or reorder the +// loop across CycleClock::Now() reads. +ABSL_ATTRIBUTE_NOINLINE void RunLatencyProbe() { + volatile uint64_t sink = 1; + uint64_t state = sink; + for (int i = 0; i < kProbeSteps; ++i) { + state = (state ^ (state >> 29)) * kProbeMixMultiplier; + } + sink = state; +} + +// Runs `kProbeReadings` probe trials, ratchets `min_probe_cycles` downward so +// upward timing noise (preemption, SMT contention, frequency drops) cannot +// inflate T, and returns the calibrated spin limit in CycleClock cycles. +// Requires `spin_limit_updating` to be held by the caller. +int64_t CalibrateSpinLimitCycles(double freq, int num_cpus) { + for (int r = 0; r < kProbeReadings; ++r) { + const int64_t start = CycleClock::Now(); + RunLatencyProbe(); + const int64_t elapsed = CycleClock::Now() - start; + if (elapsed > 0 && elapsed < min_probe_cycles) { + min_probe_cycles = elapsed; + } + } + if (min_probe_cycles == std::numeric_limits::max()) { + return 0; + } + + const double probe_ns = static_cast(min_probe_cycles) * 1e9 / freq; + const double compute_per_cpu = + kProbeReferenceCompute * + std::pow(kProbeReferenceNs / probe_ns, kProbeExponent); + const double limit_ns = std::min( + kSpinLimitMaxNs, kSpinLimitNs * kSpinLimitReferenceCapacity / + (compute_per_cpu * num_cpus)); + return static_cast(limit_ns * freq / 1e9 + 0.5); +} + +int CalibrateSpinIterations( + void (*absl_nonnull probe_fn)(int steps), + std::atomic* absl_nonnull min_loop_cycles) { + const int64_t limit_cycles = GetSpinLimitCycles(); + if (limit_cycles <= 0) { + return 0; + } + + volatile int probe_steps = kProbeSteps; + const int steps = probe_steps; + int64_t local_min = std::numeric_limits::max(); + for (int r = 0; r < kProbeReadings; ++r) { + const int64_t start = CycleClock::Now(); + probe_fn(steps); + const int64_t elapsed = CycleClock::Now() - start; + if (elapsed > 0 && elapsed < local_min) { + local_min = elapsed; + } + } + + int64_t best_cycles = min_loop_cycles->load(std::memory_order_relaxed); + while (local_min < best_cycles && + !min_loop_cycles->compare_exchange_weak( + best_cycles, local_min, std::memory_order_relaxed, + std::memory_order_relaxed)) { + } + best_cycles = std::min(best_cycles, local_min); + if (best_cycles == std::numeric_limits::max()) { + return kMinSpinIterations; + } + + const double iters = static_cast(limit_cycles) * kProbeSteps / + static_cast(best_cycles); + return static_cast( + std::clamp(iters + 0.5, kMinSpinIterations, kMaxSpinIterations)); +} + +// Spins up to `c` iterations waiting for bit 0 (`kSpinLockHeld`) of +// `*lockword` to clear, returning the last value read. Out-of-line so +// ProbeSpinLockLoop() and SpinLock::SpinLoop() execute the same compiled loop. +ABSL_ATTRIBUTE_NOINLINE uint32_t +SpinLockLoop(const std::atomic* lockword, int c) { + constexpr uint32_t kSpinLockHeldBit = 1; + uint32_t lock_value; + do { + lock_value = lockword->load(std::memory_order_relaxed); + } while ((lock_value & kSpinLockHeldBit) != 0 && --c > 0); + return lock_value; +} + +void ProbeSpinLockLoop(int steps) { + std::atomic held_word{1}; + const std::atomic* volatile ptr = &held_word; + volatile uint32_t sink = SpinLockLoop(ptr, steps); + (void)sink; +} + +} // namespace + +int64_t GetSpinLimitCycles() { + return spin_limit_cycles.load(std::memory_order_relaxed); +} + +uint32_t MaybeUpdateSpinLimitCycles() { + // Fast path when not due: one clock read and one relaxed atomic load. + if (CycleClock::Now() < + next_spin_limit_update_cycles.load(std::memory_order_relaxed)) { + return spin_limit_generation.load(std::memory_order_acquire); + } + + // Only one thread calibrates at a time; others skip rather than wait. + if (spin_limit_updating.load(std::memory_order_relaxed) || + spin_limit_updating.exchange(true, std::memory_order_acquire)) { + return spin_limit_generation.load(std::memory_order_acquire); + } + + uint32_t gen; + const int num_cpus = NumCPUs(); + const double freq = CycleClock::Frequency(); + if (num_cpus > 1 && freq >= kMinCycleClockFrequency) { + spin_limit_cycles.store(CalibrateSpinLimitCycles(freq, num_cpus), + std::memory_order_relaxed); + + // Back off exponentially from 10ms to 10s so early contention quickly + // recovers from a cold/throttled initial probe, while steady state checks + // rarely. Measure from after probing completes. + const int shift = std::min( + spin_limit_update_count.fetch_add(1, std::memory_order_relaxed), + kMaxUpdateBackoffShift); + const int64_t interval_ms = + std::min(kInitialUpdateIntervalMs << shift, kMaxUpdateIntervalMs); + const int64_t interval_cycles = + static_cast(static_cast(interval_ms) * freq / 1e3); + const int64_t now = CycleClock::Now(); + const int64_t next_update = (now <= kNeverUpdate - interval_cycles) + ? now + interval_cycles + : kNeverUpdate; + next_spin_limit_update_cycles.store(next_update, std::memory_order_relaxed); + gen = spin_limit_generation.fetch_add(1, std::memory_order_release) + 1; + } else { + next_spin_limit_update_cycles.store(kNeverUpdate, + std::memory_order_relaxed); + gen = spin_limit_generation.load(std::memory_order_acquire); + } + spin_limit_updating.store(false, std::memory_order_release); + return gen; +} + +void MaybeUpdateSpinIterations( + std::atomic* absl_nonnull iterations, + void (*absl_nonnull probe_fn)(int steps), + std::atomic* absl_nonnull generation, + std::atomic* absl_nonnull min_loop_cycles) { + const uint32_t gen = MaybeUpdateSpinLimitCycles(); + uint32_t prev_gen = generation->load(std::memory_order_relaxed); + if (gen == prev_gen || + !generation->compare_exchange_strong(prev_gen, gen, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + return; + } + iterations->store(CalibrateSpinIterations(probe_fn, min_loop_cycles), + std::memory_order_relaxed); +} + +void SetSpinLimitCyclesForTesting(int64_t cycles, bool freeze) { + spin_limit_update_count.store(0, std::memory_order_relaxed); + next_spin_limit_update_cycles.store( + freeze ? kNeverUpdate : kUpdateImmediately, std::memory_order_relaxed); + spin_limit_cycles.store(cycles, std::memory_order_relaxed); + spin_limit_generation.fetch_add(1, std::memory_order_release); +} + // Monitor the lock to see if its value changes within some time period // (adaptive_spin_count_ loop iterations). The last value read from the lock // is returned from the method. @@ -119,7 +362,7 @@ uint32_t SpinLock::SpinLoop() { // adaptive_spin_count here. if (adaptive_spin_count_.load(std::memory_order_relaxed) == 0) { int current_spin_count = 0; - int new_spin_count = NumCPUs() > 1 ? 1000 : 1; + int new_spin_count = NumCPUs() > 1 ? kMinSpinIterations : 1; // If this fails, the value will remain unchanged. We may not spin for the // intended duration, but that is still safe. We will try again on the next // call to SpinLoop. @@ -128,11 +371,7 @@ uint32_t SpinLock::SpinLoop() { std::memory_order_relaxed); } int c = adaptive_spin_count_.load(std::memory_order_relaxed); - uint32_t lock_value; - do { - lock_value = lockword_.load(std::memory_order_relaxed); - } while ((lock_value & kSpinLockHeld) != 0 && --c > 0); - return lock_value; + return SpinLockLoop(&lockword_, c); } void SpinLock::SlowLock() { @@ -145,6 +384,11 @@ void SpinLock::SlowLock() { SchedulingMode scheduling_mode; if ((lock_value & kSpinLockCooperative) != 0) { scheduling_mode = SCHEDULE_COOPERATIVE_AND_KERNEL; + // Skipped for SCHEDULE_KERNEL_ONLY locks, which must remain + // async-signal-safe. + MaybeUpdateSpinIterations(&adaptive_spin_count_, &ProbeSpinLockLoop, + &spinlock_spin_generation, + &min_spinlock_loop_cycles); } else { scheduling_mode = SCHEDULE_KERNEL_ONLY; } diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index aff0c9f98ff..8e87b3e2cf5 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h @@ -329,6 +329,36 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, return lock_value; } +// Spin limits. + +// Calibrated spin iteration count bounds on multi-CPU machines. +inline constexpr int kMinSpinIterations = 1000; +inline constexpr int kMaxSpinIterations = 20000; + +// Returns the process-wide target spin duration in CycleClock cycles (0 before +// first calibration, on single-CPU machines, or without a usable CycleClock). +int64_t GetSpinLimitCycles(); + +// Initializes or refreshes the process-wide target spin duration if due, +// returning the current calibration generation. +uint32_t MaybeUpdateSpinLimitCycles(); + +// Refreshes the target spin duration if due, and when the calibration +// generation advances past `*generation`, times `probe_fn(steps)` (latching the +// fastest reading in `*min_loop_cycles`) and updates `*iterations` to the +// calibrated count clamped to [kMinSpinIterations, kMaxSpinIterations] (or 0 +// when `GetSpinLimitCycles() <= 0`). +void MaybeUpdateSpinIterations( + std::atomic* absl_nonnull iterations, + void (*absl_nonnull probe_fn)(int steps), + std::atomic* absl_nonnull generation, + std::atomic* absl_nonnull min_loop_cycles); + +// Overrides the process-wide target spin duration for tests. `freeze` +// suppresses refreshes when true and forces an immediate refresh on the next +// contention when false. +void SetSpinLimitCyclesForTesting(int64_t cycles, bool freeze = true); + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/spinlock_test_common.cc b/absl/base/spinlock_test_common.cc index a29a7f51944..7b14b3c132b 100644 --- a/absl/base/spinlock_test_common.cc +++ b/absl/base/spinlock_test_common.cc @@ -16,6 +16,7 @@ // spinlock. If the spinlock is working properly, all elements of the // array should be equal at the end of the test. +#include #include #include #include @@ -28,12 +29,14 @@ #include "gtest/gtest.h" #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" #include "absl/base/internal/low_level_scheduling.h" #include "absl/base/internal/scheduling_mode.h" #include "absl/base/internal/spinlock.h" #include "absl/base/internal/sysinfo.h" #include "absl/base/macros.h" #include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/mutex.h" #include "absl/synchronization/notification.h" constexpr uint32_t kNumThreads = 10; @@ -54,6 +57,13 @@ struct SpinLockTest { } static bool IsCooperative(const SpinLock& l) { return l.IsCooperative(); } + static int GetAdaptiveSpinCount() { return SpinLock::GetAdaptiveSpinCount(); } + static void SetAdaptiveSpinCount(int count) { + SpinLock::SetAdaptiveSpinCount(count); + } + static bool SpinLoopObservedHeld(SpinLock& l) { + return (l.SpinLoop() & SpinLock::kSpinLockHeld) != 0; + } }; namespace { @@ -296,6 +306,201 @@ TEST(SpinLockTest, ScopedLock) { std::scoped_lock l(s); } +// Saves the process-wide spin limit and SpinLock spin count, resets them to +// their uninitialized state for the duration of a test, and restores the prior +// settings on scope exit. +class ScopedSpinLimitReset { + public: + ScopedSpinLimitReset() + : saved_cycles_(GetSpinLimitCycles()), + saved_spin_count_(SpinLockTest::GetAdaptiveSpinCount()) { + SpinLockTest::SetAdaptiveSpinCount(0); + SetSpinLimitCyclesForTesting(0, /*freeze=*/false); + } + ~ScopedSpinLimitReset() { + SpinLockTest::SetAdaptiveSpinCount(saved_spin_count_); + SetSpinLimitCyclesForTesting(saved_cycles_, /*freeze=*/false); + } + ScopedSpinLimitReset(const ScopedSpinLimitReset&) = delete; + ScopedSpinLimitReset& operator=(const ScopedSpinLimitReset&) = delete; + + private: + const int64_t saved_cycles_; + const int saved_spin_count_; +}; + +// Returns true when this host has multiple CPUs and a CycleClock fine enough +// for calibration to produce a positive cycle budget. +bool PlatformSupportsTimedSpinLimit() { + ScopedSpinLimitReset reset; + MaybeUpdateSpinLimitCycles(); + return GetSpinLimitCycles() > 0; +} + +ABSL_ATTRIBUTE_NOINLINE void TestProbeLoop(int steps) { + std::atomic word{1}; + do { + (void)word.load(std::memory_order_relaxed); + } while (--steps > 0); +} + +// Verifies that under frozen `spin_limit_cycles`, a cooperative SpinLock +// waiter calibrates `SpinLock::adaptive_spin_count_` to `expected_spin_count` +// and blocks until the holder releases the lock. +void ExpectContendedSpinLockCalibratesAndBlocks(int64_t spin_limit_cycles, + int expected_spin_count) { + ScopedSpinLimitReset reset; + SpinLockTest::SetAdaptiveSpinCount(1); + SetSpinLimitCyclesForTesting(spin_limit_cycles, /*freeze=*/true); + + SpinLock lock; + std::atomic waiter_acquired{false}; + std::thread waiter; + { + SpinLockHolder holder(lock); + waiter = std::thread([&] { + SpinLockHolder waiter_lock(lock); + waiter_acquired.store(true, std::memory_order_relaxed); + }); + while (SpinLockTest::GetAdaptiveSpinCount() == 1) { + std::this_thread::yield(); + } + EXPECT_EQ(SpinLockTest::GetAdaptiveSpinCount(), expected_spin_count); + EXPECT_FALSE(waiter_acquired.load(std::memory_order_relaxed)); + } + waiter.join(); + + EXPECT_TRUE(waiter_acquired.load(std::memory_order_relaxed)); +} + +TEST(SpinLockTest, MaybeUpdateSpinLimitCyclesCalibratesBoundedCycleBudget) { + if (!PlatformSupportsTimedSpinLimit()) return; + constexpr double kMaxSpinSeconds = 5e-6; + ScopedSpinLimitReset reset; + ASSERT_EQ(GetSpinLimitCycles(), 0); + + MaybeUpdateSpinLimitCycles(); + + const int64_t cycles = GetSpinLimitCycles(); + EXPECT_GT(cycles, 0); + EXPECT_LE(static_cast(cycles), + kMaxSpinSeconds * CycleClock::Frequency() + 1.0); +} + +// Calibration latches the fastest probe reading seen by the process so +// transient slowdowns cannot inflate the spin budget on later refreshes. +TEST(SpinLockTest, + MaybeUpdateSpinLimitCyclesNeverInflatesBudgetAcrossRecalibrations) { + if (!PlatformSupportsTimedSpinLimit()) return; + ScopedSpinLimitReset reset; + + MaybeUpdateSpinLimitCycles(); + const int64_t first = GetSpinLimitCycles(); + SetSpinLimitCyclesForTesting(0, /*freeze=*/false); + MaybeUpdateSpinLimitCycles(); + const int64_t second = GetSpinLimitCycles(); + + EXPECT_GT(second, 0); + EXPECT_LE(second, first); +} + +TEST(SpinLockTest, FrozenSpinLimitOverrideSuppressesCalibration) { + constexpr int64_t kFrozenCycles = 6789; + ScopedSpinLimitReset reset; + + SetSpinLimitCyclesForTesting(kFrozenCycles, /*freeze=*/true); + MaybeUpdateSpinLimitCycles(); + + EXPECT_EQ(GetSpinLimitCycles(), kFrozenCycles); +} + +TEST(SpinLockTest, MaybeUpdateSpinIterationsClampsToBounds) { + ScopedSpinLimitReset reset; + std::atomic iters{-1}; + std::atomic gen{0}; + std::atomic min_loop_cycles{std::numeric_limits::max()}; + + SetSpinLimitCyclesForTesting(0, /*freeze=*/true); + MaybeUpdateSpinIterations(&iters, &TestProbeLoop, &gen, &min_loop_cycles); + EXPECT_EQ(iters.load(), 0); + + SetSpinLimitCyclesForTesting(1, /*freeze=*/true); + MaybeUpdateSpinIterations(&iters, &TestProbeLoop, &gen, &min_loop_cycles); + EXPECT_EQ(iters.load(), kMinSpinIterations); + + SetSpinLimitCyclesForTesting(std::numeric_limits::max() / 2, + /*freeze=*/true); + MaybeUpdateSpinIterations(&iters, &TestProbeLoop, &gen, &min_loop_cycles); + EXPECT_EQ(iters.load(), kMaxSpinIterations); +} + +TEST(SpinLockTest, SpinLoopGivesUpAfterAdaptiveSpinCountIterations) { + ScopedSpinLimitReset reset; + SpinLock lock; + SpinLockHolder holder(lock); + + EXPECT_TRUE(SpinLockTest::SpinLoopObservedHeld(lock)); + EXPECT_EQ(SpinLockTest::GetAdaptiveSpinCount(), + base_internal::NumCPUs() > 1 ? kMinSpinIterations : 1); + + SpinLockTest::SetAdaptiveSpinCount(1); + EXPECT_TRUE(SpinLockTest::SpinLoopObservedHeld(lock)); + + SpinLockTest::SetAdaptiveSpinCount(kMaxSpinIterations); + EXPECT_TRUE(SpinLockTest::SpinLoopObservedHeld(lock)); +} + +TEST(SpinLockTest, ContendedCooperativeSpinLockTriggersInitialCalibration) { + if (!PlatformSupportsTimedSpinLimit()) return; + ScopedSpinLimitReset reset; + + SpinLock lock; + std::thread waiter; + { + SpinLockHolder holder(lock); + waiter = std::thread([&] { SpinLockHolder waiter_lock(lock); }); + while (GetSpinLimitCycles() == 0) { + std::this_thread::yield(); + } + } + waiter.join(); + + EXPECT_GE(SpinLockTest::GetAdaptiveSpinCount(), kMinSpinIterations); + EXPECT_LE(SpinLockTest::GetAdaptiveSpinCount(), kMaxSpinIterations); +} + +TEST(SpinLockTest, ContendedSpinLockClampsToMinAndMaxIterations) { + if (!PlatformSupportsTimedSpinLimit()) return; + ExpectContendedSpinLockCalibratesAndBlocks(/*spin_limit_cycles=*/1, + kMinSpinIterations); + ExpectContendedSpinLockCalibratesAndBlocks( + /*spin_limit_cycles=*/std::numeric_limits::max() / 2, + kMaxSpinIterations); +} + +TEST(SpinLockTest, ContendedMutexWriterCalibratesAndBlocksWhenLimitUnset) { + if (!PlatformSupportsTimedSpinLimit()) return; + ScopedSpinLimitReset reset; + + absl::Mutex mu; + std::atomic writer_acquired{false}; + std::thread writer; + { + absl::MutexLock holder_lock(mu); + writer = std::thread([&] { + absl::MutexLock lock(mu); + writer_acquired.store(true, std::memory_order_relaxed); + }); + while (GetSpinLimitCycles() == 0) { + std::this_thread::yield(); + } + EXPECT_FALSE(writer_acquired.load(std::memory_order_relaxed)); + } + writer.join(); + + EXPECT_TRUE(writer_acquired.load(std::memory_order_relaxed)); +} + } // namespace } // namespace base_internal ABSL_NAMESPACE_END diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index d5c1a3c46e5..8b6632cdadf 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include // NOLINT(build/c++11) #include "absl/base/attributes.h" @@ -137,8 +138,9 @@ enum DelayMode { AGGRESSIVE, GENTLE }; struct ABSL_CACHELINE_ALIGNED MutexGlobals { absl::once_flag once; - // Note: this variable is initialized separately in Mutex::LockSlow, - // so that Mutex::Lock does not have a stack frame in optimized build. + // Note: this variable is initialized separately in + // Mutex::LockSlowWithSpinning, so that Mutex::lock does not have a stack + // frame in optimized build. std::atomic spinloop_iterations{0}; int32_t mutex_sleep_spins[2] = {}; absl::Duration mutex_sleep_time; @@ -1571,10 +1573,14 @@ void Mutex::AssertNotHeld() const { } } -// Attempt to acquire *mu, and return whether successful. The implementation -// may spin for a short while if the lock cannot be acquired immediately. -static bool TryAcquireWithSpinning(std::atomic* mu) { - int c = globals.spinloop_iterations.load(std::memory_order_relaxed); +// Attempt to acquire *mu within `c` loop iterations, and return whether +// successful. Out-of-line so ProbeMutexSpinLoop() and LockSlowWithSpinning() +// execute the same compiled loop. +ABSL_ATTRIBUTE_NOINLINE static bool TryAcquireWithSpinning( + std::atomic* mu, int c) { + if (c <= 0) { + return false; + } do { // do/while somewhat faster on AMD intptr_t v = mu->load(std::memory_order_relaxed); if ((v & (kMuReader | kMuEvent)) != 0) { @@ -1589,6 +1595,30 @@ static bool TryAcquireWithSpinning(std::atomic* mu) { return false; } +static void ProbeMutexSpinLoop(int steps) { + std::atomic held_mu{kMuWriter}; + std::atomic* volatile mu_ptr = &held_mu; + volatile bool sink = TryAcquireWithSpinning(mu_ptr, steps); + (void)sink; +} + +ABSL_CONST_INIT static std::atomic mutex_spin_generation{0}; +ABSL_CONST_INIT static std::atomic min_mutex_loop_cycles{ + std::numeric_limits::max()}; + +// Out-of-line so non-tail calls on the slow path do not force a stack frame in +// Mutex::lock(). +ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlowWithSpinning() { + if (!TryAcquireWithSpinning( + &this->mu_, + globals.spinloop_iterations.load(std::memory_order_relaxed))) { + base_internal::MaybeUpdateSpinIterations( + &globals.spinloop_iterations, &ProbeMutexSpinLoop, + &mutex_spin_generation, &min_mutex_loop_cycles); + this->LockSlow(kExclusive, nullptr, 0); + } +} + void Mutex::lock() { ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); DebugOnlyDeadlockCheck(this); @@ -1598,10 +1628,7 @@ void Mutex::lock() { ABSL_PREDICT_FALSE(!mu_.compare_exchange_strong( v, kMuWriter | v, std::memory_order_acquire, std::memory_order_relaxed))) { - // try spin acquire, then slow loop - if (ABSL_PREDICT_FALSE(!TryAcquireWithSpinning(&this->mu_))) { - this->LockSlow(kExclusive, nullptr, 0); - } + this->LockSlowWithSpinning(); } DebugOnlyLockEnter(this); ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); @@ -1890,22 +1917,6 @@ static intptr_t IgnoreWaitingWritersMask(int flag) { // Internal version of LockWhen(). See LockSlowWithDeadline() ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition* cond, int flags) { - // Note: we specifically initialize spinloop_iterations after the first use - // in TryAcquireWithSpinning so that Lock function does not have any non-tail - // calls and consequently a stack frame. It's fine to have spinloop_iterations - // uninitialized (meaning no spinning) in all initial uncontended Lock calls - // and in the first contended call. After that we will have - // spinloop_iterations properly initialized. - if (ABSL_PREDICT_FALSE( - globals.spinloop_iterations.load(std::memory_order_relaxed) == 0)) { - if (absl::base_internal::NumCPUs() > 1) { - // If this is multiprocessor, allow spinning. - globals.spinloop_iterations.store(1500, std::memory_order_relaxed); - } else { - // If this a uniprocessor, only yield/sleep. - globals.spinloop_iterations.store(-1, std::memory_order_relaxed); - } - } ABSL_RAW_CHECK( this->LockSlowWithDeadline(how, cond, KernelTimeout::Never(), flags), "condition untrue on return from LockSlow"); diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h index 0fbc9a55f2b..f1f3e50bdae 100644 --- a/absl/synchronization/mutex.h +++ b/absl/synchronization/mutex.h @@ -562,6 +562,7 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED Mutex { ABSL_ATTRIBUTE_COLD void LockSlow(MuHow absl_nonnull how, const Condition* absl_nullable cond, int flags); + ABSL_ATTRIBUTE_COLD void LockSlowWithSpinning(); // slow path release ABSL_ATTRIBUTE_COLD void UnlockSlow(SynchWaitParams* absl_nullable waitp); // TryLock slow path.