Size SpinLock and Mutex spin time limits to machine capacity.

absl/base/internal/spinlock.cc

@@ -14,8 +14,11 @@
 
 #include "absl/base/internal/spinlock.h"
 
+#include <algorithm>
 #include <atomic>
+#include <cmath>
 #include <cstdint>
+#include <limits>
 
 #include "absl/base/attributes.h"
 #include "absl/base/call_once.h"
@@ -110,29 +113,197 @@ static void ReportLockWait(SpinLock& lock, int64_t wait_start_time,
   ABSL_TSAN_MUTEX_POST_DIVERT(&lock, 0);
 }
 
+namespace {
+
+// Spin duration target:
+//
+//   T = min(kSpinLimitMaxNs,
+//           kSpinLimitNs * kSpinLimitReferenceCapacity / (g * N))
+//
+// where N is NumCPUs() and g is the estimated speed of one logical CPU in
+// compute units (~1.0 for a baseline server core), so g * N is the machine's
+// compute capacity. T is a duration rather than an iteration count because
+// spin-loop iteration time varies by 2-4x across CPUs, and scales inversely
+// with capacity because job schedulers pack work in proportion to capacity.
+constexpr int64_t kSpinLimitNs = 1400;
+constexpr int kSpinLimitReferenceCapacity = 250;
+constexpr int64_t kSpinLimitMaxNs = 5000;
+
+// Empirical power-law fit `g = 1.65 * (1000ns / probe_ns)^1.2` mapping the
+// 1,024-step latency probe duration (`kProbeReadings` trials per calibration,
+// ~4-6us total) to per-logical-CPU compute units.
+constexpr int kProbeReadings = 4;
+constexpr int kProbeSteps = 1024;
+constexpr uint64_t kProbeMixMultiplier = 0xbf58476d1ce4e5b9ULL;  // SplitMix64.
+constexpr int64_t kProbeReferenceNs = 1000;
+constexpr double kProbeReferenceCompute = 1.65;
+constexpr double kProbeExponent = 1.2;
+
+// Refresh schedule: starts at 10ms and doubles after each calibration up to
+// 10s (`10ms * 2^10 = 10,240ms >= 10,000ms`).
+constexpr int64_t kInitialUpdateIntervalMs = 10;
+constexpr int64_t kMaxUpdateIntervalMs = 10000;
+constexpr int kMaxUpdateBackoffShift = 10;
+
+// Sentinel deadlines for `next_spin_limit_update_cycles`: `min()` forces the
+// next `MaybeUpdateSpinLimitCycles()` call to calibrate immediately, and
+// `max()` disables future updates (for tests, single-CPU hosts, or
+// uncalibrated clocks).
+constexpr int64_t kUpdateImmediately = std::numeric_limits<int64_t>::min();
+constexpr int64_t kNeverUpdate = std::numeric_limits<int64_t>::max();
+
+// Minimum usable CycleClock rate (1 kHz). Real CycleClock implementations run
+// at >= 1 MHz; smaller values indicate an uncalibrated fallback (e.g.
+// NominalCPUFrequency() returning 1.0 when OS frequency detection is
+// unavailable).
+constexpr double kMinCycleClockFrequency = 1e3;
+
+// Process-wide spin limit calibration state. `spin_limit_cycles` is 0 until the
+// first calibration (and stays 0 on single-CPU or clockless platforms).
+// `spin_limit_updating` serializes calibration (and `min_probe_cycles` updates)
+// so concurrent callers skip rather than wait.
+ABSL_CONST_INIT std::atomic<int64_t> spin_limit_cycles{0};
+ABSL_CONST_INIT std::atomic<int64_t> next_spin_limit_update_cycles{
+    kUpdateImmediately};
+ABSL_CONST_INIT std::atomic<int> spin_limit_update_count{0};
+ABSL_CONST_INIT std::atomic<bool> spin_limit_updating{false};
+ABSL_CONST_INIT int64_t min_probe_cycles =
+    std::numeric_limits<int64_t>::max();
+
+// Measures single-core integer execution latency via a `kProbeSteps` serial
+// dependency chain (xorshift + 64-bit multiply). Seeded and consumed through
+// `volatile` so the compiler cannot constant-fold, eliminate, or reorder the
+// loop across CycleClock::Now() reads.
+ABSL_ATTRIBUTE_NOINLINE void RunLatencyProbe() {
+  volatile uint64_t sink = 1;
+  uint64_t state = sink;
+  for (int i = 0; i < kProbeSteps; ++i) {
+    state = (state ^ (state >> 29)) * kProbeMixMultiplier;
+  }
+  sink = state;
+}
+
+// Runs `kProbeReadings` probe trials, ratchets `min_probe_cycles` downward so
+// upward timing noise (preemption, SMT contention, frequency drops) cannot
+// inflate T, and returns the calibrated spin limit in CycleClock cycles.
+// Requires `spin_limit_updating` to be held by the caller.
+int64_t CalibrateSpinLimitCycles() {
+  for (int r = 0; r < kProbeReadings; ++r) {
+    const int64_t start = CycleClock::Now();
+    RunLatencyProbe();
+    const int64_t elapsed = CycleClock::Now() - start;
+    if (elapsed > 0 && elapsed < min_probe_cycles) {
+      min_probe_cycles = elapsed;
+    }
+  }
+
+  const double freq = CycleClock::Frequency();
+  const double probe_ns = static_cast<double>(min_probe_cycles) * 1e9 / freq;
+  const double compute_per_cpu =
+      kProbeReferenceCompute *
+      std::pow(kProbeReferenceNs / probe_ns, kProbeExponent);
+  const double limit_ns = std::min<double>(
+      kSpinLimitMaxNs, kSpinLimitNs * kSpinLimitReferenceCapacity /
+                           (compute_per_cpu * NumCPUs()));
+  return static_cast<int64_t>(limit_ns * freq / 1e9 + 0.5);
+}
+
+}  // namespace
+
+int64_t GetSpinLimitCycles() {
+  return spin_limit_cycles.load(std::memory_order_relaxed);
+}
+
+void MaybeUpdateSpinLimitCycles() {
+  // Fast path when not due: one clock read and one relaxed atomic load.
+  if (CycleClock::Now() <
+      next_spin_limit_update_cycles.load(std::memory_order_relaxed)) {
+    return;
+  }
+
+  // Only one thread calibrates at a time; others skip rather than wait.
+  if (spin_limit_updating.load(std::memory_order_relaxed) ||
+      spin_limit_updating.exchange(true, std::memory_order_acquire)) {
+    return;
+  }
+
+  const double freq = CycleClock::Frequency();
+  if (NumCPUs() > 1 && freq >= kMinCycleClockFrequency) {
+    spin_limit_cycles.store(CalibrateSpinLimitCycles(),
+                            std::memory_order_relaxed);
+
+    // Back off exponentially from 10ms to 10s so early contention quickly
+    // recovers from a cold/throttled initial probe, while steady state checks
+    // rarely. Measure from after probing completes.
+    const int shift = std::min(
+        spin_limit_update_count.fetch_add(1, std::memory_order_relaxed),
+        kMaxUpdateBackoffShift);
+    const int64_t interval_ms =
+        std::min(kInitialUpdateIntervalMs << shift, kMaxUpdateIntervalMs);
+    const int64_t interval_cycles =
+        static_cast<int64_t>(static_cast<double>(interval_ms) * freq / 1e3);
+    const int64_t now = CycleClock::Now();
+    const int64_t next_update = (now <= kNeverUpdate - interval_cycles)
+                                    ? now + interval_cycles
+                                    : kNeverUpdate;
+    next_spin_limit_update_cycles.store(next_update, std::memory_order_relaxed);
+  } else {
+    next_spin_limit_update_cycles.store(kNeverUpdate,
+                                        std::memory_order_relaxed);
+  }
+  spin_limit_updating.store(false, std::memory_order_release);
+}
+
+void SetSpinLimitCyclesForTesting(int64_t cycles, bool freeze) {
+  spin_limit_update_count.store(0, std::memory_order_relaxed);
+  next_spin_limit_update_cycles.store(
+      freeze ? kNeverUpdate : kUpdateImmediately, std::memory_order_relaxed);
+  spin_limit_cycles.store(cycles, std::memory_order_relaxed);
+}
+
 // Monitor the lock to see if its value changes within some time period
-// (adaptive_spin_count_ loop iterations). The last value read from the lock
-// is returned from the method.
+// (GetSpinLimitCycles(), or adaptive_spin_count_ loop iterations when no cycle
+// limit is active). The last value read from the lock is returned.
 ABSL_CONST_INIT std::atomic<int> SpinLock::adaptive_spin_count_{0};
 uint32_t SpinLock::SpinLoop() {
   // We are already in the slow path of SpinLock, initialize the
   // adaptive_spin_count here.
   if (adaptive_spin_count_.load(std::memory_order_relaxed) == 0) {
     int current_spin_count = 0;
-    int new_spin_count = NumCPUs() > 1 ? 1000 : 1;
+    int new_spin_count = NumCPUs() > 1 ? kMinSpinIterations : 1;
     // If this fails, the value will remain unchanged. We may not spin for the
     // intended duration, but that is still safe. We will try again on the next
     // call to SpinLoop.
     adaptive_spin_count_.compare_exchange_weak(
         current_spin_count, new_spin_count, std::memory_order_relaxed,
         std::memory_order_relaxed);
   }
-  int c = adaptive_spin_count_.load(std::memory_order_relaxed);
+  const int64_t limit_cycles = GetSpinLimitCycles();
+  const uint64_t start =
+      limit_cycles > 0 ? static_cast<uint64_t>(CycleClock::Now()) : 0;
+
+  // Spin in batches so CycleClock::Now() is only polled after the initial
+  // kMinSpinIterations pass and every kSpinClockCheckInterval iterations
+  // thereafter.
+  int batch = limit_cycles > 0
+                  ? kMinSpinIterations
+                  : adaptive_spin_count_.load(std::memory_order_relaxed);
+  int spun = 0;
   uint32_t lock_value;
-  do {
-    lock_value = lockword_.load(std::memory_order_relaxed);
-  } while ((lock_value & kSpinLockHeld) != 0 && --c > 0);
-  return lock_value;
+  while (true) {
+    int c = batch;
+    do {
+      lock_value = lockword_.load(std::memory_order_relaxed);
+    } while ((lock_value & kSpinLockHeld) != 0 && --c > 0);
+    spun += batch;
+    if ((lock_value & kSpinLockHeld) == 0 || limit_cycles <= 0 ||
+        spun >= kMaxSpinIterations ||
+        static_cast<uint64_t>(CycleClock::Now()) - start >=
+            static_cast<uint64_t>(limit_cycles)) {
+      return lock_value;
+    }
+    batch = kSpinClockCheckInterval;
+  }
 }
 
 void SpinLock::SlowLock() {
@@ -145,6 +316,10 @@ void SpinLock::SlowLock() {
   SchedulingMode scheduling_mode;
   if ((lock_value & kSpinLockCooperative) != 0) {
     scheduling_mode = SCHEDULE_COOPERATIVE_AND_KERNEL;
+    // Refresh the spin limit before wait_start_time is recorded so calibration
+    // is not reported as lock wait time (skipped for async-signal-safe
+    // SCHEDULE_KERNEL_ONLY locks).
+    MaybeUpdateSpinLimitCycles();
   } else {
     scheduling_mode = SCHEDULE_KERNEL_ONLY;
   }

absl/base/internal/spinlock.h

@@ -329,6 +329,37 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value,
   return lock_value;
 }
 
+// Spin limits.
+//
+// A thread that finds a SpinLock or exclusive Mutex held spins for up to a
+// process-wide cycle budget (calibrated from CPU count and single-core speed,
+// capped at 5us; see spinlock.cc) before blocking.
+
+// Initial unpolled spin batch on multi-CPU machines, and fixed spin iteration
+// count when no cycle limit is active (before calibration or without a clock).
+inline constexpr int kMinSpinIterations = 1000;
+// Hard iteration cap (>5us on current CPUs) as a fallback if CycleClock stalls
+// or is miscalibrated.
+inline constexpr int kMaxSpinIterations = 20000;
+// Spin-loop iterations between CycleClock::Now() checks after the initial
+// `kMinSpinIterations` pass, amortizing clock-read overhead.
+inline constexpr int kSpinClockCheckInterval = 256;
+
+// Returns the process-wide spin limit in CycleClock cycles, or 0 when spinning
+// should fall back to a fixed iteration count (before first calibration, on
+// single-CPU machines, or when no cycle clock is available).
+int64_t GetSpinLimitCycles();
+
+// Initializes or refreshes the process-wide spin limit if due, returning
+// immediately when not due or if another thread is already calibrating.
+// Call on contended slow paths before recording wait-time timestamps.
+void MaybeUpdateSpinLimitCycles();
+
+// Overrides the process-wide spin limit for tests. `freeze` suppresses
+// refreshes when true and forces an immediate refresh on the next contention
+// when false.
+void SetSpinLimitCyclesForTesting(int64_t cycles, bool freeze = true);
+
 }  // namespace base_internal
 ABSL_NAMESPACE_END
 }  // namespace absl