fix: read cgroupv2 cpu usage from the cgroup that owns cpu.max

cgroupv2.go

@@ -89,37 +89,78 @@ func getFromParentOrFallback[T any](
 }
 
 func (s cgroupV2Statter) cpuUsed() (used float64, err error) {
-	cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat)
+	periodUs, err := s.cpuPeriod()
+	if err != nil {
+		return 0, xerrors.Errorf("get cpu period: %w", err)
+	}
 
-	usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec")
+	limit, err := s.cpuLimit()
 	if err != nil {
-		return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err)
+		return 0, xerrors.Errorf("get cpu quota: %w", err)
 	}
-	periodUs, err := s.cpuPeriod()
+
+	// Read usage from the cgroup that owns the limit so that used and
+	// total describe the same domain. Otherwise sibling cgroups under
+	// the limit owner (for example /docker/<id> next to /init.scope in
+	// docker-in-docker) would count against the limit but not the usage.
+	if limit != nil && limit.owner.path != s.path {
+		if usedUs, err := limit.owner.cpuUsedUs(); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) {
+				return 0, xerrors.Errorf("read limit owner cpu: %w", err)
+			}
+			// The limit owner does not expose usage files, so fall
+			// back to the process's own cgroup below.
+		} else {
+			return float64(usedUs) / periodUs, nil
+		}
+	}
+
+	usedUs, err := s.cpuUsedUs()
 	if err != nil {
-		return 0, xerrors.Errorf("get cpu period: %w", err)
+		return 0, err
 	}
 
-	return float64(usageUs) / periodUs, nil
+	return float64(usedUs) / periodUs, nil
 }
 
-func (s cgroupV2Statter) cpuQuota() (float64, error) {
+// cpuLimit is the effective cpu.max quota and the cgroup that sets it.
+// owner is kept so usage can be read from the same cgroup the limit is
+// enforced on.
+type cpuLimit struct {
+	owner   *cgroupV2Statter
+	quotaUs float64
+}
+
+// cpuLimit returns the nearest cgroup, starting at s and walking up,
+// that sets a cpu.max quota. It returns nil when no cgroup sets a limit.
+func (s *cgroupV2Statter) cpuLimit() (*cpuLimit, error) {
 	cpuMaxPath := filepath.Join(s.path, cgroupV2CPUMax)
 
 	quotaUs, err := readInt64SepIdx(s.fs, cpuMaxPath, " ", 0)
 	if err != nil {
 		if !errors.Is(err, strconv.ErrSyntax) && !errors.Is(err, fs.ErrNotExist) {
-			return 0, xerrors.Errorf("get cpu quota: %w", err)
+			return nil, xerrors.Errorf("get cpu quota: %w", err)
 		}
 
 		// If the value is not a valid integer, assume it is the string
-		// 'max' and that there is no limit set. Try the parent or return -1.
-		return getFromParentOrFallback(&s, (*cgroupV2Statter).cpuQuota, func() (float64, error) {
-			return -1.0, nil
+		// 'max' and that there is no limit set. Try the parent or return nil (no limit).
+		return getFromParentOrFallback(s, (*cgroupV2Statter).cpuLimit, func() (*cpuLimit, error) {
+			return nil, nil
 		})
 	}
 
-	return float64(quotaUs), nil
+	return &cpuLimit{owner: s, quotaUs: float64(quotaUs)}, nil
+}
+
+func (s cgroupV2Statter) cpuUsedUs() (int64, error) {
+	cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat)
+
+	usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec")
+	if err != nil {
+		return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err)
+	}
+
+	return usageUs, nil
 }
 
 func (s cgroupV2Statter) cpuPeriod() (float64, error) {
@@ -148,12 +189,15 @@ func (s cgroupV2Statter) cpuTotal() (total float64, err error) {
 		return 0, xerrors.Errorf("get cpu period: %w", err)
 	}
 
-	quotaUs, err := s.cpuQuota()
+	limit, err := s.cpuLimit()
 	if err != nil {
 		return 0, xerrors.Errorf("get cpu quota: %w", err)
 	}
+	if limit == nil {
+		return -1.0 / periodUs, nil
+	}
 
-	return quotaUs / periodUs, nil
+	return limit.quotaUs / periodUs, nil
 }
 
 func (s cgroupV2Statter) memoryMaxBytes() (*float64, error) {

stat_internal_test.go

@@ -440,6 +440,32 @@ func TestStatter(t *testing.T) {
 				assert.Equal(t, "cores", cpu.Unit)
 			})
 
+			t.Run("CPU/UsageFromLimitOwner", func(t *testing.T) {
+				t.Parallel()
+
+				fs := initFS(t, fsContainerCgroupV2DockerInDockerCPU)
+				fakeWait := func(time.Duration) {
+					// The root's usage includes both children: 10000us from
+					// /init.scope and 190000us from the sibling /docker/<id>.
+					mungeFS(t, fs, filepath.Join(cgroupRootPath, cgroupV2CPUStat), "usage_usec 200000")
+					mungeFS(t, fs, filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUStat), "usage_usec 10000")
+					mungeFS(t, fs, filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUStat), "usage_usec 190000")
+				}
+				s, err := New(WithFS(fs), withWait(fakeWait), withIsCgroupV2(true))
+				require.NoError(t, err)
+
+				cpu, err := s.ContainerCPU()
+				require.NoError(t, err)
+
+				require.NotNil(t, cpu)
+				// Usage comes from the root that owns cpu.max, which
+				// includes the sibling /docker/<id> cgroup.
+				assert.Equal(t, 2.0, cpu.Used)
+				require.NotNil(t, cpu.Total)
+				assert.Equal(t, 2.5, *cpu.Total)
+				assert.Equal(t, "cores", cpu.Unit)
+			})
+
 			t.Run("Memory/Limit", func(t *testing.T) {
 				t.Parallel()
 
@@ -819,6 +845,26 @@ sysboxfs /proc/sys sysboxfs rw,nosuid,nodev,noexec,relatime 0 0`,
 		filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryStat):       "inactive_file 268435456",
 		filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryUsageBytes): "536870912",
 	}
+	// fsContainerCgroupV2DockerInDockerCPU simulates a docker-in-docker workspace.
+	// The agent runs in /init.scope and a nested container runs in the
+	// sibling /docker/<id>. Only the container root sets cpu.max, so
+	// usage must be read from the root to include both children.
+	fsContainerCgroupV2DockerInDockerCPU = map[string]string{
+		procOneCgroup:  "0::/init.scope",
+		procSelfCgroup: "0::/init.scope",
+		procMounts: `overlay / overlay rw,relatime,lowerdir=/some/path:/some/path,upperdir=/some/path:/some/path,workdir=/some/path:/some/path 0 0
+proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0`,
+		sysCgroupType: "domain",
+
+		filepath.Join(cgroupRootPath, cgroupV2CPUMax):  "250000 100000",
+		filepath.Join(cgroupRootPath, cgroupV2CPUStat): "usage_usec 0",
+
+		filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUMax):  "max 100000",
+		filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUStat): "usage_usec 0",
+
+		filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUMax):  "max 100000",
+		filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUStat): "usage_usec 0",
+	}
 	// fsContainerCgroupV2InitScopeNoCPUMax simulates a scenario where cpu.max
 	// doesn't exist at any level in the hierarchy. Tests the default period fallback.
 	fsContainerCgroupV2InitScopeNoCPUMax = map[string]string{