diff --git a/cgroupv2.go b/cgroupv2.go index e3b7aaa..add70ce 100644 --- a/cgroupv2.go +++ b/cgroupv2.go @@ -89,37 +89,78 @@ func getFromParentOrFallback[T any]( } func (s cgroupV2Statter) cpuUsed() (used float64, err error) { - cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat) + periodUs, err := s.cpuPeriod() + if err != nil { + return 0, xerrors.Errorf("get cpu period: %w", err) + } - usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec") + limit, err := s.cpuLimit() if err != nil { - return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err) + return 0, xerrors.Errorf("get cpu quota: %w", err) } - periodUs, err := s.cpuPeriod() + + // Read usage from the cgroup that owns the limit so that used and + // total describe the same domain. Otherwise sibling cgroups under + // the limit owner (for example /docker/ next to /init.scope in + // docker-in-docker) would count against the limit but not the usage. + if limit != nil && limit.owner.path != s.path { + if usedUs, err := limit.owner.cpuUsedUs(); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return 0, xerrors.Errorf("read limit owner cpu: %w", err) + } + // The limit owner does not expose usage files, so fall + // back to the process's own cgroup below. + } else { + return float64(usedUs) / periodUs, nil + } + } + + usedUs, err := s.cpuUsedUs() if err != nil { - return 0, xerrors.Errorf("get cpu period: %w", err) + return 0, err } - return float64(usageUs) / periodUs, nil + return float64(usedUs) / periodUs, nil } -func (s cgroupV2Statter) cpuQuota() (float64, error) { +// cpuLimit is the effective cpu.max quota and the cgroup that sets it. +// owner is kept so usage can be read from the same cgroup the limit is +// enforced on. +type cpuLimit struct { + owner *cgroupV2Statter + quotaUs float64 +} + +// cpuLimit returns the nearest cgroup, starting at s and walking up, +// that sets a cpu.max quota. It returns nil when no cgroup sets a limit. +func (s *cgroupV2Statter) cpuLimit() (*cpuLimit, error) { cpuMaxPath := filepath.Join(s.path, cgroupV2CPUMax) quotaUs, err := readInt64SepIdx(s.fs, cpuMaxPath, " ", 0) if err != nil { if !errors.Is(err, strconv.ErrSyntax) && !errors.Is(err, fs.ErrNotExist) { - return 0, xerrors.Errorf("get cpu quota: %w", err) + return nil, xerrors.Errorf("get cpu quota: %w", err) } // If the value is not a valid integer, assume it is the string - // 'max' and that there is no limit set. Try the parent or return -1. - return getFromParentOrFallback(&s, (*cgroupV2Statter).cpuQuota, func() (float64, error) { - return -1.0, nil + // 'max' and that there is no limit set. Try the parent or return nil (no limit). + return getFromParentOrFallback(s, (*cgroupV2Statter).cpuLimit, func() (*cpuLimit, error) { + return nil, nil }) } - return float64(quotaUs), nil + return &cpuLimit{owner: s, quotaUs: float64(quotaUs)}, nil +} + +func (s cgroupV2Statter) cpuUsedUs() (int64, error) { + cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat) + + usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec") + if err != nil { + return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err) + } + + return usageUs, nil } func (s cgroupV2Statter) cpuPeriod() (float64, error) { @@ -148,12 +189,15 @@ func (s cgroupV2Statter) cpuTotal() (total float64, err error) { return 0, xerrors.Errorf("get cpu period: %w", err) } - quotaUs, err := s.cpuQuota() + limit, err := s.cpuLimit() if err != nil { return 0, xerrors.Errorf("get cpu quota: %w", err) } + if limit == nil { + return -1.0 / periodUs, nil + } - return quotaUs / periodUs, nil + return limit.quotaUs / periodUs, nil } func (s cgroupV2Statter) memoryMaxBytes() (*float64, error) { diff --git a/stat_internal_test.go b/stat_internal_test.go index cc139de..dc246c2 100644 --- a/stat_internal_test.go +++ b/stat_internal_test.go @@ -440,6 +440,32 @@ func TestStatter(t *testing.T) { assert.Equal(t, "cores", cpu.Unit) }) + t.Run("CPU/UsageFromLimitOwner", func(t *testing.T) { + t.Parallel() + + fs := initFS(t, fsContainerCgroupV2DockerInDockerCPU) + fakeWait := func(time.Duration) { + // The root's usage includes both children: 10000us from + // /init.scope and 190000us from the sibling /docker/. + mungeFS(t, fs, filepath.Join(cgroupRootPath, cgroupV2CPUStat), "usage_usec 200000") + mungeFS(t, fs, filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUStat), "usage_usec 10000") + mungeFS(t, fs, filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUStat), "usage_usec 190000") + } + s, err := New(WithFS(fs), withWait(fakeWait), withIsCgroupV2(true)) + require.NoError(t, err) + + cpu, err := s.ContainerCPU() + require.NoError(t, err) + + require.NotNil(t, cpu) + // Usage comes from the root that owns cpu.max, which + // includes the sibling /docker/ cgroup. + assert.Equal(t, 2.0, cpu.Used) + require.NotNil(t, cpu.Total) + assert.Equal(t, 2.5, *cpu.Total) + assert.Equal(t, "cores", cpu.Unit) + }) + t.Run("Memory/Limit", func(t *testing.T) { t.Parallel() @@ -819,6 +845,26 @@ sysboxfs /proc/sys sysboxfs rw,nosuid,nodev,noexec,relatime 0 0`, filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryStat): "inactive_file 268435456", filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryUsageBytes): "536870912", } + // fsContainerCgroupV2DockerInDockerCPU simulates a docker-in-docker workspace. + // The agent runs in /init.scope and a nested container runs in the + // sibling /docker/. Only the container root sets cpu.max, so + // usage must be read from the root to include both children. + fsContainerCgroupV2DockerInDockerCPU = map[string]string{ + procOneCgroup: "0::/init.scope", + procSelfCgroup: "0::/init.scope", + procMounts: `overlay / overlay rw,relatime,lowerdir=/some/path:/some/path,upperdir=/some/path:/some/path,workdir=/some/path:/some/path 0 0 +proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0`, + sysCgroupType: "domain", + + filepath.Join(cgroupRootPath, cgroupV2CPUMax): "250000 100000", + filepath.Join(cgroupRootPath, cgroupV2CPUStat): "usage_usec 0", + + filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUMax): "max 100000", + filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUStat): "usage_usec 0", + + filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUMax): "max 100000", + filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2CPUStat): "usage_usec 0", + } // fsContainerCgroupV2InitScopeNoCPUMax simulates a scenario where cpu.max // doesn't exist at any level in the hierarchy. Tests the default period fallback. fsContainerCgroupV2InitScopeNoCPUMax = map[string]string{