From c2d1a2acd2460f62f6ea7c57ae1b7d15fb57e009 Mon Sep 17 00:00:00 2001 From: default Date: Sun, 25 Oct 2026 20:39:45 +0000 Subject: [PATCH] fix: report the cgroupv2 limit with the least room left for container cpu and memory --- README.md | 39 ++++++++ cgroup.go | 86 ++++++++++++----- cgroupv1.go | 9 ++ cgroupv2.go | 213 ++++++++++++++++++++++++++++++++---------- stat_internal_test.go | 163 ++++++++++++++++++++++++++++++++ 5 files changed, 438 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 4cbfb01..a8c47cb 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,45 @@ Container CPU: 0.5/2.5 cores (20%) Container Memory: 256/1024 MiB (25%) ``` +## How container limits are reported + +On cgroup v2, a container's cgroup can have child cgroups. For example, in a +docker-in-docker workspace the process runs in `/init.scope` and nested +containers run in `/docker/`. Limits can be set at any level, and the +kernel enforces every limit from the process's cgroup up to the container root. + +`ContainerCPU` and `ContainerMemory` handle this as follows: + +1. Check each cgroup from the process's cgroup up to the container root for a + limit (`cpu.max` for CPU, `memory.max` for memory). +1. For each limit found, read usage from the same cgroup that sets it. This + usage includes every cgroup below it, so processes that share the limit + are counted. + - CPU: the change in `usage_usec` from `cpu.stat` over the sample interval. + - Memory: `memory.current` minus `inactive_file` from `memory.stat`. +1. Report the limit with the least room left (limit minus usage), since that + is the one that will be hit first. If two limits have the same room left, + the one nearest to the process is reported. +1. If no cgroup sets a limit, report usage of the process's own cgroup with + no total. + +If a cgroup sets a limit but does not have the usage files, usage of the +process's own cgroup is used for that limit. + +Only one limit is reported. The result answers "how close is this process to +being throttled or killed," not "what are all the limits." + +### Limits outside the container + +clistat can only see cgroups inside the container's cgroup namespace. Limits +set above the container root are not visible. For example, a Kubernetes +pod-level limit is not visible from inside a container that has its own +cgroup namespace. If such a limit is tighter than the container's own limit, +clistat reports the container's limit. + +On cgroup v1, clistat reads the limit and usage from the container's own +cgroup only. + ## Testing The library includes a comprehensive test suite that can be run with: diff --git a/cgroup.go b/cgroup.go index de06a05..4eb7e3d 100644 --- a/cgroup.go +++ b/cgroup.go @@ -30,11 +30,20 @@ const ( var errNotContainerized = errors.New("not containerized") type cgroupStatter interface { - cpuUsed() (used float64, err error) - cpuTotal() (total float64, err error) + cpuCounters() ([]cpuCounter, error) memory(p Prefix) (*Result, error) } +// cpuCounter is a cumulative CPU usage counter paired with the quota that +// applies to it. +type cpuCounter struct { + // total is the quota in cores. A value <= 0 means no quota. + total float64 + // used returns the cumulative usage counter, in the same unit as total + // once the difference between two samples is taken. + used func() (float64, error) +} + func (s *Statter) getCgroupStatter() (cgroupStatter, error) { isContainerized, err := s.IsContainerized() if err != nil { @@ -60,21 +69,27 @@ func (s *Statter) getCgroupStatter() (cgroupStatter, error) { // This is calculated as difference of two samples of the // CPU usage of the container cgroup. // The total is read from the relevant path in /sys/fs/cgroup. -// If there is no limit set, the total is assumed to be the -// number of host cores multiplied by the CFS period. +// On cgroup v2, every quota from the process's cgroup up to the container +// root applies, so the one with the least room left is reported, with +// usage read from the cgroup that sets it. +// If there is no limit set, Total is nil. // If the system is not containerized, this always returns nil. func (s *Statter) ContainerCPU() (*Result, error) { if s.cgroupStatter == nil { return nil, nil //nolint: nilnil } - total, err := s.cgroupStatter.cpuTotal() + counters, err := s.cgroupStatter.cpuCounters() if err != nil { - return nil, xerrors.Errorf("get total cpu: %w", err) + return nil, xerrors.Errorf("get cpu counters: %w", err) } - used1, err := s.cgroupStatter.cpuUsed() - if err != nil { - return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) + + first := make([]float64, len(counters)) + for i, c := range counters { + first[i], err = c.used() + if err != nil { + return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) + } } // The measurements in /sys/fs/cgroup are counters. @@ -83,26 +98,44 @@ func (s *Statter) ContainerCPU() (*Result, error) { // We can't do anything about that. s.wait(s.sampleInterval) - used2, err := s.cgroupStatter.cpuUsed() - if err != nil { - return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) - } + var ( + best *Result + bestHeadroom float64 + ) + for i, c := range counters { + used2, err := c.used() + if err != nil { + return nil, xerrors.Errorf("get cgroup CPU usage: %w", err) + } - if used2 < used1 { - // Someone reset the counter. Best we can do is count from zero. - used1 = 0 - } + used1 := first[i] + if used2 < used1 { + // Someone reset the counter. Best we can do is count from zero. + used1 = 0 + } - r := &Result{ - Unit: "cores", - Used: used2 - used1, - Prefix: PrefixDefault, - } + r := &Result{ + Unit: "cores", + Used: used2 - used1, + Prefix: PrefixDefault, + } + if c.total <= 0 { + if best == nil { + best = r + } + continue + } - if total > 0 { - r.Total = ptr.To(total) + // Ties go to the earlier counter, which is nearer to the process. + r.Total = ptr.To(c.total) + headroom := c.total - r.Used + if best == nil || best.Total == nil || headroom < bestHeadroom { + best = r + bestHeadroom = headroom + } } - return r, nil + + return best, nil } func (s *Statter) isCgroupV2() bool { @@ -110,6 +143,9 @@ func (s *Statter) isCgroupV2() bool { } // ContainerMemory returns the memory usage of the container cgroup. +// On cgroup v2, every memory.max from the process's cgroup up to the +// container root applies, so the one with the least room left is +// reported, with usage read from the cgroup that sets it. // If the system is not containerized, this always returns nil. func (s *Statter) ContainerMemory(p Prefix) (*Result, error) { if s.cgroupStatter == nil { diff --git a/cgroupv1.go b/cgroupv1.go index 13e4baa..f2a64f4 100644 --- a/cgroupv1.go +++ b/cgroupv1.go @@ -39,6 +39,15 @@ type cgroupV1Statter struct { fs afero.Fs } +func (s cgroupV1Statter) cpuCounters() ([]cpuCounter, error) { + total, err := s.cpuTotal() + if err != nil { + return nil, xerrors.Errorf("get total cpu: %w", err) + } + + return []cpuCounter{{total: total, used: s.cpuUsed}}, nil +} + func (s cgroupV1Statter) cpuUsed() (used float64, err error) { usageNs, err := readInt64(s.fs, cgroupV1CPUAcctUsage) if err != nil { diff --git a/cgroupv2.go b/cgroupv2.go index e3b7aaa..082051a 100644 --- a/cgroupv2.go +++ b/cgroupv2.go @@ -88,38 +88,103 @@ func getFromParentOrFallback[T any]( return fallback() } -func (s cgroupV2Statter) cpuUsed() (used float64, err error) { - cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat) +// cpuCounters returns one counter for every cgroup, from s up to the +// root, whose cpu.max sets a quota. The kernel enforces each of these +// quotas, so the caller reports whichever has the least room left. +// +// Each counter reads usage from the cgroup that sets the quota. cpu.stat +// includes all descendants, so this counts sibling cgroups that share the +// quota (for example /docker/ next to /init.scope in docker-in-docker). +// +// If no cgroup sets a quota, it returns a single unlimited counter for s. +func (s cgroupV2Statter) cpuCounters() ([]cpuCounter, error) { + var counters []cpuCounter + for c := &s; c != nil; c = c.parent { + quotaUs, periodUs, ok, err := c.cpuMaxLocal() + if err != nil { + return nil, err + } + if !ok { + continue + } - usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec") - if err != nil { - return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err) + counters = append(counters, cpuCounter{ + total: quotaUs / periodUs, + used: func() (float64, error) { + usageUs, err := s.cpuUsageUsAt(c) + if err != nil { + return 0, err + } + return float64(usageUs) / periodUs, nil + }, + }) + } + if len(counters) > 0 { + return counters, nil } + periodUs, err := s.cpuPeriod() if err != nil { - return 0, xerrors.Errorf("get cpu period: %w", err) + return nil, xerrors.Errorf("get cpu period: %w", err) } - return float64(usageUs) / periodUs, nil + return []cpuCounter{{ + used: func() (float64, error) { + usageUs, err := s.cpuUsageUsLocal() + if err != nil { + return 0, err + } + return float64(usageUs) / periodUs, nil + }, + }}, nil } -func (s cgroupV2Statter) cpuQuota() (float64, error) { +// cpuMaxLocal reads this cgroup's own cpu.max. ok is false when the file +// does not exist or the quota is "max". +func (s cgroupV2Statter) cpuMaxLocal() (quotaUs, periodUs float64, ok bool, err error) { cpuMaxPath := filepath.Join(s.path, cgroupV2CPUMax) - quotaUs, err := readInt64SepIdx(s.fs, cpuMaxPath, " ", 0) + quota, err := readInt64SepIdx(s.fs, cpuMaxPath, " ", 0) if err != nil { if !errors.Is(err, strconv.ErrSyntax) && !errors.Is(err, fs.ErrNotExist) { - return 0, xerrors.Errorf("get cpu quota: %w", err) + return 0, 0, false, xerrors.Errorf("get cpu quota: %w", err) } + return 0, 0, false, nil + } - // If the value is not a valid integer, assume it is the string - // 'max' and that there is no limit set. Try the parent or return -1. - return getFromParentOrFallback(&s, (*cgroupV2Statter).cpuQuota, func() (float64, error) { - return -1.0, nil - }) + period, err := readInt64SepIdx(s.fs, cpuMaxPath, " ", 1) + if err != nil { + return 0, 0, false, xerrors.Errorf("get cpu period: %w", err) + } + + return float64(quota), float64(period), true, nil +} + +// cpuUsageUsAt returns usage_usec for the quota set on c. If c does not +// expose cpu.stat, it falls back to the process's own cgroup. +func (s cgroupV2Statter) cpuUsageUsAt(c *cgroupV2Statter) (int64, error) { + if usageUs, err := c.cpuUsageUsLocal(); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return 0, xerrors.Errorf("read limit owner cpu: %w", err) + } + // The limit owner does not expose cpu.stat, so fall back to the + // process's own cgroup below. + } else { + return usageUs, nil } - return float64(quotaUs), nil + return s.cpuUsageUsLocal() +} + +func (s cgroupV2Statter) cpuUsageUsLocal() (int64, error) { + cpuStatPath := filepath.Join(s.path, cgroupV2CPUStat) + + usageUs, err := readInt64Prefix(s.fs, cpuStatPath, "usage_usec") + if err != nil { + return 0, xerrors.Errorf("get cgroupv2 cpu used: %w", err) + } + + return usageUs, nil } func (s cgroupV2Statter) cpuPeriod() (float64, error) { @@ -142,38 +207,73 @@ func (s cgroupV2Statter) cpuPeriod() (float64, error) { return float64(periodUs), nil } -func (s cgroupV2Statter) cpuTotal() (total float64, err error) { - periodUs, err := s.cpuPeriod() +// memoryMaxLocal reads this cgroup's own memory.max. ok is false when the +// file does not exist or the value is "max". +func (s cgroupV2Statter) memoryMaxLocal() (maxBytes float64, ok bool, err error) { + memoryMaxPath := filepath.Join(s.path, cgroupV2MemoryMaxBytes) + + maxUsageBytes, err := readInt64(s.fs, memoryMaxPath) if err != nil { - return 0, xerrors.Errorf("get cpu period: %w", err) + if !errors.Is(err, strconv.ErrSyntax) && !errors.Is(err, fs.ErrNotExist) { + return 0, false, xerrors.Errorf("read memory total: %w", err) + } + return 0, false, nil } - quotaUs, err := s.cpuQuota() - if err != nil { - return 0, xerrors.Errorf("get cpu quota: %w", err) + return float64(maxUsageBytes), true, nil +} + +// memoryUsedAt returns memory.current minus inactive_file for the limit +// set on c. memory.current includes all descendants, so this counts +// sibling cgroups that share the limit. If c does not expose these files, +// it falls back to the process's own cgroup. +func (s cgroupV2Statter) memoryUsedAt(c *cgroupV2Statter) (float64, error) { + if used, err := c.memoryUsedLocal(); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return 0, xerrors.Errorf("read limit owner memory: %w", err) + } + // The limit owner does not expose usage files, so fall back to + // the process's own cgroup below. + } else { + return float64(used), nil } - return quotaUs / periodUs, nil + used, err := s.memoryUsed() + if err != nil { + return 0, err + } + return float64(used), nil } -func (s cgroupV2Statter) memoryMaxBytes() (*float64, error) { - memoryMaxPath := filepath.Join(s.path, cgroupV2MemoryMaxBytes) +// memoryUsedLocal reads usage from this cgroup's own files only. +func (s cgroupV2Statter) memoryUsedLocal() (int64, error) { + currUsageBytes, err := readInt64(s.fs, filepath.Join(s.path, cgroupV2MemoryUsageBytes)) + if err != nil { + return 0, xerrors.Errorf("read memory current: %w", err) + } - maxUsageBytes, err := readInt64(s.fs, memoryMaxPath) + inactiveFileBytes, err := readInt64Prefix(s.fs, filepath.Join(s.path, cgroupV2MemoryStat), "inactive_file") if err != nil { - if !errors.Is(err, strconv.ErrSyntax) && !errors.Is(err, fs.ErrNotExist) { - return nil, xerrors.Errorf("read memory total: %w", err) - } + return 0, xerrors.Errorf("read memory stat inactive_file: %w", err) + } - // If the value is not a valid integer _or_ the memory max file - // does not exist, we can assume that the limit is 'max'. - // Try the parent or return nil (no limit). - return getFromParentOrFallback(&s, (*cgroupV2Statter).memoryMaxBytes, func() (*float64, error) { - return nil, nil - }) + return currUsageBytes - inactiveFileBytes, nil +} + +// memoryUsed reads usage for this cgroup, taking each file from the +// nearest cgroup that has it. +func (s cgroupV2Statter) memoryUsed() (int64, error) { + currUsageBytes, err := s.memoryCurrentBytes() + if err != nil { + return 0, xerrors.Errorf("read memory usage: %w", err) + } + + inactiveFileBytes, err := s.memoryInactiveFileBytes() + if err != nil { + return 0, xerrors.Errorf("read memory stats: %w", err) } - return ptr.To(float64(maxUsageBytes)), nil + return currUsageBytes - inactiveFileBytes, nil } func (s cgroupV2Statter) memoryCurrentBytes() (int64, error) { @@ -218,22 +318,41 @@ func (s cgroupV2Statter) memory(p Prefix) (*Result, error) { Unit: "B", Prefix: p, } - if total, err := s.memoryMaxBytes(); err != nil { - return nil, xerrors.Errorf("read memory total: %w", err) - } else { - r.Total = total - } - currUsageBytes, err := s.memoryCurrentBytes() - if err != nil { - return nil, xerrors.Errorf("read memory usage: %w", err) + // The kernel charges memory to every cgroup from s up to the root and + // fails the allocation if any of them is over its memory.max. Report + // the limit with the least room left, since that one is hit first. + // Ties go to the cgroup nearest to s. + var headroom float64 + for c := &s; c != nil; c = c.parent { + limit, ok, err := c.memoryMaxLocal() + if err != nil { + return nil, err + } + if !ok { + continue + } + + used, err := s.memoryUsedAt(c) + if err != nil { + return nil, err + } + + if r.Total == nil || limit-used < headroom { + r.Total = ptr.To(limit) + r.Used = used + headroom = limit - used + } + } + if r.Total != nil { + return r, nil } - inactiveFileBytes, err := s.memoryInactiveFileBytes() + used, err := s.memoryUsed() if err != nil { - return nil, xerrors.Errorf("read memory stats: %w", err) + return nil, err } - r.Used = float64(currUsageBytes - inactiveFileBytes) + r.Used = float64(used) return r, nil } diff --git a/stat_internal_test.go b/stat_internal_test.go index cc139de..806dd44 100644 --- a/stat_internal_test.go +++ b/stat_internal_test.go @@ -522,6 +522,169 @@ func TestStatter(t *testing.T) { assert.Equal(t, "B", mem.Unit) }) }) + + t.Run("NestedLimits", func(t *testing.T) { + t.Parallel() + + // The process runs in /init.scope. A sibling /docker/ + // shares the container root's limits. + const ( + mib = 1 << 20 + gib = 1 << 30 + docker = "docker/aa86ac98959eeedeae0ecb6e0c9ddd8ae8b97a9d0fdccccf7ea7a474f4e0bb1f" + ) + root := cgroupRootPath + initScope := filepath.Join(cgroupRootPath, "init.scope") + sibling := filepath.Join(cgroupRootPath, docker) + + memoryFS := func(t *testing.T, initScopeMax string, rootUsed, initScopeUsed int64) afero.Fs { + t.Helper() + return initFS(t, map[string]string{ + procOneCgroup: "0::/init.scope", + procSelfCgroup: "0::/init.scope", + procMounts: fsContainerCgroupV2[procMounts], + sysCgroupType: "domain", + + filepath.Join(root, cgroupV2MemoryMaxBytes): strconv.Itoa(gib), + filepath.Join(root, cgroupV2MemoryUsageBytes): strconv.FormatInt(rootUsed, 10), + filepath.Join(root, cgroupV2MemoryStat): "inactive_file 0", + filepath.Join(initScope, cgroupV2MemoryMaxBytes): initScopeMax, + filepath.Join(initScope, cgroupV2MemoryUsageBytes): strconv.FormatInt(initScopeUsed, 10), + filepath.Join(initScope, cgroupV2MemoryStat): "inactive_file 0", + filepath.Join(sibling, cgroupV2MemoryMaxBytes): "max", + filepath.Join(sibling, cgroupV2MemoryUsageBytes): strconv.FormatInt(rootUsed-initScopeUsed, 10), + filepath.Join(sibling, cgroupV2MemoryStat): "inactive_file 0", + }) + } + + cpuFS := func(t *testing.T, initScopeMax string) afero.Fs { + t.Helper() + return initFS(t, map[string]string{ + procOneCgroup: "0::/init.scope", + procSelfCgroup: "0::/init.scope", + procMounts: fsContainerCgroupV2[procMounts], + sysCgroupType: "domain", + + filepath.Join(root, cgroupV2CPUMax): "200000 100000", + filepath.Join(root, cgroupV2CPUStat): "usage_usec 0", + filepath.Join(initScope, cgroupV2CPUMax): initScopeMax, + filepath.Join(initScope, cgroupV2CPUStat): "usage_usec 0", + filepath.Join(sibling, cgroupV2CPUMax): "max 100000", + filepath.Join(sibling, cgroupV2CPUStat): "usage_usec 0", + }) + } + + // sampleCPU makes /init.scope use initScopeUs and the sibling + // use siblingUs during the sample interval. + sampleCPU := func(t *testing.T, fs afero.Fs, initScopeUs, siblingUs int64) func(time.Duration) { + return func(time.Duration) { + mungeFS(t, fs, filepath.Join(root, cgroupV2CPUStat), fmt.Sprintf("usage_usec %d", initScopeUs+siblingUs)) + mungeFS(t, fs, filepath.Join(initScope, cgroupV2CPUStat), fmt.Sprintf("usage_usec %d", initScopeUs)) + mungeFS(t, fs, filepath.Join(sibling, cgroupV2CPUStat), fmt.Sprintf("usage_usec %d", siblingUs)) + } + } + + t.Run("Memory/UsageFromLimitOwner", func(t *testing.T) { + t.Parallel() + + // Only the root sets a limit. Usage must include the sibling. + fs := memoryFS(t, "max", 850*mib, 10*mib) + s, err := New(WithFS(fs), withNoWait, withIsCgroupV2(true)) + require.NoError(t, err) + + mem, err := s.ContainerMemory(PrefixDefault) + require.NoError(t, err) + require.NotNil(t, mem) + assert.Equal(t, float64(850*mib), mem.Used) + require.NotNil(t, mem.Total) + assert.Equal(t, float64(gib), *mem.Total) + }) + + t.Run("Memory/AncestorLimitHitFirst", func(t *testing.T) { + t.Parallel() + + // /init.scope has a 600 MiB limit and uses 10 MiB, but the + // sibling has used most of the root's 1 GiB. The root has + // less room left, so it is reported. + fs := memoryFS(t, strconv.Itoa(600*mib), 860*mib, 10*mib) + s, err := New(WithFS(fs), withNoWait, withIsCgroupV2(true)) + require.NoError(t, err) + + mem, err := s.ContainerMemory(PrefixDefault) + require.NoError(t, err) + require.NotNil(t, mem) + assert.Equal(t, float64(860*mib), mem.Used) + require.NotNil(t, mem.Total) + assert.Equal(t, float64(gib), *mem.Total) + }) + + t.Run("Memory/NearestLimitHitFirst", func(t *testing.T) { + t.Parallel() + + // /init.scope uses 550 of its 600 MiB, while the root has + // 400 MiB left. /init.scope has less room, so it is reported. + fs := memoryFS(t, strconv.Itoa(600*mib), 624*mib, 550*mib) + s, err := New(WithFS(fs), withNoWait, withIsCgroupV2(true)) + require.NoError(t, err) + + mem, err := s.ContainerMemory(PrefixDefault) + require.NoError(t, err) + require.NotNil(t, mem) + assert.Equal(t, float64(550*mib), mem.Used) + require.NotNil(t, mem.Total) + assert.Equal(t, float64(600*mib), *mem.Total) + }) + + t.Run("CPU/UsageFromLimitOwner", func(t *testing.T) { + t.Parallel() + + // Only the root sets a quota. Usage must include the sibling. + fs := cpuFS(t, "max 100000") + s, err := New(WithFS(fs), withWait(sampleCPU(t, fs, 1000, 150000)), withIsCgroupV2(true)) + require.NoError(t, err) + + cpu, err := s.ContainerCPU() + require.NoError(t, err) + require.NotNil(t, cpu) + assert.Equal(t, 1.51, cpu.Used) + require.NotNil(t, cpu.Total) + assert.Equal(t, 2.0, *cpu.Total) + }) + + t.Run("CPU/AncestorLimitHitFirst", func(t *testing.T) { + t.Parallel() + + // /init.scope has a 1.5 core quota and uses 0.01, but the + // sibling uses 1.8 of the root's 2 cores. + fs := cpuFS(t, "150000 100000") + s, err := New(WithFS(fs), withWait(sampleCPU(t, fs, 1000, 180000)), withIsCgroupV2(true)) + require.NoError(t, err) + + cpu, err := s.ContainerCPU() + require.NoError(t, err) + require.NotNil(t, cpu) + assert.Equal(t, 1.81, cpu.Used) + require.NotNil(t, cpu.Total) + assert.Equal(t, 2.0, *cpu.Total) + }) + + t.Run("CPU/NearestLimitHitFirst", func(t *testing.T) { + t.Parallel() + + // /init.scope uses 1.4 of its 1.5 cores, while the root + // has 0.5 cores left. /init.scope has less room. + fs := cpuFS(t, "150000 100000") + s, err := New(WithFS(fs), withWait(sampleCPU(t, fs, 140000, 10000)), withIsCgroupV2(true)) + require.NoError(t, err) + + cpu, err := s.ContainerCPU() + require.NoError(t, err) + require.NotNil(t, cpu) + assert.Equal(t, 1.4, cpu.Used) + require.NotNil(t, cpu.Total) + assert.Equal(t, 1.5, *cpu.Total) + }) + }) }) }