From 31b693cfb86edc6289a7752f23a01b2c8f0c59ba Mon Sep 17 00:00:00 2001 From: default Date: Fri, 23 Oct 2026 19:03:56 +0000 Subject: [PATCH 1/2] fix: read cgroupv2 memory usage from the cgroup that owns memory.max --- cgroupv2.go | 61 ++++++++++++++++++++++++++++++++++--------- stat_internal_test.go | 44 +++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/cgroupv2.go b/cgroupv2.go index e3b7aaa..f871c03 100644 --- a/cgroupv2.go +++ b/cgroupv2.go @@ -156,7 +156,15 @@ func (s cgroupV2Statter) cpuTotal() (total float64, err error) { return quotaUs / periodUs, nil } -func (s cgroupV2Statter) memoryMaxBytes() (*float64, error) { +// memoryLimit is the effective memory.max and the cgroup that sets it. +type memoryLimit struct { + owner *cgroupV2Statter + bytes float64 +} + +// memoryLimit returns the nearest cgroup, starting at s and walking up, +// that sets memory.max. It returns nil when no cgroup sets a limit. +func (s *cgroupV2Statter) memoryLimit() (*memoryLimit, error) { memoryMaxPath := filepath.Join(s.path, cgroupV2MemoryMaxBytes) maxUsageBytes, err := readInt64(s.fs, memoryMaxPath) @@ -168,12 +176,27 @@ func (s cgroupV2Statter) memoryMaxBytes() (*float64, error) { // If the value is not a valid integer _or_ the memory max file // does not exist, we can assume that the limit is 'max'. // Try the parent or return nil (no limit). - return getFromParentOrFallback(&s, (*cgroupV2Statter).memoryMaxBytes, func() (*float64, error) { + return getFromParentOrFallback(s, (*cgroupV2Statter).memoryLimit, func() (*memoryLimit, error) { return nil, nil }) } - return ptr.To(float64(maxUsageBytes)), nil + return &memoryLimit{owner: s, bytes: float64(maxUsageBytes)}, nil +} + +// memoryUsedBytes returns memory.current minus inactive_file. +func (s cgroupV2Statter) memoryUsedBytes() (int64, error) { + currUsageBytes, err := s.memoryCurrentBytes() + if err != nil { + return 0, xerrors.Errorf("read memory usage: %w", err) + } + + inactiveFileBytes, err := s.memoryInactiveFileBytes() + if err != nil { + return 0, xerrors.Errorf("read memory stats: %w", err) + } + + return currUsageBytes - inactiveFileBytes, nil } func (s cgroupV2Statter) memoryCurrentBytes() (int64, error) { @@ -218,22 +241,36 @@ func (s cgroupV2Statter) memory(p Prefix) (*Result, error) { Unit: "B", Prefix: p, } - if total, err := s.memoryMaxBytes(); err != nil { + limit, err := s.memoryLimit() + if err != nil { return nil, xerrors.Errorf("read memory total: %w", err) - } else { - r.Total = total } - currUsageBytes, err := s.memoryCurrentBytes() - if err != nil { - return nil, xerrors.Errorf("read memory usage: %w", err) + // Read usage from the cgroup that owns the limit so that used and + // total describe the same domain. Otherwise sibling cgroups under + // the limit owner (for example /docker/ next to /init.scope in + // docker-in-docker) would count against the limit but not the usage. + if limit != nil { + r.Total = ptr.To(limit.bytes) + if limit.owner.path != s.path { + used, err := limit.owner.memoryUsedBytes() + if err == nil { + r.Used = float64(used) + return r, nil + } + if !errors.Is(err, fs.ErrNotExist) { + return nil, xerrors.Errorf("read limit owner memory: %w", err) + } + // The limit owner does not expose usage files, so fall back + // to the process's own cgroup. + } } - inactiveFileBytes, err := s.memoryInactiveFileBytes() + used, err := s.memoryUsedBytes() if err != nil { - return nil, xerrors.Errorf("read memory stats: %w", err) + return nil, err } - r.Used = float64(currUsageBytes - inactiveFileBytes) + r.Used = float64(used) return r, nil } diff --git a/stat_internal_test.go b/stat_internal_test.go index cc139de..7c0797a 100644 --- a/stat_internal_test.go +++ b/stat_internal_test.go @@ -507,6 +507,25 @@ func TestStatter(t *testing.T) { assert.Equal(t, "B", mem.Unit) }) + t.Run("Memory/UsageFromLimitOwner", func(t *testing.T) { + t.Parallel() + + fs := initFS(t, fsContainerCgroupV2DockerInDocker) + s, err := New(WithFS(fs), withNoWait, withIsCgroupV2(true)) + require.NoError(t, err) + + mem, err := s.ContainerMemory(PrefixDefault) + require.NoError(t, err) + + require.NotNil(t, mem) + // Usage comes from the root that owns memory.max, which + // includes the sibling /docker/ cgroup. + assert.Equal(t, 805306368.0, mem.Used) + require.NotNil(t, mem.Total) + assert.Equal(t, 1073741824.0, *mem.Total) + assert.Equal(t, "B", mem.Unit) + }) + t.Run("Memory/StatInParent", func(t *testing.T) { t.Parallel() @@ -835,6 +854,31 @@ sysboxfs /proc/sys sysboxfs rw,nosuid,nodev,noexec,relatime 0 0`, filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryStat): "inactive_file 268435456", filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryUsageBytes): "536870912", } + // fsContainerCgroupV2DockerInDocker simulates a docker-in-docker workspace. + // The agent runs in /init.scope and a nested container runs in the + // sibling /docker/. Only the container root sets memory.max, so + // usage must be read from the root to include both children. + fsContainerCgroupV2DockerInDocker = map[string]string{ + procOneCgroup: "0::/init.scope", + procSelfCgroup: "0::/init.scope", + procMounts: `overlay / overlay rw,relatime,lowerdir=/some/path:/some/path,upperdir=/some/path:/some/path,workdir=/some/path:/some/path 0 0 +proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0`, + sysCgroupType: "domain", + + filepath.Join(cgroupRootPath, cgroupV2CPUMax): "max 100000", + filepath.Join(cgroupRootPath, cgroupV2MemoryMaxBytes): "1073741824", + filepath.Join(cgroupRootPath, cgroupV2MemoryUsageBytes): "939524096", + filepath.Join(cgroupRootPath, cgroupV2MemoryStat): "inactive_file 134217728", + + filepath.Join(cgroupRootPath, "init.scope", cgroupV2CPUStat): "usage_usec 0", + filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryMaxBytes): "max", + filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryUsageBytes): "67108864", + filepath.Join(cgroupRootPath, "init.scope", cgroupV2MemoryStat): "inactive_file 0", + + filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2MemoryMaxBytes): "max", + filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2MemoryUsageBytes): "872415232", + filepath.Join(cgroupRootPath, cgroupV2Path, cgroupV2MemoryStat): "inactive_file 134217728", + } // Variant where child has memory.stat but NOT memory.current (should inherit from root) fsContainerCgroupV2KubernetesMissingMemoryCurrent = map[string]string{ procOneCgroup: "0::/", From c17444741458a9b3536832b7e885a34bdf752e55 Mon Sep 17 00:00:00 2001 From: default Date: Fri, 23 Oct 2026 19:21:15 +0000 Subject: [PATCH 2/2] refactor: address review feedback on cgroupv2 memory usage --- cgroupv2.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cgroupv2.go b/cgroupv2.go index f871c03..89d6e60 100644 --- a/cgroupv2.go +++ b/cgroupv2.go @@ -157,6 +157,8 @@ func (s cgroupV2Statter) cpuTotal() (total float64, err error) { } // memoryLimit is the effective memory.max and the cgroup that sets it. +// owner is kept so usage can be read from the same cgroup the limit is +// enforced on. type memoryLimit struct { owner *cgroupV2Statter bytes float64 @@ -184,7 +186,6 @@ func (s *cgroupV2Statter) memoryLimit() (*memoryLimit, error) { return &memoryLimit{owner: s, bytes: float64(maxUsageBytes)}, nil } -// memoryUsedBytes returns memory.current minus inactive_file. func (s cgroupV2Statter) memoryUsedBytes() (int64, error) { currUsageBytes, err := s.memoryCurrentBytes() if err != nil { @@ -252,17 +253,18 @@ func (s cgroupV2Statter) memory(p Prefix) (*Result, error) { // docker-in-docker) would count against the limit but not the usage. if limit != nil { r.Total = ptr.To(limit.bytes) + if limit.owner.path != s.path { - used, err := limit.owner.memoryUsedBytes() - if err == nil { + if used, err := limit.owner.memoryUsedBytes(); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, xerrors.Errorf("read limit owner memory: %w", err) + } + // The limit owner does not expose usage files, so fall + // back to the process's own cgroup below. + } else { r.Used = float64(used) return r, nil } - if !errors.Is(err, fs.ErrNotExist) { - return nil, xerrors.Errorf("read limit owner memory: %w", err) - } - // The limit owner does not expose usage files, so fall back - // to the process's own cgroup. } }