fix: report the cgroupv2 limit with the least room left for container cpu and memory

README.md

@@ -105,6 +105,45 @@ Container CPU: 0.5/2.5 cores (20%)
 Container Memory: 256/1024 MiB (25%)
 ```
 
+## How container limits are reported
+
+On cgroup v2, a container's cgroup can have child cgroups. For example, in a
+docker-in-docker workspace the process runs in `/init.scope` and nested
+containers run in `/docker/<id>`. Limits can be set at any level, and the
+kernel enforces every limit from the process's cgroup up to the container root.
+
+`ContainerCPU` and `ContainerMemory` handle this as follows:
+
+1. Check each cgroup from the process's cgroup up to the container root for a
+   limit (`cpu.max` for CPU, `memory.max` for memory).
+1. For each limit found, read usage from the same cgroup that sets it. This
+   usage includes every cgroup below it, so processes that share the limit
+   are counted.
+   - CPU: the change in `usage_usec` from `cpu.stat` over the sample interval.
+   - Memory: `memory.current` minus `inactive_file` from `memory.stat`.
+1. Report the limit with the least room left (limit minus usage), since that
+   is the one that will be hit first. If two limits have the same room left,
+   the one nearest to the process is reported.
+1. If no cgroup sets a limit, report usage of the process's own cgroup with
+   no total.
+
+If a cgroup sets a limit but does not have the usage files, usage of the
+process's own cgroup is used for that limit.
+
+Only one limit is reported. The result answers "how close is this process to
+being throttled or killed," not "what are all the limits."
+
+### Limits outside the container
+
+clistat can only see cgroups inside the container's cgroup namespace. Limits
+set above the container root are not visible. For example, a Kubernetes
+pod-level limit is not visible from inside a container that has its own
+cgroup namespace. If such a limit is tighter than the container's own limit,
+clistat reports the container's limit.
+
+On cgroup v1, clistat reads the limit and usage from the container's own
+cgroup only.
+
 ## Testing
 
 The library includes a comprehensive test suite that can be run with:

cgroup.go

@@ -30,11 +30,20 @@ const (
 var errNotContainerized = errors.New("not containerized")
 
 type cgroupStatter interface {
-	cpuUsed() (used float64, err error)
-	cpuTotal() (total float64, err error)
+	cpuCounters() ([]cpuCounter, error)
 	memory(p Prefix) (*Result, error)
 }
 
+// cpuCounter is a cumulative CPU usage counter paired with the quota that
+// applies to it.
+type cpuCounter struct {
+	// total is the quota in cores. A value <= 0 means no quota.
+	total float64
+	// used returns the cumulative usage counter, in the same unit as total
+	// once the difference between two samples is taken.
+	used func() (float64, error)
+}
+
 func (s *Statter) getCgroupStatter() (cgroupStatter, error) {
 	isContainerized, err := s.IsContainerized()
 	if err != nil {
@@ -60,21 +69,27 @@ func (s *Statter) getCgroupStatter() (cgroupStatter, error) {
 // This is calculated as difference of two samples of the
 // CPU usage of the container cgroup.
 // The total is read from the relevant path in /sys/fs/cgroup.
-// If there is no limit set, the total is assumed to be the
-// number of host cores multiplied by the CFS period.
+// On cgroup v2, every quota from the process's cgroup up to the container
+// root applies, so the one with the least room left is reported, with
+// usage read from the cgroup that sets it.
+// If there is no limit set, Total is nil.
 // If the system is not containerized, this always returns nil.
 func (s *Statter) ContainerCPU() (*Result, error) {
 	if s.cgroupStatter == nil {
 		return nil, nil //nolint: nilnil
 	}
 
-	total, err := s.cgroupStatter.cpuTotal()
+	counters, err := s.cgroupStatter.cpuCounters()
 	if err != nil {
-		return nil, xerrors.Errorf("get total cpu: %w", err)
+		return nil, xerrors.Errorf("get cpu counters: %w", err)
 	}
-	used1, err := s.cgroupStatter.cpuUsed()
-	if err != nil {
-		return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
+
+	first := make([]float64, len(counters))
+	for i, c := range counters {
+		first[i], err = c.used()
+		if err != nil {
+			return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
+		}
 	}
 
 	// The measurements in /sys/fs/cgroup are counters.
@@ -83,33 +98,54 @@ func (s *Statter) ContainerCPU() (*Result, error) {
 	// We can't do anything about that.
 	s.wait(s.sampleInterval)
 
-	used2, err := s.cgroupStatter.cpuUsed()
-	if err != nil {
-		return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
-	}
+	var (
+		best         *Result
+		bestHeadroom float64
+	)
+	for i, c := range counters {
+		used2, err := c.used()
+		if err != nil {
+			return nil, xerrors.Errorf("get cgroup CPU usage: %w", err)
+		}
 
-	if used2 < used1 {
-		// Someone reset the counter. Best we can do is count from zero.
-		used1 = 0
-	}
+		used1 := first[i]
+		if used2 < used1 {
+			// Someone reset the counter. Best we can do is count from zero.
+			used1 = 0
+		}
 
-	r := &Result{
-		Unit:   "cores",
-		Used:   used2 - used1,
-		Prefix: PrefixDefault,
-	}
+		r := &Result{
+			Unit:   "cores",
+			Used:   used2 - used1,
+			Prefix: PrefixDefault,
+		}
+		if c.total <= 0 {
+			if best == nil {
+				best = r
+			}
+			continue
+		}
 
-	if total > 0 {
-		r.Total = ptr.To(total)
+		// Ties go to the earlier counter, which is nearer to the process.
+		r.Total = ptr.To(c.total)
+		headroom := c.total - r.Used
+		if best == nil || best.Total == nil || headroom < bestHeadroom {
+			best = r
+			bestHeadroom = headroom
+		}
 	}
-	return r, nil
+
+	return best, nil
 }
 
 func (s *Statter) isCgroupV2() bool {
 	return s.cgroupV2Detector(s.fs)
 }
 
 // ContainerMemory returns the memory usage of the container cgroup.
+// On cgroup v2, every memory.max from the process's cgroup up to the
+// container root applies, so the one with the least room left is
+// reported, with usage read from the cgroup that sets it.
 // If the system is not containerized, this always returns nil.
 func (s *Statter) ContainerMemory(p Prefix) (*Result, error) {
 	if s.cgroupStatter == nil {

cgroupv1.go

@@ -39,6 +39,15 @@ type cgroupV1Statter struct {
 	fs afero.Fs
 }
 
+func (s cgroupV1Statter) cpuCounters() ([]cpuCounter, error) {
+	total, err := s.cpuTotal()
+	if err != nil {
+		return nil, xerrors.Errorf("get total cpu: %w", err)
+	}
+
+	return []cpuCounter{{total: total, used: s.cpuUsed}}, nil
+}
+
 func (s cgroupV1Statter) cpuUsed() (used float64, err error) {
 	usageNs, err := readInt64(s.fs, cgroupV1CPUAcctUsage)
 	if err != nil {