fix(profiler): make the JVM and Python profilers actually produce a profile

.github/workflows/code-verify.yml

@@ -25,3 +25,39 @@ jobs:
 
       - name: Test
         run: GOARCH=amd64 GOOS=linux go test -p 1 ./... -coverprofile=coverage.out
+
+  # The Dockerfiles were only ever built by release.yml, which runs *after*
+  # merge — so a Dockerfile that silently produces a broken image (an empty
+  # async-profiler/build shipped for two months) reached every environment
+  # before anyone noticed. Build them here too, amd64 only to keep it quick.
+  # Keep this matrix in sync with the one in release.yml.
+  images:
+    name: image (${{ matrix.image }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: application-profiler-jvm
+            dockerfile: docker/jvm/alpine/Dockerfile
+          - image: application-profiler-bpf
+            dockerfile: docker/bpf/Dockerfile
+          - image: application-profiler-perf
+            dockerfile: docker/perf/Dockerfile
+          - image: application-profiler-python
+            dockerfile: docker/python/Dockerfile
+          - image: application-profiler-ruby
+            dockerfile: docker/ruby/Dockerfile
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build ${{ matrix.image }}
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: false

docker/jvm/alpine/Dockerfile

@@ -8,15 +8,42 @@ RUN go get -d -v ./...
 WORKDIR /go/src/github.com/nudgebee/application-profiler/cmd/agent
 RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /go/bin/agent
 
-FROM amazoncorretto:26-alpine-jdk AS async-profiler-builder
+# JDK 17, not a current one: async-profiler 2.9's Makefile compiles its Java
+# helper with a hardcoded `-source 7 -target 7`, which javac rejects from JDK 20
+# onwards ("Source option 7 is no longer supported"). On JDK 26 that killed the
+# very first make target, leaving build/ empty.
+FROM amazoncorretto:17-alpine-jdk AS async-profiler-builder
 ARG TARGETPLATFORM
 ARG ASYNCP_VERSION=2.9
 RUN apk add --no-cache curl g++ git linux-headers make musl-dev tar wget
 WORKDIR /src
 RUN git clone --branch v${ASYNCP_VERSION} --depth 1 https://github.com/async-profiler/async-profiler
 WORKDIR /src/async-profiler
-RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all native; \
-    tree -d /src;
+# `all` only — `native` is upstream's release-packaging target and expects the
+# published per-arch tarballs to be sitting in the source tree, so it always
+# failed here. The trailing `;` that used to swallow that failure also swallowed
+# a broken `all`, which is how an empty build/ got published. Assert instead:
+# every artefact profiler.sh needs at runtime must exist before we ship a layer.
+RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all \
+    && test -f build/libasyncProfiler.so \
+    && test -x build/jattach \
+    && test -x build/fdtransfer
+
+# Second build of the same source against glibc. libasyncProfiler.so is
+# dlopen'd by the *target* JVM, not by anything in this image, so it has to
+# match the target's libc: a musl .so in a temurin/corretto/distroless JVM
+# fails the attach with "libc.musl-x86_64.so.1: cannot open shared object
+# file". jattach and fdtransfer stay musl — those run here, in this alpine
+# image. Same JDK 17 pin, same reason as above.
+FROM amazoncorretto:17 AS async-profiler-builder-glibc
+ARG TARGETPLATFORM
+ARG ASYNCP_VERSION=2.9
+RUN yum install -y gcc-c++ git make tar
+WORKDIR /src
+RUN git clone --branch v${ASYNCP_VERSION} --depth 1 https://github.com/async-profiler/async-profiler
+WORKDIR /src/async-profiler
+RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all \
+    && test -f build/libasyncProfiler.so
 
 FROM eclipse-temurin:$JAVA_VERSION_TAG-alpine AS tools
 ARG TARGETPLATFORM
@@ -38,7 +65,19 @@ RUN mkdir -p /app/async-profiler/build  \
 COPY --from=agentbuild /go/bin/agent /app
 COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/jvm/profiler.sh /app/async-profiler
 COPY --from=async-profiler-builder /src/async-profiler/build /app/async-profiler/build
+# Ship both libc builds of the injected library. The default name — the one
+# profiler.sh hands to the JVM — is the glibc build, because that is what most
+# JVM images are; the agent swaps the musl one in when the target's rootfs is
+# musl (internal/agent/profiler/jvm/async_profiler.go).
+RUN mv /app/async-profiler/build/libasyncProfiler.so /app/async-profiler/build/libasyncProfiler-musl.so
+COPY --from=async-profiler-builder-glibc /src/async-profiler/build/libasyncProfiler.so /app/async-profiler/build/libasyncProfiler.so
 COPY --from=tools /jcmd /opt/jdk
+# COPY of a directory is happy to copy nothing, so re-assert in the final stage:
+# a shipped image without these fails at profile time, not at build time.
+RUN test -f /app/async-profiler/build/libasyncProfiler.so \
+    && test -f /app/async-profiler/build/libasyncProfiler-musl.so \
+    && test -x /app/async-profiler/build/jattach \
+    && test -x /app/async-profiler/build/fdtransfer
 RUN chmod +x /app/async-profiler/profiler.sh  \
     && mkdir -p /app/jfr/settings
 COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/jvm/jfr-profile.jfc /app/jfr/settings

docker/python/Dockerfile

@@ -6,8 +6,13 @@ RUN go get -d -v ./...
 WORKDIR /go/src/github.com/nudgebee/application-profiler/cmd/agent
 RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /go/bin/agent
 
-FROM python:3.13.6-slim-bullseye AS pyspybuild
-RUN pip3 install py-spy==0.4.0  \
+# bookworm, not bullseye: Debian 11 is past EOL and its security suite has
+# moved to archive, so `apt-get install git` here 404s — this stage no longer
+# builds at all on bullseye.
+FROM python:3.13-slim-bookworm AS pyspybuild
+# 0.4.2 is the first release that understands CPython 3.14. On 0.4.0 a 3.14
+# target fails with the misleading "No python processes found in process <pid>".
+RUN pip3 install py-spy==0.4.2  \
     && apt-get update && apt-get install -y git \
     && git clone https://github.com/brendangregg/FlameGraph
 
@@ -18,7 +23,11 @@ COPY --from=pyspybuild /FlameGraph /app/FlameGraph
 COPY --from=agentbuild /go/bin/agent /app/agent
 COPY --from=pyspybuild /usr/local/bin/py-spy /app/py-spy
 COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/get-ps-command.sh /app/get-ps-command.sh
-RUN pip3 install austin-dist==3.7.0
+# austin 4 is the first release that can read CPython 3.14; on 3.7.0 a 3.14
+# target is reported as "not a Python process". It only writes the binary MOJO
+# format, so austin-python comes along for mojo2austin, which converts it back
+# to the text format the raw artefact and flamegraph.pl are built around.
+RUN pip3 install austin-dist==4.0.0 austin-python==1.7.1
 RUN chmod +x /app/get-ps-command.sh
 
 CMD [ "/app/agent" ]

internal/agent/profiler/jvm/async_profiler.go

@@ -26,8 +26,13 @@ import (
 )
 
 const (
-	asyncProfilerDir              = "/tmp/async-profiler"
-	profilerSh                    = asyncProfilerDir + "/profiler.sh"
+	asyncProfilerDir = "/tmp/async-profiler"
+	profilerSh       = asyncProfilerDir + "/profiler.sh"
+	// profilerLib is the path profiler.sh hands to the target JVM to dlopen.
+	// The image ships the glibc build under this name and the musl build
+	// beside it; selectProfilerLibrary swaps them when the target is musl.
+	profilerLib                   = asyncProfilerDir + "/build/libasyncProfiler.so"
+	profilerLibMusl               = asyncProfilerDir + "/build/libasyncProfiler-musl.so"
 	asyncProfilerDelayBetweenJobs = 2 * time.Second
 )
 
@@ -57,6 +62,7 @@ type AsyncProfilerManager interface {
 	removeTmpDir() error
 	linkTmpDirToTargetTmpDir(string) error
 	copyProfilerToTmpDir() error
+	selectProfilerLibrary(string) error
 	invoke(*job.ProfilingJob, string) (error, time.Duration)
 	cleanUp(*job.ProfilingJob, string)
 }
@@ -108,7 +114,28 @@ func (j *AsyncProfiler) SetUp(job *job.ProfilingJob) error {
 		j.targetPIDs = pids
 	}
 
-	return j.copyProfilerToTmpDir()
+	if err := j.copyProfilerToTmpDir(); err != nil {
+		return err
+	}
+
+	return j.selectProfilerLibrary(targetFs)
+}
+
+// targetUsesMusl reports whether the target container's root filesystem is
+// musl-based (alpine and friends), by looking for the musl dynamic loader.
+func targetUsesMusl(targetFs string) bool {
+	// An empty root would make the patterns below relative to our own
+	// working directory — and this image is alpine, so every target would
+	// look musl-based. Treat "unknown" as the shipped default (glibc).
+	if targetFs == "" {
+		return false
+	}
+	for _, dir := range []string{"lib", "usr/lib"} {
+		if matches, _ := filepath.Glob(filepath.Join(targetFs, dir, "ld-musl-*.so.1")); len(matches) > 0 {
+			return true
+		}
+	}
+	return false
 }
 
 func (j *asyncProfilerManager) removeTmpDir() error {
@@ -124,6 +151,21 @@ func (j *asyncProfilerManager) copyProfilerToTmpDir() error {
 	return cmd.Run()
 }
 
+// selectProfilerLibrary points libasyncProfiler.so at the build matching the
+// target's libc. The library is dlopen'd by the target JVM rather than by us,
+// so a mismatch fails the attach with "libc.musl-x86_64.so.1: cannot open
+// shared object file" — and the profile comes back empty with no clue why.
+// The image ships the glibc build under the default name, so only musl
+// targets need the swap.
+func (j *asyncProfilerManager) selectProfilerLibrary(targetFs string) error {
+	if !targetUsesMusl(targetFs) {
+		return nil
+	}
+	log.DebugLogLn("The target is musl-based; using the musl build of libasyncProfiler.so")
+	cmd := j.commander.Command("cp", "-f", profilerLibMusl, profilerLib)
+	return cmd.Run()
+}
+
 func (j *AsyncProfiler) Invoke(job *job.ProfilingJob) (error, time.Duration) {
 	start := time.Now()
 

internal/agent/profiler/jvm/async_profiler_fake.go

@@ -115,6 +115,19 @@ func (f *fakeAsyncProfilerManager) copyProfilerToTmpDir() error {
 	return err
 }
 
+func (f *fakeAsyncProfilerManager) selectProfilerLibrary(s string) error {
+	var err error
+	f.fakeMethods["selectProfilerLibrary"].invokes++
+	if f.fakeMethods["selectProfilerLibrary"].fakeReturnValues != nil && len(f.fakeMethods["selectProfilerLibrary"].fakeReturnValues) > 0 {
+		f.fakeMethods["selectProfilerLibrary"].indexExecution++
+		arg0 := f.fakeMethods["selectProfilerLibrary"].fakeReturnValues[f.fakeMethods["selectProfilerLibrary"].indexExecution-1].([]interface{})[0]
+		if arg0 != nil {
+			err = arg0.(error)
+		}
+	}
+	return err
+}
+
 func (f *fakeAsyncProfilerManager) cleanUp(profilingJob *job.ProfilingJob, s string) {
 	f.fakeMethods["cleanUp"].invokes++
 	if f.fakeMethods["cleanUp"].fakeReturnValues != nil && len(f.fakeMethods["cleanUp"].fakeReturnValues) > 0 {

internal/agent/profiler/jvm/async_profiler_test.go

@@ -42,6 +42,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager.On("removeTmpDir").Return(nil)
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
+				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -73,6 +74,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager.On("removeTmpDir").Return(nil)
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
+				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -105,6 +107,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) {
 				asyncProfilerManager.On("removeTmpDir").Return(nil)
 				asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil)
 				asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil)
+				asyncProfilerManager.On("selectProfilerLibrary").Return(nil)
 
 				return fields{
 						AsyncProfiler: &AsyncProfiler{
@@ -423,6 +426,75 @@ func Test_asyncProfilerManager_copyProfilerToTmpDir(t *testing.T) {
 	assert.Nil(t, a.copyProfilerToTmpDir())
 }
 
+// Test_targetUsesMusl — libasyncProfiler.so is dlopen'd by the target JVM, so
+// getting this wrong fails the attach with "libc.musl-x86_64.so.1: cannot open
+// shared object file" and the profile comes back empty.
+func Test_targetUsesMusl(t *testing.T) {
+	tests := []struct {
+		name   string
+		create string // file to create under the fake target rootfs
+		want   bool
+	}{
+		{name: "alpine target", create: "lib/ld-musl-x86_64.so.1", want: true},
+		{name: "alpine target on arm", create: "lib/ld-musl-aarch64.so.1", want: true},
+		{name: "musl loader under usr/lib", create: "usr/lib/ld-musl-x86_64.so.1", want: true},
+		{name: "glibc target", create: "lib/ld-linux-x86-64.so.2", want: false},
+		{name: "empty rootfs", create: "", want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			root := t.TempDir()
+			if tt.create != "" {
+				p := filepath.Join(root, tt.create)
+				require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
+				_, err := os.Create(p)
+				require.NoError(t, err)
+			}
+			assert.Equal(t, tt.want, targetUsesMusl(root))
+		})
+	}
+
+	// An unresolved root must not make the glob relative to our own working
+	// directory: this image is alpine, so that would report every target as
+	// musl-based.
+	t.Run("empty target root", func(t *testing.T) {
+		assert.False(t, targetUsesMusl(""))
+	})
+}
+
+// Test_asyncProfilerManager_selectProfilerLibrary — a glibc target must be
+// left alone (the shipped default is already glibc); only a musl target
+// triggers the copy that swaps the library.
+func Test_asyncProfilerManager_selectProfilerLibrary(t *testing.T) {
+	t.Run("glibc target does not copy", func(t *testing.T) {
+		commander := executil.NewFakeCommander()
+		commander.On("Command").Return(exec.Command("false"))
+		a := NewAsyncProfiler(commander, publish.NewFakePublisher())
+
+		root := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755))
+		_, err := os.Create(filepath.Join(root, "lib", "ld-linux-x86-64.so.2"))
+		require.NoError(t, err)
+
+		// `false` would error if it ran — a nil result proves it did not.
+		assert.Nil(t, a.selectProfilerLibrary(root))
+	})
+
+	t.Run("musl target copies the musl build over the default", func(t *testing.T) {
+		commander := executil.NewFakeCommander()
+		commander.On("Command").Return(exec.Command("true"))
+		a := NewAsyncProfiler(commander, publish.NewFakePublisher())
+
+		root := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755))
+		_, err := os.Create(filepath.Join(root, "lib", "ld-musl-x86_64.so.1"))
+		require.NoError(t, err)
+
+		assert.Nil(t, a.selectProfilerLibrary(root))
+		assert.Equal(t, 1, commander.On("Command").InvokedTimes())
+	})
+}
+
 func Test_asyncProfilerManager_invoke(t *testing.T) {
 	type fields struct {
 		AsyncProfiler *AsyncProfiler