From 2401e8942ad526262fddd70d765485d83954edcc Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Fri, 18 Sep 2026 13:50:12 +0530 Subject: [PATCH 1/3] fix(jvm): ship an async-profiler that builds, and one that loads Two independent reasons Java flamegraphs could never work. The build: async-profiler 2.9's Makefile compiles its Java helper with a hardcoded `-source 7`, which javac rejects from JDK 20 onwards, so `make` died on its first target under the JDK 26 builder. A trailing `;` swallowed that failure and `COPY` happily copied the resulting empty build/ into the image, so every published tag since 2026-05-14 shipped no libasyncProfiler.so, no jattach and no fdtransfer - profiles failed at runtime with "/tmp/async-profiler/build/fdtransfer: not found". Pin the builder to JDK 17, drop the `native` target (upstream's release-packaging step, which expects prebuilt tarballs in the tree and always failed here), and assert the three artefacts exist before shipping a layer. The library: libasyncProfiler.so is dlopen'd by the *target* JVM, not by anything in our image, so a musl build fails on every glibc JVM with "libc.musl-x86_64.so.1: cannot open shared object file". Build the library twice and ship both; the agent now picks the one matching the target's libc. Also: py-spy 0.4.0 cannot read CPython 3.14 and reports it as "No python processes found in process " - bump to 0.4.2. And that stage no longer builds at all, because Debian 11 is past EOL and its security suite has moved to archive; move it to bookworm. Finally, build the images in code-verify. They were only ever built by release.yml, after merge, which is how a broken image shipped for two months. --- .github/workflows/code-verify.yml | 36 ++++++++++ docker/jvm/alpine/Dockerfile | 45 ++++++++++++- docker/python/Dockerfile | 9 ++- internal/agent/profiler/jvm/async_profiler.go | 42 +++++++++++- .../agent/profiler/jvm/async_profiler_fake.go | 13 ++++ .../agent/profiler/jvm/async_profiler_test.go | 65 +++++++++++++++++++ 6 files changed, 202 insertions(+), 8 deletions(-) diff --git a/.github/workflows/code-verify.yml b/.github/workflows/code-verify.yml index 3c4f05f..4811a0f 100644 --- a/.github/workflows/code-verify.yml +++ b/.github/workflows/code-verify.yml @@ -25,3 +25,39 @@ jobs: - name: Test run: GOARCH=amd64 GOOS=linux go test -p 1 ./... -coverprofile=coverage.out + + # The Dockerfiles were only ever built by release.yml, which runs *after* + # merge — so a Dockerfile that silently produces a broken image (an empty + # async-profiler/build shipped for two months) reached every environment + # before anyone noticed. Build them here too, amd64 only to keep it quick. + # Keep this matrix in sync with the one in release.yml. + images: + name: image (${{ matrix.image }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - image: application-profiler-jvm + dockerfile: docker/jvm/alpine/Dockerfile + - image: application-profiler-bpf + dockerfile: docker/bpf/Dockerfile + - image: application-profiler-perf + dockerfile: docker/perf/Dockerfile + - image: application-profiler-python + dockerfile: docker/python/Dockerfile + - image: application-profiler-ruby + dockerfile: docker/ruby/Dockerfile + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build ${{ matrix.image }} + uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: false diff --git a/docker/jvm/alpine/Dockerfile b/docker/jvm/alpine/Dockerfile index 35c9c1a..f83c2f2 100644 --- a/docker/jvm/alpine/Dockerfile +++ b/docker/jvm/alpine/Dockerfile @@ -8,15 +8,42 @@ RUN go get -d -v ./... WORKDIR /go/src/github.com/nudgebee/application-profiler/cmd/agent RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /go/bin/agent -FROM amazoncorretto:26-alpine-jdk AS async-profiler-builder +# JDK 17, not a current one: async-profiler 2.9's Makefile compiles its Java +# helper with a hardcoded `-source 7 -target 7`, which javac rejects from JDK 20 +# onwards ("Source option 7 is no longer supported"). On JDK 26 that killed the +# very first make target, leaving build/ empty. +FROM amazoncorretto:17-alpine-jdk AS async-profiler-builder ARG TARGETPLATFORM ARG ASYNCP_VERSION=2.9 RUN apk add --no-cache curl g++ git linux-headers make musl-dev tar wget WORKDIR /src RUN git clone --branch v${ASYNCP_VERSION} --depth 1 https://github.com/async-profiler/async-profiler WORKDIR /src/async-profiler -RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all native; \ - tree -d /src; +# `all` only — `native` is upstream's release-packaging target and expects the +# published per-arch tarballs to be sitting in the source tree, so it always +# failed here. The trailing `;` that used to swallow that failure also swallowed +# a broken `all`, which is how an empty build/ got published. Assert instead: +# every artefact profiler.sh needs at runtime must exist before we ship a layer. +RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all \ + && test -f build/libasyncProfiler.so \ + && test -x build/jattach \ + && test -x build/fdtransfer + +# Second build of the same source against glibc. libasyncProfiler.so is +# dlopen'd by the *target* JVM, not by anything in this image, so it has to +# match the target's libc: a musl .so in a temurin/corretto/distroless JVM +# fails the attach with "libc.musl-x86_64.so.1: cannot open shared object +# file". jattach and fdtransfer stay musl — those run here, in this alpine +# image. Same JDK 17 pin, same reason as above. +FROM amazoncorretto:17 AS async-profiler-builder-glibc +ARG TARGETPLATFORM +ARG ASYNCP_VERSION=2.9 +RUN yum install -y gcc-c++ git make tar +WORKDIR /src +RUN git clone --branch v${ASYNCP_VERSION} --depth 1 https://github.com/async-profiler/async-profiler +WORKDIR /src/async-profiler +RUN make CXXFLAGS='-O3 -fno-omit-frame-pointer -fvisibility=hidden -D_LARGEFILE64_SOURCE' all \ + && test -f build/libasyncProfiler.so FROM eclipse-temurin:$JAVA_VERSION_TAG-alpine AS tools ARG TARGETPLATFORM @@ -38,7 +65,19 @@ RUN mkdir -p /app/async-profiler/build \ COPY --from=agentbuild /go/bin/agent /app COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/jvm/profiler.sh /app/async-profiler COPY --from=async-profiler-builder /src/async-profiler/build /app/async-profiler/build +# Ship both libc builds of the injected library. The default name — the one +# profiler.sh hands to the JVM — is the glibc build, because that is what most +# JVM images are; the agent swaps the musl one in when the target's rootfs is +# musl (internal/agent/profiler/jvm/async_profiler.go). +RUN mv /app/async-profiler/build/libasyncProfiler.so /app/async-profiler/build/libasyncProfiler-musl.so +COPY --from=async-profiler-builder-glibc /src/async-profiler/build/libasyncProfiler.so /app/async-profiler/build/libasyncProfiler.so COPY --from=tools /jcmd /opt/jdk +# COPY of a directory is happy to copy nothing, so re-assert in the final stage: +# a shipped image without these fails at profile time, not at build time. +RUN test -f /app/async-profiler/build/libasyncProfiler.so \ + && test -f /app/async-profiler/build/libasyncProfiler-musl.so \ + && test -x /app/async-profiler/build/jattach \ + && test -x /app/async-profiler/build/fdtransfer RUN chmod +x /app/async-profiler/profiler.sh \ && mkdir -p /app/jfr/settings COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/jvm/jfr-profile.jfc /app/jfr/settings diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index da477b1..b5dd193 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -6,8 +6,13 @@ RUN go get -d -v ./... WORKDIR /go/src/github.com/nudgebee/application-profiler/cmd/agent RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /go/bin/agent -FROM python:3.13.6-slim-bullseye AS pyspybuild -RUN pip3 install py-spy==0.4.0 \ +# bookworm, not bullseye: Debian 11 is past EOL and its security suite has +# moved to archive, so `apt-get install git` here 404s — this stage no longer +# builds at all on bullseye. +FROM python:3.13-slim-bookworm AS pyspybuild +# 0.4.2 is the first release that understands CPython 3.14. On 0.4.0 a 3.14 +# target fails with the misleading "No python processes found in process ". +RUN pip3 install py-spy==0.4.2 \ && apt-get update && apt-get install -y git \ && git clone https://github.com/brendangregg/FlameGraph diff --git a/internal/agent/profiler/jvm/async_profiler.go b/internal/agent/profiler/jvm/async_profiler.go index 731f154..a4a37e9 100644 --- a/internal/agent/profiler/jvm/async_profiler.go +++ b/internal/agent/profiler/jvm/async_profiler.go @@ -26,8 +26,13 @@ import ( ) const ( - asyncProfilerDir = "/tmp/async-profiler" - profilerSh = asyncProfilerDir + "/profiler.sh" + asyncProfilerDir = "/tmp/async-profiler" + profilerSh = asyncProfilerDir + "/profiler.sh" + // profilerLib is the path profiler.sh hands to the target JVM to dlopen. + // The image ships the glibc build under this name and the musl build + // beside it; selectProfilerLibrary swaps them when the target is musl. + profilerLib = asyncProfilerDir + "/build/libasyncProfiler.so" + profilerLibMusl = asyncProfilerDir + "/build/libasyncProfiler-musl.so" asyncProfilerDelayBetweenJobs = 2 * time.Second ) @@ -57,6 +62,7 @@ type AsyncProfilerManager interface { removeTmpDir() error linkTmpDirToTargetTmpDir(string) error copyProfilerToTmpDir() error + selectProfilerLibrary(string) error invoke(*job.ProfilingJob, string) (error, time.Duration) cleanUp(*job.ProfilingJob, string) } @@ -108,7 +114,22 @@ func (j *AsyncProfiler) SetUp(job *job.ProfilingJob) error { j.targetPIDs = pids } - return j.copyProfilerToTmpDir() + if err := j.copyProfilerToTmpDir(); err != nil { + return err + } + + return j.selectProfilerLibrary(targetFs) +} + +// targetUsesMusl reports whether the target container's root filesystem is +// musl-based (alpine and friends), by looking for the musl dynamic loader. +func targetUsesMusl(targetFs string) bool { + for _, dir := range []string{"lib", "usr/lib"} { + if matches, _ := filepath.Glob(filepath.Join(targetFs, dir, "ld-musl-*.so.1")); len(matches) > 0 { + return true + } + } + return false } func (j *asyncProfilerManager) removeTmpDir() error { @@ -124,6 +145,21 @@ func (j *asyncProfilerManager) copyProfilerToTmpDir() error { return cmd.Run() } +// selectProfilerLibrary points libasyncProfiler.so at the build matching the +// target's libc. The library is dlopen'd by the target JVM rather than by us, +// so a mismatch fails the attach with "libc.musl-x86_64.so.1: cannot open +// shared object file" — and the profile comes back empty with no clue why. +// The image ships the glibc build under the default name, so only musl +// targets need the swap. +func (j *asyncProfilerManager) selectProfilerLibrary(targetFs string) error { + if !targetUsesMusl(targetFs) { + return nil + } + log.DebugLogLn("The target is musl-based; using the musl build of libasyncProfiler.so") + cmd := j.commander.Command("cp", "-f", profilerLibMusl, profilerLib) + return cmd.Run() +} + func (j *AsyncProfiler) Invoke(job *job.ProfilingJob) (error, time.Duration) { start := time.Now() diff --git a/internal/agent/profiler/jvm/async_profiler_fake.go b/internal/agent/profiler/jvm/async_profiler_fake.go index ccfe45e..73ef552 100644 --- a/internal/agent/profiler/jvm/async_profiler_fake.go +++ b/internal/agent/profiler/jvm/async_profiler_fake.go @@ -115,6 +115,19 @@ func (f *fakeAsyncProfilerManager) copyProfilerToTmpDir() error { return err } +func (f *fakeAsyncProfilerManager) selectProfilerLibrary(s string) error { + var err error + f.fakeMethods["selectProfilerLibrary"].invokes++ + if f.fakeMethods["selectProfilerLibrary"].fakeReturnValues != nil && len(f.fakeMethods["selectProfilerLibrary"].fakeReturnValues) > 0 { + f.fakeMethods["selectProfilerLibrary"].indexExecution++ + arg0 := f.fakeMethods["selectProfilerLibrary"].fakeReturnValues[f.fakeMethods["selectProfilerLibrary"].indexExecution-1].([]interface{})[0] + if arg0 != nil { + err = arg0.(error) + } + } + return err +} + func (f *fakeAsyncProfilerManager) cleanUp(profilingJob *job.ProfilingJob, s string) { f.fakeMethods["cleanUp"].invokes++ if f.fakeMethods["cleanUp"].fakeReturnValues != nil && len(f.fakeMethods["cleanUp"].fakeReturnValues) > 0 { diff --git a/internal/agent/profiler/jvm/async_profiler_test.go b/internal/agent/profiler/jvm/async_profiler_test.go index d298400..aa38548 100644 --- a/internal/agent/profiler/jvm/async_profiler_test.go +++ b/internal/agent/profiler/jvm/async_profiler_test.go @@ -42,6 +42,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) { asyncProfilerManager.On("removeTmpDir").Return(nil) asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil) asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil) + asyncProfilerManager.On("selectProfilerLibrary").Return(nil) return fields{ AsyncProfiler: &AsyncProfiler{ @@ -73,6 +74,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) { asyncProfilerManager.On("removeTmpDir").Return(nil) asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil) asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil) + asyncProfilerManager.On("selectProfilerLibrary").Return(nil) return fields{ AsyncProfiler: &AsyncProfiler{ @@ -105,6 +107,7 @@ func TestAsyncProfiler_SetUp(t *testing.T) { asyncProfilerManager.On("removeTmpDir").Return(nil) asyncProfilerManager.On("linkTmpDirToTargetTmpDir").Return(nil) asyncProfilerManager.On("copyProfilerToTmpDir").Return(nil) + asyncProfilerManager.On("selectProfilerLibrary").Return(nil) return fields{ AsyncProfiler: &AsyncProfiler{ @@ -423,6 +426,68 @@ func Test_asyncProfilerManager_copyProfilerToTmpDir(t *testing.T) { assert.Nil(t, a.copyProfilerToTmpDir()) } +// Test_targetUsesMusl — libasyncProfiler.so is dlopen'd by the target JVM, so +// getting this wrong fails the attach with "libc.musl-x86_64.so.1: cannot open +// shared object file" and the profile comes back empty. +func Test_targetUsesMusl(t *testing.T) { + tests := []struct { + name string + create string // file to create under the fake target rootfs + want bool + }{ + {name: "alpine target", create: "lib/ld-musl-x86_64.so.1", want: true}, + {name: "alpine target on arm", create: "lib/ld-musl-aarch64.so.1", want: true}, + {name: "musl loader under usr/lib", create: "usr/lib/ld-musl-x86_64.so.1", want: true}, + {name: "glibc target", create: "lib/ld-linux-x86-64.so.2", want: false}, + {name: "empty rootfs", create: "", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + if tt.create != "" { + p := filepath.Join(root, tt.create) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + _, err := os.Create(p) + require.NoError(t, err) + } + assert.Equal(t, tt.want, targetUsesMusl(root)) + }) + } +} + +// Test_asyncProfilerManager_selectProfilerLibrary — a glibc target must be +// left alone (the shipped default is already glibc); only a musl target +// triggers the copy that swaps the library. +func Test_asyncProfilerManager_selectProfilerLibrary(t *testing.T) { + t.Run("glibc target does not copy", func(t *testing.T) { + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("false")) + a := NewAsyncProfiler(commander, publish.NewFakePublisher()) + + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) + _, err := os.Create(filepath.Join(root, "lib", "ld-linux-x86-64.so.2")) + require.NoError(t, err) + + // `false` would error if it ran — a nil result proves it did not. + assert.Nil(t, a.selectProfilerLibrary(root)) + }) + + t.Run("musl target copies the musl build over the default", func(t *testing.T) { + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("true")) + a := NewAsyncProfiler(commander, publish.NewFakePublisher()) + + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "lib"), 0o755)) + _, err := os.Create(filepath.Join(root, "lib", "ld-musl-x86_64.so.1")) + require.NoError(t, err) + + assert.Nil(t, a.selectProfilerLibrary(root)) + assert.Equal(t, 1, commander.On("Command").InvokedTimes()) + }) +} + func Test_asyncProfilerManager_invoke(t *testing.T) { type fields struct { AsyncProfiler *AsyncProfiler From 3022a5ebbaeedbaf180b518a9309b9a6af5eddf1 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Fri, 18 Sep 2026 14:00:04 +0530 Subject: [PATCH 2/3] fix(jvm): treat an unresolved target root as glibc, not musl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty targetFs made the loader glob relative to our own working directory — and this image is alpine, so every target would have been reported as musl-based. --- internal/agent/profiler/jvm/async_profiler.go | 6 ++++++ internal/agent/profiler/jvm/async_profiler_test.go | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/internal/agent/profiler/jvm/async_profiler.go b/internal/agent/profiler/jvm/async_profiler.go index a4a37e9..afd4daa 100644 --- a/internal/agent/profiler/jvm/async_profiler.go +++ b/internal/agent/profiler/jvm/async_profiler.go @@ -124,6 +124,12 @@ func (j *AsyncProfiler) SetUp(job *job.ProfilingJob) error { // targetUsesMusl reports whether the target container's root filesystem is // musl-based (alpine and friends), by looking for the musl dynamic loader. func targetUsesMusl(targetFs string) bool { + // An empty root would make the patterns below relative to our own + // working directory — and this image is alpine, so every target would + // look musl-based. Treat "unknown" as the shipped default (glibc). + if targetFs == "" { + return false + } for _, dir := range []string{"lib", "usr/lib"} { if matches, _ := filepath.Glob(filepath.Join(targetFs, dir, "ld-musl-*.so.1")); len(matches) > 0 { return true diff --git a/internal/agent/profiler/jvm/async_profiler_test.go b/internal/agent/profiler/jvm/async_profiler_test.go index aa38548..33cc3cc 100644 --- a/internal/agent/profiler/jvm/async_profiler_test.go +++ b/internal/agent/profiler/jvm/async_profiler_test.go @@ -453,6 +453,13 @@ func Test_targetUsesMusl(t *testing.T) { assert.Equal(t, tt.want, targetUsesMusl(root)) }) } + + // An unresolved root must not make the glob relative to our own working + // directory: this image is alpine, so that would report every target as + // musl-based. + t.Run("empty target root", func(t *testing.T) { + assert.False(t, targetUsesMusl("")) + }) } // Test_asyncProfilerManager_selectProfilerLibrary — a glibc target must be From 91a1022aca2216f2e70d2e46cefc3a644c417235 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Fri, 18 Sep 2026 15:15:33 +0530 Subject: [PATCH 3/3] fix(python): return the samples austin collected, and read 3.14 Two defects in the austin path, both verified on the dev cluster. Python memory profiles came back as a zero-byte artefact reported as a success. austin writes its samples to the -o file, but the non-flamegraph branch then overwrote that file with the command's stdout - which is empty, because austin prints nothing on stdout when -o is given. Stop overwriting it, and fail loudly when austin genuinely collected nothing rather than publishing an empty file. austin 3.7.0 cannot read CPython 3.14; it reports the target as "not a Python process", the same class of gap py-spy 0.4.0 had. Bump to austin 4, which reads 3.14 but only writes the binary MOJO format, so convert it back with mojo2austin to the text format the raw artefact and flamegraph.pl are built around. austin 4 also ends every exposure-limited run by returning -SIGINT (254), which is not a failure. --- docker/python/Dockerfile | 6 +- internal/agent/profiler/python_austin.go | 64 ++++++++++++++- .../profiler/python_austin_output_test.go | 81 +++++++++++++++++++ 3 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 internal/agent/profiler/python_austin_output_test.go diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile index b5dd193..fbf27d9 100644 --- a/docker/python/Dockerfile +++ b/docker/python/Dockerfile @@ -23,7 +23,11 @@ COPY --from=pyspybuild /FlameGraph /app/FlameGraph COPY --from=agentbuild /go/bin/agent /app/agent COPY --from=pyspybuild /usr/local/bin/py-spy /app/py-spy COPY --from=agentbuild /go/src/github.com/nudgebee/application-profiler/contrib/get-ps-command.sh /app/get-ps-command.sh -RUN pip3 install austin-dist==3.7.0 +# austin 4 is the first release that can read CPython 3.14; on 3.7.0 a 3.14 +# target is reported as "not a Python process". It only writes the binary MOJO +# format, so austin-python comes along for mojo2austin, which converts it back +# to the text format the raw artefact and flamegraph.pl are built around. +RUN pip3 install austin-dist==4.0.0 austin-python==1.7.1 RUN chmod +x /app/get-ps-command.sh CMD [ "/app/agent" ] \ No newline at end of file diff --git a/internal/agent/profiler/python_austin.go b/internal/agent/profiler/python_austin.go index 0d9b51d..fbc7050 100644 --- a/internal/agent/profiler/python_austin.go +++ b/internal/agent/profiler/python_austin.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "fmt" + "io" + "os" "os/exec" "strconv" "time" @@ -25,8 +27,20 @@ import ( const ( austinLocation = "austin" + // mojo2austinLocation converts austin's binary MOJO output back to the + // line-per-sample text format. austin 4 dropped the text writer, and the + // text format is what the raw artefact and flamegraph.pl both consume. + mojo2austinLocation = "mojo2austin" + // austinInterruptedExitCode is what austin returns when sampling ends on + // a signal — `retval = -interrupt_signal` in its main(), so SIGINT (2) + // surfaces as 254. It ends every exposure-limited (-x) run, including the + // successful ones, so it can't be treated as a failure on its own. + austinInterruptedExitCode = 254 ) +// mojoMagic prefixes austin's binary output format. +var mojoMagic = []byte{'M', 'O', 'J', 3} + var austinPythonCommand = func(commander executil.Commander, job *job.ProfilingJob, pid string, fileName string) *exec.Cmd { interval := strconv.Itoa(int(job.Interval.Seconds())) args := []string{} @@ -114,11 +128,24 @@ func (p *austinPythonManager) invoke(job *job.ProfilingJob, pid string) (error, cmd.Stdout = &out cmd.Stderr = &stderr err := cmd.Run() - if err != nil { + if err != nil && !austinStoppedOnSignal(err) { log.ErrorLogLn(out.String()) return errors.Wrapf(err, "could not launch profiler: %s", stderr.String()), time.Since(start) } + // austin 4 always writes the binary MOJO format; convert it back to the + // text format the raw artefact and flamegraph.pl are built around. + if err := p.convertMojo(fileName); err != nil { + return errors.Wrap(err, "could not convert the profile to text"), time.Since(start) + } + + // austin writes its samples to the -o file, so an empty one means it + // attached but read nothing — report that instead of publishing a + // zero-byte artefact as a success. + if file.IsEmpty(fileName) { + return errors.Errorf("no samples collected (PID: %s): %s", pid, stderr.String()), time.Since(start) + } + // result file name is composed by the job info and the pid resultFileName := common.GetResultFile(common.TmpDir(), job.Tool, job.OutputType, pid, job.Iteration) if job.OutputType == api.FlameGraph { @@ -127,13 +154,44 @@ func (p *austinPythonManager) invoke(job *job.ProfilingJob, pid string) (error, log.ErrorLogLn(fmt.Sprintf("could not generate flamegraph (PID: %s): %s", pid, err.Error())) return nil, time.Since(start) } - } else { - file.Write(resultFileName, out.String()) } + // Nothing to do for the other output types: austin has already written + // the samples to resultFileName (fileName == resultFileName there). + // Copying cmd stdout over it, as this used to, truncated every non- + // flamegraph profile to zero bytes — austin prints nothing on stdout + // when -o is given. return p.publisher.Do(job.Compressor, resultFileName, job.OutputType), time.Since(start) } +// austinStoppedOnSignal reports whether austin exited the way an +// exposure-limited (-x) run always does: sampling ends on a signal and +// austin returns -SIGINT. The samples are already on disk at that point. +func austinStoppedOnSignal(err error) bool { + var exitErr *exec.ExitError + return errors.As(err, &exitErr) && exitErr.ExitCode() == austinInterruptedExitCode +} + +// convertMojo rewrites fileName in place when it holds austin's binary MOJO +// output. A no-op for the text format, so it is safe across austin versions. +func (p *austinPythonManager) convertMojo(fileName string) error { + f, err := os.Open(fileName) //nolint:gosec // path built by us from TmpDir + if err != nil { + return err + } + header := make([]byte, len(mojoMagic)) + n, _ := io.ReadFull(f, header) + _ = f.Close() + if !bytes.Equal(header[:n], mojoMagic) { + return nil + } + textFileName := fileName + ".txt" + if err := p.commander.Command(mojo2austinLocation, fileName, textFileName).Run(); err != nil { + return err + } + return os.Rename(textFileName, fileName) +} + func (p *austinPythonManager) handleFlamegraph(job *job.ProfilingJob, flameGrapher flamegraph.FrameGrapher, rawFileName string, flameFileName string) error { if job.OutputType == api.FlameGraph { diff --git a/internal/agent/profiler/python_austin_output_test.go b/internal/agent/profiler/python_austin_output_test.go new file mode 100644 index 0000000..c1596dd --- /dev/null +++ b/internal/agent/profiler/python_austin_output_test.go @@ -0,0 +1,81 @@ +package profiler + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + executil "github.com/nudgebee/application-profiler/internal/agent/util/exec" + "github.com/nudgebee/application-profiler/internal/agent/util/publish" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_austinStoppedOnSignal — austin ends every exposure-limited run by +// returning -SIGINT, so treating that as a failure would reject every +// successful profile. +func Test_austinStoppedOnSignal(t *testing.T) { + t.Run("254 is the normal end of an exposure-limited run", func(t *testing.T) { + err := exec.Command("sh", "-c", "exit 254").Run() + require.Error(t, err) + assert.True(t, austinStoppedOnSignal(err)) + }) + + t.Run("a real failure is still a failure", func(t *testing.T) { + err := exec.Command("sh", "-c", "exit 1").Run() + require.Error(t, err) + assert.False(t, austinStoppedOnSignal(err)) + }) + + t.Run("nil", func(t *testing.T) { + assert.False(t, austinStoppedOnSignal(nil)) + }) +} + +// Test_convertMojo — austin 4 writes the binary MOJO format, which neither +// the raw artefact nor flamegraph.pl can read; anything else must be passed +// through untouched so the conversion is safe across austin versions. +func Test_convertMojo(t *testing.T) { + t.Run("MOJO output is converted in place", func(t *testing.T) { + dir := t.TempDir() + profile := filepath.Join(dir, "profile.raw") + require.NoError(t, os.WriteFile(profile, append(mojoMagic, 1, 2, 3), 0o600)) + + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("sh", "-c", "echo 'P1;T1;f.py:main:1 42' > "+profile+".txt")) + m := &austinPythonManager{commander: commander, publisher: publish.NewFakePublisher()} + + require.NoError(t, m.convertMojo(profile)) + + converted, err := os.ReadFile(profile) + require.NoError(t, err) + assert.Equal(t, "P1;T1;f.py:main:1 42\n", string(converted)) + assert.NoFileExists(t, profile+".txt", "the intermediate file should be renamed, not left behind") + }) + + t.Run("text output is left alone", func(t *testing.T) { + dir := t.TempDir() + profile := filepath.Join(dir, "profile.raw") + require.NoError(t, os.WriteFile(profile, []byte("# austin: 3.7.0\nP1;T1;f.py:main:1 42\n"), 0o600)) + + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("false")) + m := &austinPythonManager{commander: commander, publisher: publish.NewFakePublisher()} + + require.NoError(t, m.convertMojo(profile)) + assert.Equal(t, 0, commander.On("Command").InvokedTimes(), "no conversion should run for text output") + }) + + t.Run("an empty profile is not mistaken for MOJO", func(t *testing.T) { + dir := t.TempDir() + profile := filepath.Join(dir, "profile.raw") + require.NoError(t, os.WriteFile(profile, nil, 0o600)) + + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("false")) + m := &austinPythonManager{commander: commander, publisher: publish.NewFakePublisher()} + + require.NoError(t, m.convertMojo(profile)) + }) +}