diff --git a/build.gradle.kts b/build.gradle.kts index 48d30d3..d9345e9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -183,6 +183,18 @@ listOf("scaleFixtures" to "generate", "scaleCheck" to "run").forEach { (taskName } } +tasks.register("resumeReadCheck") { + group = "verification" + description = "Measure synthetic profile-capture resume reads and elapsed time (opt-in)." + dependsOn(tasks.testClasses) + val externalJar = providers.gradleProperty("cliJar") + classpath = if (externalJar.isPresent) files(file(externalJar.get())) + sourceSets.test.get().runtimeClasspath + else sourceSets.test.get().runtimeClasspath + mainClass.set("dev.scouterctl.ResumeReadBenchmarkKt") + maxHeapSize = "512m" + args(providers.gradleProperty("resumeReadResults").getOrElse("build/resume-read-results")) +} + // Opt-in fixture for matching legacy Collector/Agent releases on Java 8. val legacyFixture = sourceSets.create("legacyFixture") configurations.named(legacyFixture.implementationConfigurationName) { diff --git a/docs/capture-resume-performance.md b/docs/capture-resume-performance.md new file mode 100644 index 0000000..070749c --- /dev/null +++ b/docs/capture-resume-performance.md @@ -0,0 +1,37 @@ +# Profile capture resume work + +Profile capture (`capture --include-profiles`) validates every committed page before recovering profiles or appending a new page. It checks SHA-256, JSON/record structure, object/time scope, profile identity and transaction uniqueness. The resume path collects existing transaction IDs during this same traversal; it does not reread all pages solely to rebuild that set. A duplicate incoming transaction is still rejected before its profile is fetched or its page is committed. + +Pages with unavailable profiles require an additional read for recovery and can be rewritten with new checksums. Already execution-complete captures still validate their committed pages and retry unavailable profiles before returning. Unreferenced or partial files are not committed pages. Capture formats, error codes and CLI budgets are unchanged. + +## Reproduce the measurement + +With JDK 21, run the opt-in application-level measurement: + +```sh +./gradlew --no-daemon resumeReadCheck +``` + +Use `gradlew.bat` on Windows. Results require a new directory; `-PresumeReadResults=PATH` selects one. Preserve an earlier result directory before choosing a new path for another trial. This task is separate from the default tests and from offline `inspect compare` scale validation. + +To compare an earlier implementation, preserve its fat JAR and run the current measurement harness with that JAR first on the runtime classpath: + +```sh +./gradlew --no-daemon resumeReadCheck -PcliJar=PATH/TO/BASELINE-all.jar -PresumeReadResults=build/resume-read-before +``` + +Use separate fresh JVM runs for each trial and compare the recorded implementation class hashes. Supplying a baseline JAR changes the measured application/store implementation; it does not replace the current synthetic fixture or measurement code. + +The fixture uses the real file-backed `EvidenceStore` and a synthetic observation source, without a Collector, credentials or simulated network delay. It captures 120 pages in three successive 40-page application invocations, then resumes the execution-complete capture once. Both 2 and 100 records per page are measured, with 20 small structural SQL steps per profile. Previously committed profiles must not be fetched again. This is local application/filesystem work, not production Collector throughput or packaged CLI startup time. + +`report.json` records OS, architecture, JDK, CPU count, heap limit, implementation class hashes and each invocation's existing page/record count, page bytes, read bytes, full-file scan-count histogram, elapsed capture time, preparation time to the first new XLog page request, added pages and profile requests. A completed capture makes no new page request, so its preparation time is null. `reads-*.jfr` retains the raw [JDK Flight Recorder](https://docs.oracle.com/en/java/javase/21/docs/api/jdk.jfr/jdk/jfr/Recording.html) events. + +JFR `jdk.FileRead` events use a zero duration threshold and are filtered to each fixture's page paths. The measurement sums positive read bytes per page and divides by that page's byte size; it does not count low-level read calls as whole scans. Capture timing excludes recorder start/dump/stop and report processing, but includes manifest I/O, page validation, synthetic profile creation, page writes and checkpointing. Recorder overhead within the measured operation remains. Preparation ends at the first synthetic page request and includes local validation/recovery. OS cache, JIT, GC and host contention are not controlled; timings are observations, not test thresholds or a universal speedup claim. No maximum-size memory or production crossover claim follows from these fixtures. + +## Remaining bounds and tradeoffs + +For an ordinary incomplete capture with no unavailable profiles, the existing-page scan counts are 0, 40 and 80 over the three capture invocations, rather than 0, 80 and 160 before the change. The subsequent completed resume scans all 120 pages once in both versions. Extra reads for unavailable-profile recovery are outside this ordinary-case count. + +Combining the passes halves these prefix scans; it does not eliminate their cumulative growth. With B new pages per invocation and P = mB total pages, ordinary repeated resumes scan Bm(m-1)/2 = (P²/B - P)/2 existing pages. Each individual resume remains linear in the committed prefix. Transaction-ID membership still requires memory proportional to committed records. Collecting through the existing validated callback retains the application's ID set while the store's uniqueness set is live, increasing simultaneous set storage; peak heap was not measured here. + +The CLI still allows `--max-pages` 1..40 and profile page sizes up to 100, with the existing capture limit of 10,000 pages. A bounded single-process continuation could retain IDs between batches and avoid repeated prefix validation, but it would need an explicit total-work budget, cancellation/checkpoint behavior and rules for on-disk changes between batches. Changing the current per-invocation budget contract is outside this fix. Persisting an unchecked ID cache or skipping validation on later invocations would weaken integrity guarantees and is not a substitute. diff --git a/docs/capture-scale-validation.md b/docs/capture-scale-validation.md index ef4f598..7bff012 100644 --- a/docs/capture-scale-validation.md +++ b/docs/capture-scale-validation.md @@ -1,5 +1,7 @@ # Capture scale validation +For synthetic profile-capture resume reads and preparation timing, see [profile capture resume work](capture-resume-performance.md). The workflow below measures offline comparison, a separate workload. + Run the opt-in **Capture scale validation** GitHub Actions workflow on a branch to measure offline `inspect compare`. It builds one CLI JAR and one synthetic capture archive, then verifies and uses the same bytes on Linux, Windows and macOS. It does not publish a release or run on every pull request. The macOS measurement job is outside the supported release-platform scope. You can also enable `scale_validation` when manually running **CI and release**, including on a feature branch. For a local Linux, macOS or Windows checkout with JDK 21: diff --git a/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt b/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt index ed44667..52b1386 100644 --- a/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt +++ b/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt @@ -6,6 +6,7 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt query.validate() if (query.pageSize > 100) throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Profile capture page-size must be 1..100") val reader = EvidenceReader(source, query.date, request.includeText) + val ids = mutableSetOf() var manifest = if (request.resume) store.load().also { if (it.schemaVersion != "scouterctl.capture/v3") { throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Resume requires a v3 profile capture; preserve this capture and start a new directory") @@ -13,7 +14,7 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt if (it.source != source.info || it.query != query || it.endpoint != request.endpoint || it.textIncluded != request.includeText) { throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Resume source, query or text policy differs from manifest") } - store.validate(it) + store.forEachRecord(it) { record -> ids.add(record.xlog.txid) } } else { store.create() EvidenceManifest(source = source.info, endpoint = request.endpoint, query = query, textIncluded = request.includeText) @@ -35,8 +36,6 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt store.save(manifest) return manifest } - val ids = mutableSetOf() - manifest.pages.indices.forEach { index -> store.readPage(manifest, index).forEach { ids.add(it.xlog.txid) } } repeat(request.maxPages) { if (manifest.pages.size >= 10000) throw OperationFailure(ErrorCode.PARTIAL_RESULT, "Evidence capture limit of 10000 pages reached") val page = source.page(query, manifest.cursor) diff --git a/src/test/kotlin/dev/scouterctl/EvidenceResumeTest.kt b/src/test/kotlin/dev/scouterctl/EvidenceResumeTest.kt new file mode 100644 index 0000000..52be721 --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/EvidenceResumeTest.kt @@ -0,0 +1,55 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.files.EvidenceStore +import dev.scouterctl.application.CaptureOptions +import dev.scouterctl.application.captureEvidence +import dev.scouterctl.core.* +import io.kotest.core.spec.style.FunSpec +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.shouldBe +import java.nio.file.Files + +class EvidenceResumeTest : FunSpec({ + test("issue 74 resume reads each committed page once while retaining IDs across page budgets") { + val directory = tempdir().toPath().resolve("capture") + val store = EvidenceStore(directory) + val profileIds = mutableListOf() + val source = MemorySource({ cursor -> + val first = cursor.txid.toLong() + 1 + val last = first + 1 + XLogPage(listOf(portRow(first), portRow(last)), Cursor(portQuery.from + last, last.toString()), last < 12) + }, { id -> profileIds += id; ProfileRecord(id.toString(), emptyList()) }) + var manifest = captureEvidence(source, portQuery, CaptureOptions(false, 2, "memory"), store) + + repeat(2) { + val expectedBytes = manifest.pages.associate { directory.resolve(it.name) to Files.size(directory.resolve(it.name)) } + val measured = measureResumeReads(directory) { + captureEvidence(source, portQuery, CaptureOptions(true, 2, "memory"), store) + } + measured.bytes shouldBe expectedBytes + manifest = measured.result + } + + manifest.complete shouldBe true + profileIds shouldBe (1L..12L).toList() + manifest.pages.flatMapIndexed { index, _ -> store.readPage(manifest, index) }.map { it.xlog.txid } shouldBe + (1L..12L).map(Long::toString) + } + + test("resume rejects an incoming transaction already committed before fetching or appending its profile") { + val store = EvidenceStore(tempdir().toPath().resolve("capture")) + var profiles = 0 + val source = MemorySource({ XLogPage(listOf(portRow(1)), Cursor(portQuery.from + 1, "1"), true) }, + { id -> profiles++; ProfileRecord(id.toString(), emptyList()) }) + val checkpoint = captureEvidence(source, portQuery, CaptureOptions(false, 1, "memory"), store) + + shouldThrow { + captureEvidence(source, portQuery, CaptureOptions(true, 1, "memory"), store) + }.code shouldBe ErrorCode.PROTOCOL_ERROR + + profiles shouldBe 1 + store.load().pages shouldBe checkpoint.pages + store.load().cursor shouldBe checkpoint.cursor + } +}) diff --git a/src/test/kotlin/dev/scouterctl/ResumeReadBenchmark.kt b/src/test/kotlin/dev/scouterctl/ResumeReadBenchmark.kt new file mode 100644 index 0000000..ebcf080 --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/ResumeReadBenchmark.kt @@ -0,0 +1,93 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.files.EvidenceStore +import dev.scouterctl.adapters.serialization.json +import dev.scouterctl.application.CaptureOptions +import dev.scouterctl.application.captureEvidence +import dev.scouterctl.core.* +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import java.nio.file.Files +import java.nio.file.Path +import java.security.MessageDigest + +@Serializable +private data class ResumeReadSample( + val pageSize: Int, val invocation: Int, val existingPages: Int, val existingRecords: Int, + val existingPageBytes: Long, val existingReadBytes: Long, val scanCounts: Map, + val preparationNanos: Long?, val captureNanos: Long, val addedPages: Int, val profileRequests: Int, + val executionComplete: Boolean, +) + +@Serializable +private data class ResumeReadReport( + val os: String, val architecture: String, val javaVersion: String, val processors: Int, + val maximumHeapBytes: Long, val classSha256: Map, val samples: List, +) + +fun main(args: Array) { + require(args.size == 1) { "Use a new result directory" } + val root = Files.createDirectory(Path.of(args.single()).toAbsolutePath()) + val samples = mutableListOf() + for (pageSize in listOf(2, 100)) { + val directory = root.resolve("records-per-page-$pageSize") + val fixture = ResumeBenchmarkFixture(pageSize) + val store = EvidenceStore(directory) + var previous: EvidenceManifest? = null + repeat(4) { index -> + val pages = previous?.pages.orEmpty() + val sizes = pages.associate { directory.resolve(it.name) to Files.size(directory.resolve(it.name)) } + val profilesBefore = fixture.profileRequests + fixture.firstPageNanos = null + var started = 0L + val measurement = measureResumeReads(directory) { + started = System.nanoTime() + captureEvidence(fixture.source, fixture.query, CaptureOptions(index > 0, 40, "memory"), store) + } + val result = measurement.result + val reads = sizes.map { (file, size) -> + val bytes = measurement.bytes[file] ?: 0L + check(bytes > 0 && bytes % size == 0L) { "Incomplete file-read measurement for $file" } + (bytes / size).toString() + }.groupingBy { it }.eachCount() + val added = result.pages.size - pages.size + val profiles = fixture.profileRequests - profilesBefore + check(profiles == added * pageSize) { "Committed profiles were fetched again" } + check(result.pages.size == minOf((index + 1) * 40, 120)) + check(result.pages.sumOf { it.records } == result.pages.size * pageSize) + check(result.executionComplete == (index >= 2)) + samples += ResumeReadSample(pageSize, index + 1, pages.size, pages.sumOf { it.records }, + sizes.values.sum(), sizes.keys.sumOf { measurement.bytes[it] ?: 0 }, reads, + fixture.firstPageNanos?.minus(started), measurement.elapsedNanos, added, profiles, result.executionComplete) + Files.move(directory.resolve("reads.jfr"), root.resolve("reads-$pageSize-${index + 1}.jfr")) + previous = result + } + check(fixture.profileRequests == 120 * pageSize) + store.validate(checkNotNull(previous)) + } + val classes = listOf("application/EvidenceCaptureKt", "adapters/files/EvidenceStore").associateWith { name -> + val bytes = checkNotNull(EvidenceStore::class.java.getResourceAsStream("/dev/scouterctl/$name.class")) + .use { it.readAllBytes() } + MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) } + } + val report = ResumeReadReport(System.getProperty("os.name") + " " + System.getProperty("os.version"), + System.getProperty("os.arch"), System.getProperty("java.runtime.version"), Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().maxMemory(), classes, samples) + Files.writeString(root.resolve("report.json"), json.encodeToString(report)) + println("Recorded ${samples.size} capture invocations in ${root.resolve("report.json")}") +} + +private class ResumeBenchmarkFixture(pageSize: Int) { + val query = portQuery.copy(to = portQuery.from + 3_600_000, pageSize = pageSize) + var profileRequests = 0 + var firstPageNanos: Long? = null + val source = MemorySource({ cursor -> + if (firstPageNanos == null) firstPageNanos = System.nanoTime() + val first = cursor.txid.toLong() + 1 + val last = first + pageSize - 1 + XLogPage((first..last).map { portRow(it) }, Cursor(query.from + last, last.toString()), last < 120L * pageSize) + }, { id -> + profileRequests++ + ProfileRecord(id.toString(), List(20) { order -> ProfileStep("sql", order, SqlProfile(123, 1, 0)) }) + }) +} diff --git a/src/test/kotlin/dev/scouterctl/ResumeReadMeasurement.kt b/src/test/kotlin/dev/scouterctl/ResumeReadMeasurement.kt new file mode 100644 index 0000000..974af4b --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/ResumeReadMeasurement.kt @@ -0,0 +1,36 @@ +package dev.scouterctl + +import jdk.jfr.Recording +import jdk.jfr.consumer.RecordingFile +import java.nio.file.Path +import java.time.Duration + +internal data class ResumeReadMeasurement(val result: T, val bytes: Map, val elapsedNanos: Long) + +internal fun measureResumeReads(directory: Path, operation: () -> T): ResumeReadMeasurement { + val recordingFile = directory.resolve("reads.jfr") + val measuredDirectory = directory.toAbsolutePath().normalize() + val elapsed: Long + val result: T + Recording().use { recording -> + recording.setToDisk(false) + recording.enable("jdk.FileRead").withThreshold(Duration.ZERO).withoutStackTrace() + recording.start() + val started = System.nanoTime() + result = operation() + elapsed = System.nanoTime() - started + recording.dump(recordingFile) + recording.stop() + } + val bytes = mutableMapOf() + RecordingFile(recordingFile).use { events -> + while (events.hasMoreEvents()) { + val event = events.readEvent() + if (event.eventType.name != "jdk.FileRead" || event.getLong("bytesRead") <= 0) continue + val path = Path.of(event.getString("path")).toAbsolutePath().normalize() + if (path.parent != measuredDirectory || !path.fileName.toString().startsWith("page-")) continue + bytes.merge(path, event.getLong("bytesRead"), Long::plus) + } + } + return ResumeReadMeasurement(result, bytes, elapsed) +}