From 1fbc86e66e1cb93969c9bbaa53c0118e2eb7f21d Mon Sep 17 00:00:00 2001 From: oxspan <36991763+oxspan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:36:45 +0900 Subject: [PATCH 1/5] =?UTF-8?q?build:=20=E2=AC=86=EF=B8=8F=20Patch=20build?= =?UTF-8?q?=20and=20verification=20dependencies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle.kts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index d9345e9..b29072f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ import java.security.MessageDigest plugins { - kotlin("jvm") version "2.4.10" - kotlin("plugin.serialization") version "2.4.10" + kotlin("jvm") version "2.4.21" + kotlin("plugin.serialization") version "2.4.21" application jacoco } @@ -63,6 +63,11 @@ dependencies { testImplementation("org.jetbrains.pty4j:pty4j:0.13.12") testImplementation("com.h2database:h2:2.3.232") testImplementation(platform("org.springframework.boot:spring-boot-dependencies:3.5.16")) + testImplementation(platform("com.fasterxml.jackson:jackson-bom:2.21.7")) + testImplementation(platform("org.apache.logging.log4j:log4j-bom:2.25.5")) + listOf("core", "el", "websocket").forEach { module -> + testImplementation("org.apache.tomcat.embed:tomcat-embed-$module:10.1.61") + } testImplementation("org.springframework.boot:spring-boot-starter-web") testImplementation("org.springframework.boot:spring-boot-starter-jdbc") testImplementation(kotlin("reflect")) @@ -200,7 +205,7 @@ val legacyFixture = sourceSets.create("legacyFixture") configurations.named(legacyFixture.implementationConfigurationName) { extendsFrom(configurations.implementation.get()) } -dependencies { add(legacyFixture.implementationConfigurationName, "com.h2database:h2:1.4.200") } +dependencies { add(legacyFixture.implementationConfigurationName, "com.h2database:h2:2.2.224") } tasks.named("compileLegacyFixtureKotlin") { setSource(files("src/test/kotlin/dev/scouterctl/fixture/TestService.kt")) compilerOptions.jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_1_8) From 6f5adf60094cb67d11921a4f3bd9459c3ebfba30 Mon Sep 17 00:00:00 2001 From: oxspan <36991763+oxspan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:37:01 +0900 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20=F0=9F=90=9B=20Reject=20absolute=20m?= =?UTF-8?q?embers=20in=20command=20archives?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scouterctl/CommandArchiveSafetyTest.kt | 39 +++++++++++++++++++ .../dev/scouterctl/CommandPackageCheck.kt | 4 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/test/kotlin/dev/scouterctl/CommandArchiveSafetyTest.kt diff --git a/src/test/kotlin/dev/scouterctl/CommandArchiveSafetyTest.kt b/src/test/kotlin/dev/scouterctl/CommandArchiveSafetyTest.kt new file mode 100644 index 0000000..a1a8f67 --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/CommandArchiveSafetyTest.kt @@ -0,0 +1,39 @@ +package dev.scouterctl + +import dev.scouterctl.bootstrap.BuildInfo +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.shouldBe +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +class CommandArchiveSafetyTest : FunSpec({ + for (unsafe in listOf("/escape", "C:/escape", "../escape", "nested/../../escape", "nested\\escape")) { + test("package validation rejects unsafe relative archive member $unsafe before installation") { + val original = Path.of(System.getProperty("cli.distribution")) + val damaged = tempdir().toPath().resolve("package.zip") + val libraries = ZipFile(original.toFile()).use { zip -> + val entries = zip.entries().asSequence().toList() + ZipOutputStream(Files.newOutputStream(damaged)).use { output -> + for (entry in entries) { + output.putNextEntry(ZipEntry(entry.name)) + if (!entry.isDirectory) zip.getInputStream(entry).use { it.copyTo(output) } + output.closeEntry() + } + output.putNextEntry(ZipEntry("scouterctl-${BuildInfo.version}/$unsafe")) + output.write("unexpected outside file".toByteArray()) + output.closeEntry() + } + entries.filter { !it.isDirectory && "/lib/" in it.name }.map { it.name.substringAfter("/lib/") }.toSet() + } + + shouldThrow { + readCommandArchive(damaged, BuildInfo.version, libraries) + }.message shouldBe "Unsafe package entry" + } + } +}) diff --git a/src/test/kotlin/dev/scouterctl/CommandPackageCheck.kt b/src/test/kotlin/dev/scouterctl/CommandPackageCheck.kt index 9ba5b39..eeea14d 100644 --- a/src/test/kotlin/dev/scouterctl/CommandPackageCheck.kt +++ b/src/test/kotlin/dev/scouterctl/CommandPackageCheck.kt @@ -53,7 +53,9 @@ internal fun readCommandArchive(file: Path, version: String, libraries: Set() for (entry in zip.entries().asSequence()) { - check(entry.name.startsWith(prefix) && '\\' !in entry.name && entry.name.split('/').none { it == ".." || it == "." }) { "Unsafe package entry" } + check(entry.name.startsWith(prefix) && '\\' !in entry.name && ':' !in entry.name && + !entry.name.removePrefix(prefix).startsWith('/') && + entry.name.split('/').none { it == ".." || it == "." }) { "Unsafe package entry" } if (entry.isDirectory) continue val name = entry.name.removePrefix(prefix) check(name.isNotEmpty() && files.put(name, zip.getInputStream(entry).use { it.readBytes() }) == null) { "Duplicate package entry" } From 305a246367d5490a310c9a2066ded5b4b1e3bf10 Mon Sep 17 00:00:00 2001 From: oxspan <36991763+oxspan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:37:02 +0900 Subject: [PATCH 3/5] =?UTF-8?q?fix:=20=F0=9F=90=9B=20Identify=20safe=20loc?= =?UTF-8?q?al=20file=20failure=20categories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/capture-recovery.md | 4 ++- .../adapters/files/FileOperation.kt | 17 +++++++-- .../kotlin/dev/scouterctl/CaptureWriteTest.kt | 2 +- .../dev/scouterctl/FileOperationTest.kt | 35 +++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 src/test/kotlin/dev/scouterctl/FileOperationTest.kt diff --git a/docs/capture-recovery.md b/docs/capture-recovery.md index 963acbf..af373f8 100644 --- a/docs/capture-recovery.md +++ b/docs/capture-recovery.md @@ -24,7 +24,9 @@ Pages that were already committed are validated before resume. An incomplete pag Windows can deny an atomic file replacement while another program holds the destination open. The CLI briefly retries access-denied replacements, preserving the committed file and the atomic replacement operation. Persistent denial still returns `IO_ERROR` (exit 10); other filesystem failures are not retried. An access-denied error can also mean insufficient permissions. -The message distinguishes local file I/O from Collector transport I/O. Close programs using the capture and check its path, permissions and available disk space. If the directory contains a valid committed manifest, rerun the original command with `--resume` after correcting the cause. If the first manifest could not be committed, preserve that directory and start a new capture in an unused directory. Do not promote a `.partial` file to a manifest or remove referenced pages to force recovery. +The message distinguishes local file I/O from Collector transport I/O and includes a safe exception category, for example `Local file I/O failed (AccessDeniedException)`. The category identifies the failure reported by Java; it does not establish which process denied access or prove an antivirus cause. Raw paths and operating-system error messages are excluded. Other categories are `NoSuchFileException`, `FileAlreadyExistsException`, `AtomicMoveNotSupportedException`, `FileSystemException`, and `IOException`. Error codes and exit codes are unchanged; scripts should use the error code rather than matching message text. + +Close programs using the capture and check its path, permissions and available disk space. If the directory contains a valid committed manifest, rerun the original command with `--resume` after correcting the cause. If the first manifest could not be committed, preserve that directory and start a new capture in an unused directory. Do not promote a `.partial` file to a manifest or remove referenced pages to force recovery. ## Recovery limits diff --git a/src/main/kotlin/dev/scouterctl/adapters/files/FileOperation.kt b/src/main/kotlin/dev/scouterctl/adapters/files/FileOperation.kt index 914af9f..f49bd31 100644 --- a/src/main/kotlin/dev/scouterctl/adapters/files/FileOperation.kt +++ b/src/main/kotlin/dev/scouterctl/adapters/files/FileOperation.kt @@ -3,13 +3,26 @@ package dev.scouterctl.adapters.files import dev.scouterctl.core.* import java.io.IOException +import java.nio.file.AccessDeniedException +import java.nio.file.AtomicMoveNotSupportedException +import java.nio.file.FileAlreadyExistsException +import java.nio.file.FileSystemException +import java.nio.file.NoSuchFileException internal inline fun fileOperation(block: () -> T): T = try { block() } catch (error: OperationFailure) { throw error -} catch (_: IOException) { - throw OperationFailure(ErrorCode.IO_ERROR, "Local file I/O failed; check the path, permissions and available disk space") +} catch (error: IOException) { + val category = when (error) { + is AccessDeniedException -> "AccessDeniedException" + is NoSuchFileException -> "NoSuchFileException" + is FileAlreadyExistsException -> "FileAlreadyExistsException" + is AtomicMoveNotSupportedException -> "AtomicMoveNotSupportedException" + is FileSystemException -> "FileSystemException" + else -> "IOException" + } + throw OperationFailure(ErrorCode.IO_ERROR, "Local file I/O failed ($category); check the path, permissions and available disk space") } catch (_: IllegalArgumentException) { throw UnexpectedFailure(ErrorCode.INVALID_ARGUMENT, "Invalid argument or manifest") } catch (_: java.time.DateTimeException) { diff --git a/src/test/kotlin/dev/scouterctl/CaptureWriteTest.kt b/src/test/kotlin/dev/scouterctl/CaptureWriteTest.kt index 231809f..c12bdae 100644 --- a/src/test/kotlin/dev/scouterctl/CaptureWriteTest.kt +++ b/src/test/kotlin/dev/scouterctl/CaptureWriteTest.kt @@ -43,7 +43,7 @@ class CaptureWriteTest : FunSpec({ val checkFailure = { val failure = shouldThrow { save() } failure.code shouldBe ErrorCode.IO_ERROR - failure.message shouldBe "Local file I/O failed; check the path, permissions and available disk space" + failure.message shouldBe "Local file I/O failed (AccessDeniedException); check the path, permissions and available disk space" Files.readString(manifest) shouldBe before Files.exists(partial) shouldBe true } diff --git a/src/test/kotlin/dev/scouterctl/FileOperationTest.kt b/src/test/kotlin/dev/scouterctl/FileOperationTest.kt new file mode 100644 index 0000000..1e9cec8 --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/FileOperationTest.kt @@ -0,0 +1,35 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.files.fileOperation +import dev.scouterctl.core.ErrorCode +import dev.scouterctl.core.OperationFailure +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldNotContain +import java.io.IOException +import java.nio.file.AccessDeniedException +import java.nio.file.AtomicMoveNotSupportedException +import java.nio.file.FileAlreadyExistsException +import java.nio.file.FileSystemException +import java.nio.file.NoSuchFileException + +class FileOperationTest : FunSpec({ + val secret = "private-path-and-filesystem-reason" + val failures = listOf( + AccessDeniedException(secret, secret, secret) to "AccessDeniedException", + NoSuchFileException(secret, secret, secret) to "NoSuchFileException", + FileAlreadyExistsException(secret, secret, secret) to "FileAlreadyExistsException", + AtomicMoveNotSupportedException(secret, secret, secret) to "AtomicMoveNotSupportedException", + FileSystemException(secret, secret, secret) to "FileSystemException", + IOException(secret) to "IOException", + ) + for ((cause, category) in failures) { + test("local I/O identifies $category without disclosing paths or filesystem messages") { + val failure = shouldThrow { fileOperation { throw cause } } + failure.code shouldBe ErrorCode.IO_ERROR + failure.message shouldBe "Local file I/O failed ($category); check the path, permissions and available disk space" + failure.message!! shouldNotContain secret + } + } +}) From f568ee6ae5487263443c781efcf165eb71c66947 Mon Sep 17 00:00:00 2001 From: oxspan <36991763+oxspan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:37:03 +0900 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20live=20diagnostic?= =?UTF-8?q?s=20and=20bounded=20profile=20captures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle.kts | 29 ++++ docs/capture-resume-performance.md | 22 ++- docs/cli-contract.md | 2 + docs/collector-timezone.md | 23 +++ docs/inspection.md | 20 +++ docs/ko/collector-timezone.md | 14 ++ docs/supported-versions.md | 3 + docs/testing.md | 12 ++ .../collector/AgentDiagnosticsDecoder.kt | 94 ++++++++++++ .../adapters/collector/AgentRequests.kt | 67 ++++++++ .../adapters/collector/CollectorSource.kt | 88 ++--------- .../collector/RuntimeSettingsDecoder.kt | 43 ++++++ .../adapters/files/EvidenceStore.kt | 48 +++++- .../serialization/AgentDiagnosticsJson.kt | 30 ++++ .../serialization/CollectorTimezoneJson.kt | 5 + .../dev/scouterctl/application/Capture.kt | 10 +- .../application/CollectorTimezone.kt | 5 +- .../scouterctl/application/EvidenceCapture.kt | 21 ++- .../dev/scouterctl/application/Ports.kt | 6 + .../dev/scouterctl/cli/CaptureCancellation.kt | 25 +++ src/main/kotlin/dev/scouterctl/cli/Cli.kt | 3 +- .../kotlin/dev/scouterctl/cli/CommandGuide.kt | 11 +- .../dev/scouterctl/cli/CommandInterpreter.kt | 16 ++ .../dev/scouterctl/cli/CommandRequest.kt | 5 +- .../kotlin/dev/scouterctl/cli/JsonOutput.kt | 19 ++- .../dev/scouterctl/cli/OnlineCommands.kt | 32 ++-- .../dev/scouterctl/core/AgentDiagnostics.kt | 12 ++ .../dev/scouterctl/core/CollectorTimezone.kt | 2 + .../dev/scouterctl/AgentDiagnosticsCheck.kt | 90 +++++++++++ .../scouterctl/AgentDiagnosticsDecoderTest.kt | 93 +++++++++++ .../dev/scouterctl/AgentDiagnosticsTest.kt | 76 +++++++++ src/test/kotlin/dev/scouterctl/AgentSmoke.kt | 14 +- .../scouterctl/CaptureContinuationCheck.kt | 79 ++++++++++ .../dev/scouterctl/CaptureContinuationTest.kt | 123 +++++++++++++++ .../dev/scouterctl/CollectorTimezoneTest.kt | 39 +++++ src/test/kotlin/dev/scouterctl/MemoryPorts.kt | 2 + .../dev/scouterctl/TimezoneStorageCheck.kt | 144 ++++++++++++++++++ src/test/kotlin/dev/scouterctl/WireFixture.kt | 35 +++++ .../dev/scouterctl/fixture/TestService.kt | 14 ++ 39 files changed, 1268 insertions(+), 108 deletions(-) create mode 100644 src/main/kotlin/dev/scouterctl/adapters/collector/AgentDiagnosticsDecoder.kt create mode 100644 src/main/kotlin/dev/scouterctl/adapters/collector/AgentRequests.kt create mode 100644 src/main/kotlin/dev/scouterctl/adapters/collector/RuntimeSettingsDecoder.kt create mode 100644 src/main/kotlin/dev/scouterctl/adapters/serialization/AgentDiagnosticsJson.kt create mode 100644 src/main/kotlin/dev/scouterctl/cli/CaptureCancellation.kt create mode 100644 src/main/kotlin/dev/scouterctl/core/AgentDiagnostics.kt create mode 100644 src/test/kotlin/dev/scouterctl/AgentDiagnosticsCheck.kt create mode 100644 src/test/kotlin/dev/scouterctl/AgentDiagnosticsDecoderTest.kt create mode 100644 src/test/kotlin/dev/scouterctl/AgentDiagnosticsTest.kt create mode 100644 src/test/kotlin/dev/scouterctl/CaptureContinuationCheck.kt create mode 100644 src/test/kotlin/dev/scouterctl/CaptureContinuationTest.kt create mode 100644 src/test/kotlin/dev/scouterctl/TimezoneStorageCheck.kt diff --git a/build.gradle.kts b/build.gradle.kts index b29072f..5b4e23e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -200,6 +200,27 @@ tasks.register("resumeReadCheck") { args(providers.gradleProperty("resumeReadResults").getOrElse("build/resume-read-results")) } +tasks.register("captureContinuationCheck") { + group = "verification" + description = "Measure repeated profile-capture resumes against bounded single-process continuation." + dependsOn(tasks.testClasses) + classpath = sourceSets.test.get().runtimeClasspath + mainClass.set("dev.scouterctl.CaptureContinuationCheckKt") + maxHeapSize = "512m" + args(providers.gradleProperty("continuationResults").getOrElse("build/capture-continuation-results")) +} + +tasks.register("timezoneStorageCheck") { + group = "verification" + description = "Verify reported time zones and DST storage-date limits against an isolated Collector (opt-in)." + dependsOn(tasks.testClasses, "fatJar") + classpath = sourceSets.test.get().runtimeClasspath + mainClass.set("dev.scouterctl.TimezoneStorageCheckKt") + args(providers.gradleProperty("scouterRoot").getOrElse("../scouter"), + providers.gradleProperty("timezoneResults").getOrElse("build/timezone-storage-results")) + systemProperty("cli.jar", layout.buildDirectory.file("libs/scouterctl-${project.version}-all.jar").get().asFile.absolutePath) +} + // Opt-in fixture for matching legacy Collector/Agent releases on Java 8. val legacyFixture = sourceSets.create("legacyFixture") configurations.named(legacyFixture.implementationConfigurationName) { @@ -235,6 +256,14 @@ tasks.register("agentInspectionSmoke") { layout.buildDirectory.file("libs/scouterctl-${project.version}-all.jar").get().asFile.absolutePath)) } +tasks.register("agentDiagnosticsSmoke") { + dependsOn(tasks.testClasses, "fatJar") + classpath = sourceSets.test.get().runtimeClasspath + mainClass.set("dev.scouterctl.AgentSmokeKt") + args(providers.gradleProperty("scouterRoot").getOrElse("../scouter"), "--diagnostics-only") + systemProperty("cli.jar", layout.buildDirectory.file("libs/scouterctl-${project.version}-all.jar").get().asFile.absolutePath) +} + tasks.register("counterSmoke") { dependsOn(tasks.testClasses, "fatJar") diff --git a/docs/capture-resume-performance.md b/docs/capture-resume-performance.md index 070749c..4b446c9 100644 --- a/docs/capture-resume-performance.md +++ b/docs/capture-resume-performance.md @@ -34,4 +34,24 @@ For an ordinary incomplete capture with no unavailable profiles, the existing-pa Combining the passes halves these prefix scans; it does not eliminate their cumulative growth. With B new pages per invocation and P = mB total pages, ordinary repeated resumes scan Bm(m-1)/2 = (P²/B - P)/2 existing pages. Each individual resume remains linear in the committed prefix. Transaction-ID membership still requires memory proportional to committed records. Collecting through the existing validated callback retains the application's ID set while the store's uniqueness set is live, increasing simultaneous set storage; peak heap was not measured here. -The CLI still allows `--max-pages` 1..40 and profile page sizes up to 100, with the existing capture limit of 10,000 pages. A bounded single-process continuation could retain IDs between batches and avoid repeated prefix validation, but it would need an explicit total-work budget, cancellation/checkpoint behavior and rules for on-disk changes between batches. Changing the current per-invocation budget contract is outside this fix. Persisting an unchecked ID cache or skipping validation on later invocations would weaken integrity guarantees and is not a substitute. +The default invocation still allows `--max-pages` 1..40 and profile page sizes up to 100, with the existing capture limit of 10,000 pages. Repeated separate invocations retain the cumulative cost above. + +## Bounded single-process continuation + +Profile captures can opt into `--max-total-pages N` to continue beyond the ordinary `--max-pages` budget in the same process. N must be between `--max-pages` and 10,000 and counts **new pages in this invocation**, including after `--resume`; it is not the final capture size. Defaults and behavior without this option are unchanged. This option requires `capture --include-profiles`. + +```sh +scouterctl capture --include-profiles --resume --directory captures/incident \ + --date 20261101 --from 2026-11-01T00:00:00Z --to 2026-11-01T01:00:00Z \ + --timezone UTC --object-hash 42 --max-total-pages 1000 --no-input +``` + +The process validates the existing prefix once, retains transaction IDs in memory and checkpoints each new page before requesting the next. Reaching the budget without finishing returns the ordinary incomplete capture result (exit 8). Resume still fully validates all committed pages; no persistent unchecked ID cache or format change is introduced. Profile recovery can reread pages whose profiles were unavailable. + +Profile writers hold an exclusive `.writer.lock` for the invocation: a file lock on Unix and an exclusive file handle on Windows, including Windows access to WSL UNC paths where byte-range locking is unavailable. Concurrent writers using the same platform fail with `IO_ERROR`; readers remain available. Do not mix Windows and Linux writers on a WSL-shared directory: these platform mechanisms do not establish cross-platform coordination. A changed or removed manifest is detected before checkpoint replacement and is not overwritten. Atomic moves and advisory locks do not defend against an external process editing immutable page files, replacing lock files, or racing the final compare-and-replace. Do not modify a live capture directory. Stop the writer and use a fresh full validation/resume after external edits. This restriction also applies between ordinary page-budget boundaries inside a continued invocation. + +Cancellation is checked before each page and profile request and before committing a new page. A cooperative cancellation reports `PARTIAL_RESULT` with the last committed cursor intact; retry with `--resume`. On terminal termination the CLI allows up to five seconds for in-flight work to notice cancellation and release its writer lock. An uninterruptible network call or forced termination can outlive that grace period; already committed pages remain recoverable and unreferenced partial/page files are ignored. Cancellation does not guarantee upstream cancellation. + +ID storage is still O(committed records). Continuation removes repeated prefix scans between separately launched invocations, but it does not bound record memory independently of capture size. Choose a finite total budget for the available heap and workload. + +Run `./gradlew captureContinuationCheck -PcontinuationResults=NEW_DIRECTORY` for a file-backed comparison of repeated 40-page resumes and one continued invocation after a common 40-page seed. It measures 120 and 400 pages with 100 records and 20 structural SQL steps per profile, under a 512 MiB heap. The JSON report includes positive existing-page read bytes from JFR, measured capture elapsed time, per-heap-pool peak usage and post-operation heap usage. Pool peaks can occur at different instants, include measurement overhead and are not simultaneous peak heap or process RSS. GC timing and shared-JVM warmup affect memory and time observations; no universal speedup or memory reduction follows from these synthetic local runs. diff --git a/docs/cli-contract.md b/docs/cli-contract.md index f0c910e..e52464b 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -73,6 +73,8 @@ A consumer should accept the stream schema, process records, then require both t Online window queries reject offset mismatches against the reported Collector JVM zone before streaming or writing captures. `doctor` and `collector status` expose the property's ID and a client-rule offset with explicit provenance; unavailable metadata remains `null`. These checks do not verify the storage calendar. See [Collector time zones](collector-timezone.md). +`xlog search` and `counter history` repeat `timezoneCheck` in both `start` and `complete`; `capture` and `counter export` include it in the outer result envelope. Its `status` is `reported-offset-match` when the interval comparison succeeds, or `unavailable` when no usable Collector zone was supplied. `collectorTimezone` contains the reported ID and client-derived offset with provenance, or `null`. All four commands retain `storageDateVerification: "unverified"`: neither a reported-zone match nor exit 0 proves that an empty result means no data existed. Saved manifest formats remain unchanged. + `counter history` uses its own `scouterctl.counter-stream/v1` schema with the same start/record/complete lifecycle. Its start includes the dated target and counter metadata, and records contain stored timestamps and numeric strings. See [counter history](counter-history.md) for missing values, units and query limits. `counter export` returns a single manifest result and writes JSONL chunk files. Its `scouterctl.counter-capture/v1` format is separate from XLog captures. See [resumable counter export](counter-export.md) for budgets, checkpoints and completion semantics. diff --git a/docs/collector-timezone.md b/docs/collector-timezone.md index b1aec26..50f6730 100644 --- a/docs/collector-timezone.md +++ b/docs/collector-timezone.md @@ -33,8 +33,31 @@ scouterctl xlog search --object-hash 42 --date 20261109 --timezone Asia/Seoul \ The same instants declared as `America/Los_Angeles` with date `20261108` are rejected. A UTC timestamp suffix is still valid input with `--timezone Asia/Seoul`; the timestamp syntax itself is not required to use the Collector's offset. +## Query verification in live output + +Window queries include a `timezoneCheck` object in live output: the `start` and `complete` events of `xlog search` and `counter history`, and the outer result envelope of `capture` and `counter export`. + +```json +{"timezoneCheck":{"status":"unavailable","collectorTimezone":null},"storageDateVerification":"unverified"} +``` + +`status: "reported-offset-match"` means the declared zone matched the reported JVM zone throughout the requested interval under the client's rules. `collectorTimezone` contains the same ID, reference time, offset and provenance fields as `doctor`; its offset is for that reference time, not necessarily the historical query. `status: "unavailable"` means the Collector supplied no usable zone metadata, so no comparison was possible. Transport and malformed-protocol failures still fail the command instead of producing this status. + +The check is repeated in the terminal stream event so a consumer retaining only the completion receipt can distinguish the two states. Neither state verifies the storage calendar, and `storageDateVerification` remains `"unverified"` for all four commands. Treat an empty successful result as a completed extraction from an unverified partition, not proof that no traffic or counter samples existed. These are additive live fields; saved manifest schemas and `timezoneVerification: "operator-declared"` do not change. + ## Storage dates remain unverified This is a comparison with a **reported JVM zone**, not confirmation of the storage partition. XLog `storageDateVerification: "unverified"` and capture `timezoneVerification: "operator-declared"` remain unchanged. Capture formats and strict-reader compatibility are unchanged; the new diagnostic fields are live output only. Scouter's `DateTimeHelper` in 2.8.1 and 2.21.3 builds a calendar from local `20000101` with fixed 86,400,000-ms days. That calendar can differ from civil time around DST. For example, in `America/Chicago`, `2026-07-01T05:30:00Z` is July 1 by current zone rules but June 30 in that storage calendar. A matching zone check cannot certify the correct partition or prove that an empty result means no traffic. Confirm storage dates with the Collector operator when results are unexpected. This change does not repair that upstream calendar behavior or verify the server's cached calendar through the protocol. + +## Reproduce the storage-calendar check + +With a built Scouter 2.21.3 checkout used read-only and an installed Java 8 runtime: + +```sh +SCOUTER_COLLECTOR_JAVA=/path/to/java8/bin/java ./gradlew timezoneStorageCheck \ + -PscouterRoot=/path/to/scouter -PtimezoneResults=build/timezone-storage-trial +``` + +The result directory must not exist. The opt-in check starts isolated UTC and `America/Chicago` Collectors, writes synthetic XLogs at the same fixed instant, reads the stored partition, and runs the packaged CLI. It retains JSON output, command arguments, Collector logs and teardown receipts. It stops its processes and removes temporary account files. This verifies the 2.21.3 calendar behavior with synthetic telemetry, not every supported Collector or Agent combination. diff --git a/docs/inspection.md b/docs/inspection.md index 98fbb5e..e52a2e9 100644 --- a/docs/inspection.md +++ b/docs/inspection.md @@ -32,6 +32,26 @@ Both commands go through the selected Collector. Status includes the Agent name, Configuration lists the Agent's runtime settings and defaults. Visible settings include selected numeric and boolean controls, such as `trace_sql_parameter_max_length`, `profile_http_parameter_enabled` and `profile_http_querystring_enabled`, plus the text settings `net_collector_ip`, `obj_name` and `obj_type`. Other settings report null for current and default values with `redacted: true`, including when selected by `--key`. Neither command writes configuration. An unknown target, an offline Agent's configuration, unavailable configuration or a missing setting returns exit 7. Malformed responses return exit 6. The CLI remains on JDK 21 even when the Agent/application runs on Java 8. +## Active requests and thread dumps + +Use one nonzero Java Agent hash from `object list`: + +```sh +scouterctl agent active-requests --object-hash 123456 +scouterctl agent active-requests --object-hash 123456 --include-text +scouterctl agent thread-dump --object-hash 123456 +``` + +`active-requests` takes one live snapshot. It returns `collector`, `agent`, `requests`, `complete` and `textIncluded`. Each request has a decimal-string `txid` and `threadId`, the Agent's thread `state`, `elapsedMs` and decimal-string `threadCpuTimeMs`. CPU time is the thread's cumulative CPU time in milliseconds, not CPU used by that request; some Agents return zero when measurement is unavailable. A reported -1 is represented as null. Deferred or reactive requests may have thread ID `"0"` and state `"n/a"`. Rows are ordered by elapsed time descending. Only requests currently represented by the Agent are included; this is not a historical search or a count of all application activity. + +Service and thread names are null unless `--include-text` is supplied. Even with that option, SQL statements, client IP addresses, login names and arbitrary request descriptions are excluded. The decimal `txid` can be used with `profile get` after the request finishes and reaches Collector storage. A complete, empty snapshot succeeds; a snapshot explicitly reported incomplete returns the rows with `executionComplete: false` and exit 8. + +`thread-dump` explicitly requests raw diagnostic text and returns it in `data.lines`, with Collector and Agent identity. Thread names, stack traces, lock details and other JVM-produced text can contain sensitive data; review the output before sharing. No remote dump file or thread-control command is requested. The Agent/JVM may require attachment support; Java 9+ attachment-based Agents can require `-Djdk.attach.allowAttachSelf=true` at application startup. The CLI does not change that setting. Agent-side failures return a generic diagnostic without echoing raw attachment errors or local paths. + +Dump `completeness` is `"unknown"`: the upstream protocol does not provide a reliable truncation indicator and some implementations cap output at 100,000 lines. Command `executionComplete` means the returned response was processed, not that every thread or frame was captured. The CLI rejects over 100,000 lines or 8,388,608 decoded characters, and active snapshots over 10,000 rows. These are post-decode checks, not bounds on Agent work or total transport allocation. A timeout does not guarantee upstream cancellation. + +Unknown/offline targets, missing diagnostic replies and unavailable JVM dumps exit 7 (`NOT_FOUND`); malformed responses exit 6 (`PROTOCOL_ERROR`). The protocol cannot reliably distinguish an unsupported diagnostic from an Agent that disconnected after discovery. The commands use the existing Collector account, whose `effectivePermission` remains `unknown`; successful authentication does not establish per-operation authorization. Platform, JVM and Agent combinations need separate validation from the general Collector version list. + ## Real-time counters Use a target hash from `object list`: diff --git a/docs/ko/collector-timezone.md b/docs/ko/collector-timezone.md index 7512e5b..89cdd7a 100644 --- a/docs/ko/collector-timezone.md +++ b/docs/ko/collector-timezone.md @@ -14,6 +14,20 @@ ID는 인증된 기존 `SERVER_ENV` 요청으로 받은 JVM `user.timezone` 속 예를 들어 `Asia/Seoul`을 보고한 Collector에서는 같은 시각을 `America/Los_Angeles`와 그 지역 날짜로 선언한 요청을 거부합니다. `--from`·`--to`를 UTC 문자열로 주면서 `--timezone Asia/Seoul`을 지정하는 방식은 계속 사용할 수 있습니다. +## 조회 결과의 검증 상태 + +시간 구간 조회의 실시간 출력에는 `timezoneCheck`가 포함됩니다. `xlog search`와 `counter history`의 `start` 및 `complete` 이벤트, `capture`와 `counter export`의 최상위 결과에 기록합니다. + +```json +{"timezoneCheck":{"status":"unavailable","collectorTimezone":null},"storageDateVerification":"unverified"} +``` + +`status: "reported-offset-match"`는 요청한 전체 구간에서 선언한 시간대와 Collector가 보고한 JVM 시간대의 오프셋이 CLI 규칙으로 일치했다는 뜻입니다. `collectorTimezone`은 `doctor`와 같은 ID·기준 시각·오프셋·출처 정보를 담습니다. 이 오프셋은 기준 시각의 값이며 과거 조회 구간의 오프셋과 다를 수 있습니다. `status: "unavailable"`는 사용할 수 있는 시간대 정보가 없어 비교하지 못했다는 뜻입니다. 통신 오류나 잘못된 프로토콜 응답은 여전히 명령 실패로 처리합니다. + +마지막 이벤트에도 같은 상태를 기록하므로 완료 결과만 보관하는 도구도 두 경우를 구분할 수 있습니다. 어느 상태도 저장 달력을 검증하지 않으며, 네 명령 모두 `storageDateVerification: "unverified"`를 유지합니다. 성공한 빈 결과는 검증되지 않은 파티션의 추출 완료를 뜻하며 요청이나 지표가 없었다는 증거가 아닙니다. 새 필드는 실시간 출력에만 추가되며 저장 manifest 스키마와 `timezoneVerification: "operator-declared"`는 바뀌지 않습니다. + +## 저장 날짜의 한계 + 이 검사는 보고된 JVM 시간대와의 비교입니다. 실행 중 변경된 기본 시간대, 서버와 CLI의 시간대 데이터 차이, 실제 저장 파티션까지 보장하지 않습니다. XLog의 `storageDateVerification: "unverified"`와 캡처의 `timezoneVerification: "operator-declared"`는 유지합니다. 새 진단 필드는 실시간 출력에만 추가하며 캡처 형식은 바꾸지 않습니다. Scouter 2.8.1·2.21.3의 `DateTimeHelper`는 해당 지역의 `20000101`을 기준으로 하루를 86,400,000ms씩 증가시킵니다. 이 저장 달력은 DST가 반영된 실제 지역 날짜와 다를 수 있습니다. `America/Chicago`에서 `2026-07-01T05:30:00Z`는 지역 시간상 7월 1일이지만 저장 달력에서는 6월 30일입니다. 시간대가 일치해도 빈 결과를 트래픽 부재로 단정하지 말고, 예상과 다르면 운영자와 저장 날짜를 확인하세요. 이번 변경은 이 Scouter 저장 달력 동작을 수정하지 않습니다. 자세한 예시는 [영문 가이드](../collector-timezone.md)를 참고하세요. diff --git a/docs/supported-versions.md b/docs/supported-versions.md index bc34265..97c1800 100644 --- a/docs/supported-versions.md +++ b/docs/supported-versions.md @@ -26,10 +26,13 @@ The matching-Agent column covers a Java 8 HTTP/JDBC application with explicit se | Collector/Agent inspection and real-time counters | All releases above | | Stored counter history and checkpointed multi-day export | All releases above | | Full Collector restart recovery and native Windows client checks | 2.8.1 and 2.21.3 | +| Active-request snapshots and live thread dumps | 2.8.1 with matching Agent on Java 8; 2.21.3 with Java 21 Agent | | Host/process CPU, memory and I/O | [Separate Host Agent combinations](host-counters.md) | Windows client checks connect to Linux-hosted Collectors and Agents. They do not establish Windows-hosted Agent support. The CLI uses a pinned 2.21.3 protocol library; that library version is independent of the supported server list. +The live-diagnostics checks held a real instrumented request, read its active-request row and thread-dump stack, then matched the same transaction ID in stored XLogs after completion. They ran Linux Collectors on Java 8: the official 2.8.1 distribution with its matching Java 8 Agent, and a locally built 2.21.3 Collector/Agent combination with a Java 21 application. Self-attachment was enabled for the diagnostic fixture. They do not establish support for all listed releases, every JVM vendor, virtual-thread visibility or Windows-hosted Agents. Unavailable diagnostics fail explicitly; see [live diagnostics](inspection.md#active-requests-and-thread-dumps). + These results do not cover every Java distribution/update, Agent integration, Collector configuration or operating system. Collector 2.21.3 passed on Java 8 in these checks; running the CLI on JDK 21 does not establish Collector-on-JDK-21 support. Support for new commands or additional versions needs separate validation. ## Reproduce the checks diff --git a/docs/testing.md b/docs/testing.md index a74c8c7..14c09ce 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -13,6 +13,8 @@ The following integration tasks start isolated Collector/Agent/application proce ## Coverage +Linux release-script tests also require Bash, `jq`, `unzip` and `sha256sum` on PATH. A missing utility is an environment failure; retain the failing output and fix PATH rather than weakening the release tests. + Run the complete suite, generate JaCoCo HTML/XML reports, and enforce minimum coverage: ```sh @@ -53,6 +55,8 @@ export SCOUTER_SERVICE_CLASSPATH="$(./gradlew -q legacyFixtureClasspath)" ./gradlew agentSmoke -PscouterRoot="$SCOUTER_SOURCE" # Run only Agent status/settings checks: ./gradlew agentInspectionSmoke -PscouterRoot="$SCOUTER_SOURCE" +# Active request snapshot, live thread dump, then stored XLog correlation: +./gradlew agentDiagnosticsSmoke -PscouterRoot="$SCOUTER_SOURCE" # Counter reads, optionally with real HTTP load: SCOUTER_COUNTER_LOAD=true ./gradlew counterSmoke -PscouterRoot="$SCOUTER_SOURCE" # Stored counter history, units, boundaries and checkpointed multi-day export: @@ -63,6 +67,8 @@ SCOUTER_COUNTER_RECOVERY=true ./gradlew counterHistorySmoke `JAVA8`, `SCOUTER_DIST` and `SCOUTER_SOURCE` refer to your Java 8 installation, extracted distribution and built Scouter source checkout. The legacy JDBC dependency is test-only and is excluded from the CLI package. Build/run Gradle with JDK 21. +`agentDiagnosticsSmoke` holds one real instrumented HTTP request on a latch, observes the active request with and without names, finds the held request in a live thread dump, releases it and matches the same transaction ID in stored XLogs. It also checks empty snapshots, authentication, missing targets and unchanged Collector/account/Agent configuration. It enables self-attachment only for its isolated fixture JVM and stops its processes in `finally`. The printed evidence directory contains raw CLI JSON/stderr, `diagnostics-summary.json` and `teardown.json`. As with the other Agent checks, use `SCOUTER_COLLECTOR_*`, `SCOUTER_AGENT_JAR` and `SCOUTER_SERVICE_*` for a chosen distribution; the ordinary test suite uses protocol fixtures and cannot establish actual Agent/JVM support. + For Servlet checks, unset the legacy `SCOUTER_AGENT_JAR`, `SCOUTER_SERVICE_JAVA` and `SCOUTER_SERVICE_CLASSPATH` overrides. Use `springSmoke` with a built Java 21-capable Agent from the source checkout. `recoverySmoke` adds actual Collector stop/restart to that scenario. `collectorInspectionSmoke` starts only the Collector and checks status, settings, authentication and unchanged configuration/account files; it does not start an Agent or application. The optional `SCOUTER_SERVICE_*` and `SCOUTER_AGENT_JAR` overrides apply to `agentSmoke`, `agentInspectionSmoke`, `counterSmoke` and `counterHistorySmoke`, not `springSmoke`. See [capture recovery](capture-recovery.md) and [protocol library provenance](../vendor/README.md). @@ -102,6 +108,12 @@ Optionally pass `-PcliJar=/path/to/extracted/scouterctl-VERSION-all.jar` to veri The task prints its temporary evidence directory, including `recovery-summary.json` and process teardown results. It stops all started services and deletes the temporary account file; retain the evidence needed for review and remove the directory afterward. This real Collector/Servlet check runs separately from the portable Kotest suite. Release checks run protocol fixtures on Linux and Windows; these do not establish real Collector instrumentation on either platform. +## Verification dependency security + +The HTTP/JDBC and Spring services are test-only, loopback-bound fixtures and are excluded from command packages. The Java8 fixture uses H2 2.2.224; modern fixture tests use H2 2.3.232, which requires Java11 or later. Keep the Java8 bytecode check and real JDBC smoke when changing the legacy dependency. + +Spring Boot remains on the public 3.5 line. Test dependency platforms explicitly select patched Jackson/Log4j versions, and all embedded Tomcat modules share one patch version. These overrides require the real Spring instrumentation smoke after updates. The Spring MVC advisories [CVE-2026-47884](https://spring.io/security/cve-2026-47884) and [CVE-2026-47890](https://spring.io/security/cve-2026-47890) have no public Framework 6.2 patch as of 2026-11-15; this fixture does not configure XsltView or SSE fragments. Do not suppress the alerts or claim an unavailable public fix. Reassess when a compatible public patch is available. + ## Offline scale measurements The optional [capture scale workflow](capture-scale-validation.md) uses synthetic captures and reports timing/memory measurements. Its macOS diagnostic job is separate from the supported release-platform checks. diff --git a/src/main/kotlin/dev/scouterctl/adapters/collector/AgentDiagnosticsDecoder.kt b/src/main/kotlin/dev/scouterctl/adapters/collector/AgentDiagnosticsDecoder.kt new file mode 100644 index 0000000..4380147 --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/adapters/collector/AgentDiagnosticsDecoder.kt @@ -0,0 +1,94 @@ +package dev.scouterctl.adapters.collector + +import dev.scouterctl.core.* +import scouter.lang.pack.MapPack +import scouter.lang.pack.Pack +import scouter.lang.value.BooleanValue +import scouter.lang.value.DecimalValue +import scouter.lang.value.ListValue +import scouter.lang.value.NullValue +import scouter.lang.value.TextValue +import scouter.util.Hexa32 + +internal fun decodeActiveRequests(agent: AgentIdentity, response: List, includeText: Boolean): ActiveRequests { + val pack = diagnosticPack(response, "Active requests") + val returnedHash = (pack.get("objHash") as? DecimalValue)?.value + ?: invalidDiagnostics("Missing active-request Agent identity") + if (returnedHash != agent.objHash.toLong()) invalidDiagnostics("Active-request Agent identity differs") + if (pack.get("id") == null && pack.size() == 1) unavailableDiagnostics("Active requests") + fun column(name: String): ListValue = pack.get(name) as? ListValue + ?: invalidDiagnostics("Invalid active-request columns") + val ids = column("id") + val elapsed = column("elapsed") + val states = column("stat") + val transactions = column("txid") + val cpus = column("cpu") + val services = column("service") + val names = column("name") + if (ids.size() > 10000 || listOf(elapsed, states, transactions, cpus, services, names).any { it.size() != ids.size() }) { + invalidDiagnostics("Inconsistent or excessive active-request rows") + } + val complete = (pack.get("complete") as? BooleanValue)?.value + ?: invalidDiagnostics("Missing active-request completion flag") + fun number(column: ListValue, index: Int): Long = (column.get(index) as? DecimalValue)?.value + ?: invalidDiagnostics("Invalid active-request number") + fun text(column: ListValue, index: Int): String = (column.get(index) as? TextValue)?.value + ?: invalidDiagnostics("Invalid active-request text") + fun optionalText(column: ListValue, index: Int): String? = when (val value = column.get(index)) { + is NullValue -> null + is TextValue -> value.value + else -> invalidDiagnostics("Invalid active-request text") + } + val seen = mutableSetOf() + val requests = (0 until ids.size()).map { index -> + val id = number(ids, index) + val duration = number(elapsed, index) + val cpu = number(cpus, index) + val state = text(states, index) + if (id < 0 || duration < 0 || cpu < -1 || state !in threadStates) invalidDiagnostics("Invalid active-request values") + val rawTxid = text(transactions, index) + val txid = try { Hexa32.toLong32(rawTxid) } catch (_: NumberFormatException) { + invalidDiagnostics("Invalid active-request transaction ID") + } + if (rawTxid.isEmpty() || Hexa32.toString32(txid) != rawTxid || !seen.add(txid)) { + invalidDiagnostics("Invalid or duplicate active-request transaction ID") + } + val service = optionalText(services, index) + val name = optionalText(names, index) + ActiveRequest(txid.toString(), id.toString(), state, duration, + cpu.takeIf { it >= 0 }?.toString(), service.takeIf { includeText }, name.takeIf { includeText }) + } + return ActiveRequests(agent, requests.sortedWith(compareByDescending { it.elapsedMs }.thenBy { it.txid }), complete, includeText) +} + +internal fun decodeThreadDump(agent: AgentIdentity, response: List): AgentThreadDump { + val pack = diagnosticPack(response, "Thread dump") + val error = pack.get("error") + if (error != null) { + if (error !is TextValue) invalidDiagnostics("Invalid thread-dump error") + throw OperationFailure(ErrorCode.NOT_FOUND, "Agent could not produce a thread dump; check Agent JVM attachment support and logs") + } + val lines = pack.get("threadDump") as? ListValue ?: invalidDiagnostics("Invalid thread-dump response") + if (lines.size() == 0) unavailableDiagnostics("Thread dump") + if (lines.size() > 100000) invalidDiagnostics("Thread dump exceeds 100000 lines") + var characters = 0L + val decoded = (0 until lines.size()).map { index -> + val line = (lines.get(index) as? TextValue)?.value ?: invalidDiagnostics("Invalid thread-dump line") + characters += line.length + if (characters > 8_388_608) invalidDiagnostics("Thread dump exceeds 8388608 characters") + line + } + return AgentThreadDump(agent, decoded) +} + +private val threadStates = setOf("NEW", "RUNNABLE", "BLOCKED", "WAITING", "TIMED_WAITING", "TERMINATED", "n/a") + +private fun diagnosticPack(response: List, label: String): MapPack { + if (response.isEmpty()) unavailableDiagnostics(label) + return response.singleOrNull() as? MapPack ?: invalidDiagnostics("Expected one $label response") +} + +private fun unavailableDiagnostics(label: String): Nothing = + throw OperationFailure(ErrorCode.NOT_FOUND, "$label unavailable; the Agent may not support this diagnostic or may have disconnected") + +private fun invalidDiagnostics(message: String): Nothing = throw OperationFailure(ErrorCode.PROTOCOL_ERROR, message) diff --git a/src/main/kotlin/dev/scouterctl/adapters/collector/AgentRequests.kt b/src/main/kotlin/dev/scouterctl/adapters/collector/AgentRequests.kt new file mode 100644 index 0000000..bf0be83 --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/adapters/collector/AgentRequests.kt @@ -0,0 +1,67 @@ +package dev.scouterctl.adapters.collector + +import dev.scouterctl.core.* +import dev.scouterctl.application.agentSettingValueIsVisible +import dev.scouterctl.application.agentTextSettingValueIsVisible +import scouter.lang.pack.MapPack +import scouter.lang.pack.ObjectPack +import scouter.lang.pack.Pack +import scouter.lang.value.TextValue +import scouter.net.RequestCmd + +internal class AgentRequests(private val exchange: (String, MapPack?) -> List) { + fun status(objHash: Int): AgentStatus { + val agent = identity(objHash) + if (!agent.alive) return AgentStatus(agent, AgentJvmAvailability.OFFLINE, null) + val response = exchange(RequestCmd.OBJECT_ENV, MapPack().apply { put("objHash", objHash.toLong()) }) + if (response.isEmpty()) return AgentStatus(agent, AgentJvmAvailability.UNAVAILABLE, null) + val pack = response.singleOrNull() as? MapPack + ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected one Agent JVM response") + fun text(key: String): String? { + val value = pack.get(key) ?: return null + return (value as? TextValue)?.value + ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid Agent JVM field") + } + val javaVersion = text("java.version") + ?: return AgentStatus(agent, AgentJvmAvailability.UNAVAILABLE, null) + if (javaVersion.isBlank()) throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid Agent Java version") + return AgentStatus(agent, AgentJvmAvailability.AVAILABLE, AgentJvm(javaVersion, text("java.runtime.version"), + text("java.vm.name"), text("java.vm.version"), text("java.vendor"), text("os.name"), text("os.arch"))) + } + + fun settings(objHash: Int): AgentConfiguration { + val agent = identity(objHash) + if (!agent.alive) throw OperationFailure(ErrorCode.NOT_FOUND, "Agent is not alive") + val response = exchange(RequestCmd.LIST_CONFIGURE_WAS, MapPack().apply { put("objHash", objHash.toLong()) }) + if (response.isEmpty()) throw OperationFailure(ErrorCode.NOT_FOUND, "Agent settings are unavailable") + return AgentConfiguration(agent, decodeRuntimeSettings(response, ::agentSettingValueIsVisible, ::agentTextSettingValueIsVisible)) + } + + fun activeRequests(objHash: Int, includeText: Boolean): ActiveRequests { + val target = diagnosticAgent(objHash) + return decodeActiveRequests(target, exchange(RequestCmd.OBJECT_ACTIVE_SERVICE_LIST, + MapPack().apply { put("objHash", objHash.toLong()) }), includeText) + } + + fun threadDump(objHash: Int): AgentThreadDump { + val target = diagnosticAgent(objHash) + return decodeThreadDump(target, exchange(RequestCmd.OBJECT_THREAD_DUMP, + MapPack().apply { put("objHash", objHash.toLong()) })) + } + + private fun diagnosticAgent(objHash: Int): AgentIdentity { + if (objHash == 0) throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Diagnostics require a nonzero --object-hash") + return identity(objHash).also { + if (!it.alive) throw OperationFailure(ErrorCode.NOT_FOUND, "Agent is offline; live diagnostics are unavailable") + } + } + + fun identity(objHash: Int): AgentIdentity { + val agents = exchange(RequestCmd.OBJECT_LIST_REAL_TIME, null).map { + it as? ObjectPack ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected object pack") + }.filter { it.objHash == objHash } + if (agents.size > 1) throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Duplicate Agent identity") + val agent = agents.singleOrNull() ?: throw OperationFailure(ErrorCode.NOT_FOUND, "Agent was not found") + return AgentIdentity(agent.objHash, agent.objName ?: "", agent.objType ?: "", agent.alive, agent.version ?: "unknown") + } +} diff --git a/src/main/kotlin/dev/scouterctl/adapters/collector/CollectorSource.kt b/src/main/kotlin/dev/scouterctl/adapters/collector/CollectorSource.kt index 4eb3733..ee9e501 100644 --- a/src/main/kotlin/dev/scouterctl/adapters/collector/CollectorSource.kt +++ b/src/main/kotlin/dev/scouterctl/adapters/collector/CollectorSource.kt @@ -3,8 +3,6 @@ package dev.scouterctl.adapters.collector import dev.scouterctl.core.* import dev.scouterctl.application.ObservationSource import dev.scouterctl.application.collectorSettingValueIsVisible -import dev.scouterctl.application.agentSettingValueIsVisible -import dev.scouterctl.application.agentTextSettingValueIsVisible import scouter.io.DataInputX import scouter.io.DataOutputX @@ -13,9 +11,6 @@ import scouter.lang.TextTypes import scouter.lang.step.Step import scouter.lang.step.StepControl import scouter.lang.value.BooleanValue -import scouter.lang.value.ListValue -import scouter.lang.value.TextValue -import scouter.lang.value.Value import scouter.lang.value.DecimalValue import scouter.net.NetCafe import scouter.net.RequestCmd @@ -36,6 +31,7 @@ class CollectorSource private constructor(private val socket: Socket, private va } } private val output = DataOutputX(BufferedOutputStream(socket.getOutputStream())) + private val agents = AgentRequests(::exchange) private var session = 0L override lateinit var info: ServerInfo private set @@ -114,35 +110,17 @@ class CollectorSource private constructor(private val socket: Socket, private va } override fun settings(): List = collectorOperation { - parseSettings(exchange(RequestCmd.LIST_CONFIGURE_SERVER), ::collectorSettingValueIsVisible) + decodeRuntimeSettings(exchange(RequestCmd.LIST_CONFIGURE_SERVER), ::collectorSettingValueIsVisible) } - override fun agentStatus(objHash: Int): AgentStatus = collectorOperation { - val agent = agent(objHash) - if (!agent.alive) return@collectorOperation AgentStatus(agent, AgentJvmAvailability.OFFLINE, null) - val response = exchange(RequestCmd.OBJECT_ENV, MapPack().apply { put("objHash", objHash.toLong()) }) - if (response.isEmpty()) return@collectorOperation AgentStatus(agent, AgentJvmAvailability.UNAVAILABLE, null) - val pack = response.singleOrNull() as? MapPack - ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected one Agent JVM response") - fun text(key: String): String? { - val value = pack.get(key) ?: return null - return (value as? TextValue)?.value - ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid Agent JVM field") - } - val javaVersion = text("java.version") - ?: return@collectorOperation AgentStatus(agent, AgentJvmAvailability.UNAVAILABLE, null) - if (javaVersion.isBlank()) throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid Agent Java version") - AgentStatus(agent, AgentJvmAvailability.AVAILABLE, AgentJvm(javaVersion, text("java.runtime.version"), - text("java.vm.name"), text("java.vm.version"), text("java.vendor"), text("os.name"), text("os.arch"))) - } + override fun agentStatus(objHash: Int): AgentStatus = collectorOperation { agents.status(objHash) } - override fun agentSettings(objHash: Int): AgentConfiguration = collectorOperation { - val agent = agent(objHash) - if (!agent.alive) throw OperationFailure(ErrorCode.NOT_FOUND, "Agent is not alive") - val response = exchange(RequestCmd.LIST_CONFIGURE_WAS, MapPack().apply { put("objHash", objHash.toLong()) }) - if (response.isEmpty()) throw OperationFailure(ErrorCode.NOT_FOUND, "Agent settings are unavailable") - AgentConfiguration(agent, parseSettings(response, ::agentSettingValueIsVisible, ::agentTextSettingValueIsVisible)) - } + override fun agentSettings(objHash: Int): AgentConfiguration = collectorOperation { agents.settings(objHash) } + + override fun activeRequests(objHash: Int, includeText: Boolean): ActiveRequests = + collectorOperation { agents.activeRequests(objHash, includeText) } + + override fun threadDump(objHash: Int): AgentThreadDump = collectorOperation { agents.threadDump(objHash) } override fun counterHistory(query: CounterHistoryQuery): CounterHistory = collectorOperation { val target = decodeCounterTarget(query.objHash, datedObjects(query.date)) @@ -155,59 +133,13 @@ class CollectorSource private constructor(private val socket: Socket, private va } override fun realtimeCounters(objHash: Int): RealtimeCounters = collectorOperation { - val agent = agent(objHash) + val agent = agents.identity(objHash) val metadata = exchange(RequestCmd.GET_XML_COUNTER) val values = if (agent.alive) exchange(RequestCmd.COUNTER_REAL_TIME_OBJECT_ALL, MapPack().apply { put("objHash", objHash.toLong()) }) else emptyList() decodeCounters(agent, metadata, values) } - private fun agent(objHash: Int): AgentIdentity { - val agents = exchange(RequestCmd.OBJECT_LIST_REAL_TIME).map { - it as? ObjectPack ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected object pack") - }.filter { it.objHash == objHash } - if (agents.size > 1) throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Duplicate Agent identity") - val agent = agents.singleOrNull() ?: throw OperationFailure(ErrorCode.NOT_FOUND, "Agent was not found") - return AgentIdentity(agent.objHash, agent.objName ?: "", agent.objType ?: "", agent.alive, agent.version ?: "unknown") - } - - private fun parseSettings( - response: List, - valueIsVisible: (String) -> Boolean, - textValueIsVisible: (String) -> Boolean = { false }, - ): List { - val pack = response.singleOrNull() as? MapPack - ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected one runtime settings response") - fun items(key: String): ListValue = pack.get(key) as? ListValue - ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid runtime settings fields") - val keys = items("key") - val values = items("value") - val defaults = items("default") - if (keys.size() > 5000 || keys.size() != values.size() || keys.size() != defaults.size()) { - throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Inconsistent runtime settings lists") - } - val seen = mutableSetOf() - fun invalidValue(): Nothing = throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid operational setting value") - fun visibleValue(key: String, value: Value): String { - if (textValueIsVisible(key)) return (value as? TextValue)?.value ?: invalidValue() - return when (value) { - is DecimalValue -> value.value.toString() - is BooleanValue -> value.value.toString() - else -> invalidValue() - } - } - return (0 until keys.size()).map { index -> - val key = (keys.get(index) as? TextValue)?.value - ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid runtime setting key") - if (key.isEmpty() || key.length > 256 || !seen.add(key)) { - throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid or duplicate runtime setting key") - } - val visible = valueIsVisible(key) - RuntimeSetting(key, if (visible) visibleValue(key, values.get(index)) else null, - if (visible) visibleValue(key, defaults.get(index)) else null, !visible) - } - } - private fun datedObjects(date: String): List = decodeDatedObjects(exchange(RequestCmd.OBJECT_LIST_LOAD_DATE, MapPack().apply { put("date", date) })) diff --git a/src/main/kotlin/dev/scouterctl/adapters/collector/RuntimeSettingsDecoder.kt b/src/main/kotlin/dev/scouterctl/adapters/collector/RuntimeSettingsDecoder.kt new file mode 100644 index 0000000..05f8ace --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/adapters/collector/RuntimeSettingsDecoder.kt @@ -0,0 +1,43 @@ +package dev.scouterctl.adapters.collector + +import dev.scouterctl.core.* +import scouter.lang.pack.MapPack +import scouter.lang.pack.Pack +import scouter.lang.value.* + +internal fun decodeRuntimeSettings( + response: List, + valueIsVisible: (String) -> Boolean, + textValueIsVisible: (String) -> Boolean = { false }, +): List { + val pack = response.singleOrNull() as? MapPack + ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Expected one runtime settings response") + fun items(key: String): ListValue = pack.get(key) as? ListValue + ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid runtime settings fields") + val keys = items("key") + val values = items("value") + val defaults = items("default") + if (keys.size() > 5000 || keys.size() != values.size() || keys.size() != defaults.size()) { + throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Inconsistent runtime settings lists") + } + val seen = mutableSetOf() + fun invalidValue(): Nothing = throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid operational setting value") + fun visibleValue(key: String, value: Value): String { + if (textValueIsVisible(key)) return (value as? TextValue)?.value ?: invalidValue() + return when (value) { + is DecimalValue -> value.value.toString() + is BooleanValue -> value.value.toString() + else -> invalidValue() + } + } + return (0 until keys.size()).map { index -> + val key = (keys.get(index) as? TextValue)?.value + ?: throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid runtime setting key") + if (key.isEmpty() || key.length > 256 || !seen.add(key)) { + throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Invalid or duplicate runtime setting key") + } + val visible = valueIsVisible(key) + RuntimeSetting(key, if (visible) visibleValue(key, values.get(index)) else null, + if (visible) visibleValue(key, defaults.get(index)) else null, !visible) + } +} diff --git a/src/main/kotlin/dev/scouterctl/adapters/files/EvidenceStore.kt b/src/main/kotlin/dev/scouterctl/adapters/files/EvidenceStore.kt index ca14040..72e4bff 100644 --- a/src/main/kotlin/dev/scouterctl/adapters/files/EvidenceStore.kt +++ b/src/main/kotlin/dev/scouterctl/adapters/files/EvidenceStore.kt @@ -8,14 +8,49 @@ import kotlinx.serialization.encodeToString import java.nio.file.Files import java.nio.file.LinkOption.NOFOLLOW_LINKS import java.nio.file.Path +import java.nio.file.OpenOption +import java.nio.file.FileSystemException +import java.nio.channels.FileChannel +import java.nio.channels.OverlappingFileLockException +import java.nio.file.StandardOpenOption.CREATE +import java.nio.file.StandardOpenOption.WRITE +import com.sun.nio.file.ExtendedOpenOption.NOSHARE_READ +import com.sun.nio.file.ExtendedOpenOption.NOSHARE_WRITE +import com.sun.nio.file.ExtendedOpenOption.NOSHARE_DELETE import java.security.MessageDigest class EvidenceStore(private val directory: Path) : EvidenceRepository { + private var writing = false + private var expectedManifest: String? = null + + override fun withWriter(resume: Boolean, action: () -> T): T = fileOperation { + if (resume) requireCaptureDirectory(directory) else create() + val windows = System.getProperty("os.name").startsWith("Windows") + val options = mutableSetOf(CREATE, WRITE, NOFOLLOW_LINKS) + if (windows) options.addAll(listOf(NOSHARE_READ, NOSHARE_WRITE, NOSHARE_DELETE)) + val opened = try { FileChannel.open(directory.resolve(".writer.lock"), options) } + catch (_: FileSystemException) { writerUnavailable() } + opened.use { channel -> + if (!windows) try { + if (channel.tryLock() == null) writerUnavailable() + } catch (_: OverlappingFileLockException) { + writerUnavailable() + } + writing = true + expectedManifest = null + try { action() } finally { + writing = false + expectedManifest = null + } + } + } + override fun create() = createCaptureDirectory(directory) override fun load(): EvidenceManifest = fileOperation { requireCaptureDirectory(directory) val content = read("manifest.json", 8_388_608) + if (writing) expectedManifest = content requireCaptureSchema(content, "scouterctl.capture/v2", "scouterctl.capture/v3") val manifest = json.decodeFromString(content).toModel() manifest.query.validate() @@ -25,7 +60,16 @@ class EvidenceStore(private val directory: Path) : EvidenceRepository { } override fun save(manifest: EvidenceManifest) = fileOperation { - write("manifest.json", json.encodeToString(manifest.toDto())) + if (writing) { + val exists = Files.exists(directory.resolve("manifest.json"), NOFOLLOW_LINKS) + if ((expectedManifest == null && exists) || (expectedManifest != null && + (!exists || read("manifest.json", 8_388_608) != expectedManifest))) { + throw OperationFailure(ErrorCode.IO_ERROR, "Capture checkpoint changed during this invocation; stop other writers and validate before resuming") + } + } + val content = json.encodeToString(manifest.toDto()) + write("manifest.json", content) + if (writing) expectedManifest = content } override fun writePage(index: Int, records: List): EvidencePage = fileOperation { @@ -93,4 +137,6 @@ class EvidenceStore(private val directory: Path) : EvidenceRepository { private fun digest(bytes: ByteArray): String = MessageDigest.getInstance("SHA-256") .digest(bytes).joinToString("") { "%02x".format(it) } private fun damaged(): Nothing = throw OperationFailure(ErrorCode.IO_ERROR, "Capture integrity or record validation failed") + private fun writerUnavailable(): Nothing = throw OperationFailure(ErrorCode.IO_ERROR, + "Profile capture writer lock is unavailable; stop other writers and check write permissions") } diff --git a/src/main/kotlin/dev/scouterctl/adapters/serialization/AgentDiagnosticsJson.kt b/src/main/kotlin/dev/scouterctl/adapters/serialization/AgentDiagnosticsJson.kt new file mode 100644 index 0000000..db07600 --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/adapters/serialization/AgentDiagnosticsJson.kt @@ -0,0 +1,30 @@ +package dev.scouterctl.adapters.serialization + +import dev.scouterctl.core.* +import kotlinx.serialization.Serializable + +@Serializable +data class ActiveRequestDto( + val txid: String, val threadId: String, val state: String, val elapsedMs: Long, + val threadCpuTimeMs: String?, val service: String?, val threadName: String?, +) + +@Serializable +data class ActiveRequestsDto( + val collector: ServerInfoDto, val agent: AgentIdentityDto, val requests: List, + val complete: Boolean, val textIncluded: Boolean, +) + +@Serializable +data class AgentThreadDumpDto( + val collector: ServerInfoDto, val agent: AgentIdentityDto, val lines: List, + val completeness: String = "unknown", +) + +fun ActiveRequests.toDto(source: ServerInfo): ActiveRequestsDto = ActiveRequestsDto( + source.toDto(), agent.toDto(), requests.map { + ActiveRequestDto(it.txid, it.threadId, it.state, it.elapsedMs, it.threadCpuTimeMs, it.service, it.threadName) + }, complete, textIncluded, +) + +fun AgentThreadDump.toDto(source: ServerInfo): AgentThreadDumpDto = AgentThreadDumpDto(source.toDto(), agent.toDto(), lines) diff --git a/src/main/kotlin/dev/scouterctl/adapters/serialization/CollectorTimezoneJson.kt b/src/main/kotlin/dev/scouterctl/adapters/serialization/CollectorTimezoneJson.kt index a3d0049..59ae42f 100644 --- a/src/main/kotlin/dev/scouterctl/adapters/serialization/CollectorTimezoneJson.kt +++ b/src/main/kotlin/dev/scouterctl/adapters/serialization/CollectorTimezoneJson.kt @@ -14,6 +14,11 @@ private fun CollectorTimezone?.toJson(): JsonElement = this?.let { json.encodeToJsonElement(CollectorTimezoneDto(it.id, it.utcOffsetSeconds, it.referenceTimeEpochMs)) } ?: JsonNull +fun QueryTimezoneCheck.toJson(): JsonObject = buildJsonObject { + put("status", if (collectorTimezone == null) "unavailable" else "reported-offset-match") + put("collectorTimezone", collectorTimezone.toJson()) +} + fun ServerInfo.toLiveJson(timezone: CollectorTimezone?): JsonObject = JsonObject(json.encodeToJsonElement(toDto()).jsonObject + ("timezone" to timezone.toJson())) diff --git a/src/main/kotlin/dev/scouterctl/application/Capture.kt b/src/main/kotlin/dev/scouterctl/application/Capture.kt index 99a2718..12d785b 100644 --- a/src/main/kotlin/dev/scouterctl/application/Capture.kt +++ b/src/main/kotlin/dev/scouterctl/application/Capture.kt @@ -3,7 +3,15 @@ package dev.scouterctl.application import dev.scouterctl.core.* data class CaptureOptions(val resume: Boolean, val maxPages: Int, val endpoint: String, - val includeText: Boolean = false) + val includeText: Boolean = false, val maxTotalPages: Int? = null, + val cancelled: () -> Boolean = { false }) { + fun pageBudget(): Int { + if (maxTotalPages != null && maxTotalPages !in maxPages..10000) { + throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Use max-total-pages between max-pages and 10000") + } + return maxTotalPages ?: maxPages + } +} fun capture(source: ObservationSource, query: Query, request: CaptureOptions, store: CaptureRepository): CaptureManifest { query.validate() diff --git a/src/main/kotlin/dev/scouterctl/application/CollectorTimezone.kt b/src/main/kotlin/dev/scouterctl/application/CollectorTimezone.kt index 3da0414..95fce3d 100644 --- a/src/main/kotlin/dev/scouterctl/application/CollectorTimezone.kt +++ b/src/main/kotlin/dev/scouterctl/application/CollectorTimezone.kt @@ -11,8 +11,8 @@ fun describeCollectorTimezone(id: String, referenceTimeEpochMs: Long): Collector return CollectorTimezone(zone.id, offset.totalSeconds, referenceTimeEpochMs) } -fun validateCollectorTimezone(declared: String, from: Long, to: Long, collector: CollectorTimezone?) { - if (collector == null) return +fun validateCollectorTimezone(declared: String, from: Long, to: Long, collector: CollectorTimezone?): QueryTimezoneCheck { + if (collector == null) return QueryTimezoneCheck(null) val requested = ZoneId.of(declared).rules val reported = ZoneId.of(collector.id).rules val start = Instant.ofEpochMilli(from) @@ -31,4 +31,5 @@ fun validateCollectorTimezone(declared: String, from: Long, to: Long, collector: transition = rules.nextTransition(transition.instant) } } + return QueryTimezoneCheck(collector) } diff --git a/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt b/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt index 52b1386..2cee5ae 100644 --- a/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt +++ b/src/main/kotlin/dev/scouterctl/application/EvidenceCapture.kt @@ -5,6 +5,11 @@ import dev.scouterctl.core.* fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOptions, store: EvidenceRepository): EvidenceManifest { query.validate() if (query.pageSize > 100) throw OperationFailure(ErrorCode.INVALID_ARGUMENT, "Profile capture page-size must be 1..100") + request.pageBudget() + return store.withWriter(request.resume) { captureSession(source, query, request, store) } +} + +private fun captureSession(source: ObservationSource, query: Query, request: CaptureOptions, store: EvidenceRepository): EvidenceManifest { val reader = EvidenceReader(source, query.date, request.includeText) val ids = mutableSetOf() var manifest = if (request.resume) store.load().also { @@ -16,15 +21,18 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt } store.forEachRecord(it) { record -> ids.add(record.xlog.txid) } } else { - store.create() EvidenceManifest(source = source.info, endpoint = request.endpoint, query = query, textIncluded = request.includeText) .also(store::save) } try { for (index in manifest.pages.indices) { + checkCancellation(request) if (manifest.pages[index].unavailableProfiles == 0) continue val records = store.readPage(manifest, index).map { record -> - if (record.profileStatus == ProfileAvailability.UNAVAILABLE) reader.fetch(record.xlog) + if (record.profileStatus == ProfileAvailability.UNAVAILABLE) { + checkCancellation(request) + reader.fetch(record.xlog) + } else record } val updated = store.writePage(index, records) @@ -36,13 +44,16 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt store.save(manifest) return manifest } - repeat(request.maxPages) { + repeat(request.pageBudget()) { + checkCancellation(request) if (manifest.pages.size >= 10000) throw OperationFailure(ErrorCode.PARTIAL_RESULT, "Evidence capture limit of 10000 pages reached") val page = source.page(query, manifest.cursor) val records = page.records.map { row -> + checkCancellation(request) if (!ids.add(row.txid)) throw OperationFailure(ErrorCode.PROTOCOL_ERROR, "Duplicate transaction in capture") reader.fetch(row) } + checkCancellation(request) val saved = store.writePage(manifest.pages.size, records) manifest = manifest.copy(pages = manifest.pages + saved, cursor = page.cursor, executionComplete = !page.hasMore, lastFailure = null) @@ -57,6 +68,10 @@ fun captureEvidence(source: ObservationSource, query: Query, request: CaptureOpt } } +private fun checkCancellation(request: CaptureOptions) { + if (request.cancelled()) throw OperationFailure(ErrorCode.PARTIAL_RESULT, "Capture cancelled; resume from the last committed checkpoint with --resume") +} + private class EvidenceReader(private val source: ObservationSource, private val date: String, private val includeText: Boolean) { fun fetch(xlog: XLogRecord): EvidenceRecord { return try { diff --git a/src/main/kotlin/dev/scouterctl/application/Ports.kt b/src/main/kotlin/dev/scouterctl/application/Ports.kt index b130c7b..49d7f9e 100644 --- a/src/main/kotlin/dev/scouterctl/application/Ports.kt +++ b/src/main/kotlin/dev/scouterctl/application/Ports.kt @@ -9,6 +9,8 @@ interface ObservationSource : AutoCloseable { fun settings(): List fun agentStatus(objHash: Int): AgentStatus fun agentSettings(objHash: Int): AgentConfiguration + fun activeRequests(objHash: Int, includeText: Boolean = false): ActiveRequests + fun threadDump(objHash: Int): AgentThreadDump fun counterHistory(query: CounterHistoryQuery): CounterHistory fun realtimeCounters(objHash: Int): RealtimeCounters fun objects(date: String? = null): List @@ -25,6 +27,10 @@ interface CaptureRepository { } interface EvidenceRepository { + fun withWriter(resume: Boolean, action: () -> T): T { + if (!resume) create() + return action() + } fun create() fun load(): EvidenceManifest fun save(manifest: EvidenceManifest) diff --git a/src/main/kotlin/dev/scouterctl/cli/CaptureCancellation.kt b/src/main/kotlin/dev/scouterctl/cli/CaptureCancellation.kt new file mode 100644 index 0000000..02e6df2 --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/cli/CaptureCancellation.kt @@ -0,0 +1,25 @@ +package dev.scouterctl.cli + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +internal class CaptureCancellation : AutoCloseable { + private val requested = AtomicBoolean() + private val finished = CountDownLatch(1) + private val hook = Thread({ + requested.set(true) + finished.await(5, TimeUnit.SECONDS) + }, "scouterctl-capture-checkpoint") + + init { Runtime.getRuntime().addShutdownHook(hook) } + + fun cancelled(): Boolean = requested.get() || Thread.currentThread().isInterrupted + + override fun close() { + finished.countDown() + try { Runtime.getRuntime().removeShutdownHook(hook) } catch (_: IllegalStateException) { + // Shutdown already owns the hook; its bounded wait has been released. + } + } +} diff --git a/src/main/kotlin/dev/scouterctl/cli/Cli.kt b/src/main/kotlin/dev/scouterctl/cli/Cli.kt index 4fde5a4..0e6a61c 100644 --- a/src/main/kotlin/dev/scouterctl/cli/Cli.kt +++ b/src/main/kotlin/dev/scouterctl/cli/Cli.kt @@ -10,7 +10,7 @@ import java.nio.file.Path import java.util.concurrent.Callable @Command(name = "scouterctl", mixinStandardHelpOptions = true, showAtFileInUsageHelp = true, - description = ["Scouter Collector CLI. Commands: counter list, counter get, counter history, counter export, collector status, collector config, agent status, agent config, doctor, object list, xlog search, profile get, capture, inspect, inspect search, inspect summary, inspect compare.", + description = ["Scouter Collector CLI. Commands: counter list, counter get, counter history, counter export, collector status, collector config, agent status, agent config, agent active-requests, agent thread-dump, doctor, object list, xlog search, profile get, capture, inspect, inspect search, inspect summary, inspect compare.", "Connect: doctor --host HOST --port PORT --user ACCOUNT, then object list with the same options.", "Save/select connections: context add/list/use --name NAME. Check credentials: auth login.", "Password: SCOUTERCTL_PASSWORD, or --interactive for hidden terminal input. Passwords are never saved.", @@ -31,6 +31,7 @@ class Cli(private val dispatcher: CommandDispatcher, private val output: JsonOut @Option(names = ["--object-hash"]) var objHash: Int? = null @Option(names = ["--page-size"], description = ["Default 1000; profile capture default 25, maximum 100"]) var pageSize: Int? = null @Option(names = ["--max-pages"], defaultValue = "40") var maxPages: Int = 40 + @Option(names = ["--max-total-pages"], description = ["Profile capture: continue in one process up to this total page budget (maximum 10000)"]) var maxTotalPages: Int? = null @Option(names = ["--txid"]) var txid: Long? = null @Option(names = ["--directory"]) var directory: Path? = null @Option(names = ["--before"], description = ["inspect compare: baseline v2/v3 capture directory"]) var before: Path? = null diff --git a/src/main/kotlin/dev/scouterctl/cli/CommandGuide.kt b/src/main/kotlin/dev/scouterctl/cli/CommandGuide.kt index a756791..cabd0ae 100644 --- a/src/main/kotlin/dev/scouterctl/cli/CommandGuide.kt +++ b/src/main/kotlin/dev/scouterctl/cli/CommandGuide.kt @@ -37,6 +37,12 @@ internal val commandGuides = listOf( "agent status --object-hash 42 --user reader --no-input", connectionNotes), CommandGuide("agent config", "Read safe Agent settings; optionally select an exact key.", setOf("--object-hash"), connectionOptions + "--key", "agent config --object-hash 42 --user reader --no-input", connectionNotes), + CommandGuide("agent active-requests", "Snapshot currently active requests on one Agent.", setOf("--object-hash"), connectionOptions + "--include-text", + "agent active-requests --object-hash 42 --user reader --no-input", + connectionNotes + "--include-text reveals service and thread names; SQL, client IP and login values are excluded."), + CommandGuide("agent thread-dump", "Read a live Agent thread dump as a JSON array of lines.", setOf("--object-hash"), connectionOptions, + "agent thread-dump --object-hash 42 --user reader --no-input", + connectionNotes + "The dump contains raw diagnostic text, including thread names and stack traces; review before sharing."), CommandGuide("object list", "List live objects, or recorded objects for --date.", emptySet(), connectionOptions + "--date", "object list --user reader --no-input", connectionNotes), CommandGuide("counter list", "List available real-time counters for an object.", setOf("--object-hash"), connectionOptions, @@ -54,10 +60,11 @@ internal val commandGuides = listOf( CommandGuide("profile get", "Read one stored transaction profile.", setOf("--date", "--txid"), connectionOptions + setOf("--include-text", "--group-sql"), "profile get --date 20261101 --txid 123 --user reader --no-input", connectionNotes + "Use --group-sql for SQL hash aggregates; --include-text is still required for statements."), CommandGuide("capture", "Save a resumable XLog capture, optionally including profiles.", xlogOptions + "--directory", - connectionOptions + pagingOptions + setOf("--resume", "--include-profiles", "--include-text"), + connectionOptions + pagingOptions + setOf("--resume", "--include-profiles", "--include-text", "--max-total-pages"), "capture $windowExample --directory capture --user reader --no-input", connectionNotes + listOf("--include-text requires --include-profiles. Text can contain sensitive values.", - "Use the same query and directory with --resume. Profile capture defaults to page size 25, maximum 100.")), + "Use the same query and directory with --resume. Profile capture defaults to page size 25, maximum 100.", + "--max-total-pages requires --include-profiles and extends this invocation's budget up to 10000 pages without repeated resume scans.")), CommandGuide("inspect", "Read a captured profile offline.", setOf("--directory", "--txid"), offlineOptions + "--group-sql", "inspect --directory capture --txid 123", listOf("Use --group-sql for SQL hash aggregates; --include-text requires stored text.")), CommandGuide("inspect search", "Search captured requests offline.", setOf("--directory"), offlineOptions + diff --git a/src/main/kotlin/dev/scouterctl/cli/CommandInterpreter.kt b/src/main/kotlin/dev/scouterctl/cli/CommandInterpreter.kt index aa484fa..00783ca 100644 --- a/src/main/kotlin/dev/scouterctl/cli/CommandInterpreter.kt +++ b/src/main/kotlin/dev/scouterctl/cli/CommandInterpreter.kt @@ -8,11 +8,26 @@ import java.time.format.DateTimeFormatter internal fun Cli.interpret(): CommandRequest { val command = words.joinToString(" ") + if (maxTotalPages != null && (command != "capture" || !includeProfiles)) { + invalid("--max-total-pages requires capture --include-profiles") + } + if (maxTotalPages != null && maxTotalPages !in maxPages..10000) { + invalid("Use --max-total-pages between --max-pages and 10000") + } if (spec.commandLine().parseResult.hasMatchedOption("--group-sql") && command !in setOf("profile get", "inspect")) { invalid("--group-sql requires profile get or inspect") } if (command in setOf("context add", "context list", "context use")) return contextRequest(command) if (name != null) invalid("--name requires context add/use") + if (command in setOf("agent active-requests", "agent thread-dump")) { + allowedOptions() + val hash = required(objHash, "--object-hash") + if (hash == 0) invalid("Diagnostics require a nonzero --object-hash") + return when (command) { + "agent active-requests" -> OnlineRequest.AgentActiveRequests(hash, includeText) + else -> OnlineRequest.AgentThreadDump(hash) + } + } if (command == "inspect compare") { allowedOptions() return OfflineRequest.Compare(required(before, "--before"), required(after, "--after"), ComparisonOptions(limit ?: 20, includeText)) @@ -117,6 +132,7 @@ private fun Cli.onlineRequest(command: String): OnlineRequest { "xlog search" -> OnlineRequest.Search(required(query, "query"), maxPages, includeText) "capture" -> OnlineRequest.Capture( required(query, "query"), required(directory, "--directory"), resume, maxPages, includeProfiles, includeText, + maxTotalPages, ) else -> invalid("Unknown command") } diff --git a/src/main/kotlin/dev/scouterctl/cli/CommandRequest.kt b/src/main/kotlin/dev/scouterctl/cli/CommandRequest.kt index bc82b40..33da57d 100644 --- a/src/main/kotlin/dev/scouterctl/cli/CommandRequest.kt +++ b/src/main/kotlin/dev/scouterctl/cli/CommandRequest.kt @@ -22,6 +22,8 @@ sealed interface OnlineRequest : CommandRequest { data class CollectorConfig(val key: String?) : OnlineRequest data class AgentStatus(val objHash: Int) : OnlineRequest data class AgentConfig(val objHash: Int, val key: String?) : OnlineRequest + data class AgentActiveRequests(val objHash: Int, val includeText: Boolean) : OnlineRequest + data class AgentThreadDump(val objHash: Int) : OnlineRequest data class CounterExport(val query: CounterExportQuery, val directory: Path, val resume: Boolean, val maxChunks: Int) : OnlineRequest data class CounterHistory(val query: CounterHistoryQuery) : OnlineRequest data class Counters(val objHash: Int, val name: String?) : OnlineRequest @@ -29,5 +31,6 @@ sealed interface OnlineRequest : CommandRequest { data class Profile(val date: String, val txid: Long, val includeText: Boolean, val groupSql: Boolean = false) : OnlineRequest data class Search(val query: Query, val maxPages: Int, val includeText: Boolean = false) : OnlineRequest data class Capture(val query: Query, val directory: Path, val resume: Boolean, - val maxPages: Int, val includeProfiles: Boolean, val includeText: Boolean) : OnlineRequest + val maxPages: Int, val includeProfiles: Boolean, val includeText: Boolean, + val maxTotalPages: Int? = null) : OnlineRequest } diff --git a/src/main/kotlin/dev/scouterctl/cli/JsonOutput.kt b/src/main/kotlin/dev/scouterctl/cli/JsonOutput.kt index cecb845..70efbdc 100644 --- a/src/main/kotlin/dev/scouterctl/cli/JsonOutput.kt +++ b/src/main/kotlin/dev/scouterctl/cli/JsonOutput.kt @@ -34,24 +34,27 @@ class JsonOutput(private val stream: PrintStream) { return failure(OperationFailure(ErrorCode.PROTOCOL_ERROR, "Response could not be decoded")) } } - fun result(data: JsonElement, complete: Boolean = true, xlogCapture: Boolean = false): Int { + fun result(data: JsonElement, complete: Boolean = true, xlogCapture: Boolean = false, + timezoneCheck: QueryTimezoneCheck? = null): Int { emit(buildJsonObject { put("schemaVersion", "scouterctl.result/v1") put("ok", true) put("data", data) put("executionComplete", complete) if (xlogCapture) put("storageDateVerification", STORAGE_DATE_UNVERIFIED) + putTimezoneCheck(timezoneCheck) }) return if (complete) 0 else 8 } - fun event(event: XLogEvent) = when (event) { + fun event(event: XLogEvent, timezoneCheck: QueryTimezoneCheck? = null) = when (event) { is XLogEvent.Start -> emit(buildJsonObject { put("type", "start") put("schemaVersion", "scouterctl.stream/v1") put("source", json.encodeToJsonElement(event.source.toDto())) put("query", json.encodeToJsonElement(event.query.toDto())) put("storageDateVerification", STORAGE_DATE_UNVERIFIED) + putTimezoneCheck(timezoneCheck) }) is XLogEvent.Record -> emit(buildJsonObject { put("type", "record") @@ -70,10 +73,11 @@ class JsonOutput(private val stream: PrintStream) { put("executionComplete", true) put("upstreamCoverage", "unknown") put("sampling", "unknown") + putTimezoneCheck(timezoneCheck) }) } - fun counterHistoryEvent(event: CounterHistoryEvent) = when (event) { + fun counterHistoryEvent(event: CounterHistoryEvent, timezoneCheck: QueryTimezoneCheck? = null) = when (event) { is CounterHistoryEvent.Start -> emit(buildJsonObject { put("type", "start") put("schemaVersion", "scouterctl.counter-stream/v1") @@ -85,6 +89,7 @@ class JsonOutput(private val stream: PrintStream) { put("intervalMs", JsonNull) put("missingValues", "omitted") put("metadataScope", "current-collector-definition") + putTimezoneCheck(timezoneCheck) }) is CounterHistoryEvent.Record -> emit(buildJsonObject { put("type", "record") @@ -96,9 +101,17 @@ class JsonOutput(private val stream: PrintStream) { put("records", event.records) put("upstreamCoverage", "unknown") put("sampling", "unknown") + putTimezoneCheck(timezoneCheck) }) } + private fun JsonObjectBuilder.putTimezoneCheck(check: QueryTimezoneCheck?) { + if (check != null) { + put("timezoneCheck", check.toJson()) + put("storageDateVerification", STORAGE_DATE_UNVERIFIED) + } + } + private fun emit(value: JsonElement) { stream.println(value) if (stream.checkError()) throw OperationFailure(ErrorCode.IO_ERROR, "Output stream failed") diff --git a/src/main/kotlin/dev/scouterctl/cli/OnlineCommands.kt b/src/main/kotlin/dev/scouterctl/cli/OnlineCommands.kt index ff4b597..10772db 100644 --- a/src/main/kotlin/dev/scouterctl/cli/OnlineCommands.kt +++ b/src/main/kotlin/dev/scouterctl/cli/OnlineCommands.kt @@ -21,12 +21,12 @@ class OnlineCommands( connection.user, connection.context, connection.timeoutMs) val secret = password.read(connection.interactive, connection.noInput, request is OnlineRequest.Doctor && request.login) return source(endpoint, secret).use { observation -> - when (request) { + val timezoneCheck = when (request) { is OnlineRequest.Search -> validateCollectorTimezone(request.query.timezone, request.query.from, request.query.to, observation.timezone()) is OnlineRequest.Capture -> validateCollectorTimezone(request.query.timezone, request.query.from, request.query.to, observation.timezone()) is OnlineRequest.CounterHistory -> validateCollectorTimezone(request.query.timezone, request.query.from, request.query.to, observation.timezone()) is OnlineRequest.CounterExport -> validateCollectorTimezone(request.query.timezone, request.query.from, request.query.to, observation.timezone()) - else -> Unit + else -> null } when (request) { is OnlineRequest.Doctor -> output.result(observation.info.toLiveJson(observation.timezone())) @@ -34,23 +34,28 @@ class OnlineCommands( is OnlineRequest.CollectorConfig -> output.result(json.encodeToJsonElement(collectorSettings(observation, request.key).toSettingsDto(observation.info))) is OnlineRequest.AgentStatus -> output.result(json.encodeToJsonElement(observation.agentStatus(request.objHash).toDto(observation.info))) is OnlineRequest.AgentConfig -> output.result(json.encodeToJsonElement(agentSettings(observation, request.objHash, request.key).toDto(observation.info))) + is OnlineRequest.AgentActiveRequests -> { + val snapshot = observation.activeRequests(request.objHash, request.includeText) + output.result(json.encodeToJsonElement(snapshot.toDto(observation.info)), snapshot.complete) + } + is OnlineRequest.AgentThreadDump -> output.result(json.encodeToJsonElement(observation.threadDump(request.objHash).toDto(observation.info))) is OnlineRequest.CounterExport -> counterCaptures(request.directory).use { store -> val manifest = exportCounters(observation, request.query, CounterExportOptions(request.resume, request.maxChunks, "${endpoint.host}:${endpoint.port}"), store) - output.result(json.encodeToJsonElement(manifest.toDto()), manifest.executionComplete) + output.result(json.encodeToJsonElement(manifest.toDto()), manifest.executionComplete, timezoneCheck = timezoneCheck) } is OnlineRequest.CounterHistory -> { - streamCounterHistory(observation, request.query, output::counterHistoryEvent) + streamCounterHistory(observation, request.query) { output.counterHistoryEvent(it, timezoneCheck) } 0 } is OnlineRequest.Counters -> output.result(json.encodeToJsonElement(realtimeCounters(observation, request.objHash, request.name).toDto(observation.info))) is OnlineRequest.Objects -> output.result(json.encodeToJsonElement(observation.objects(request.date).map { it.toDto() })) is OnlineRequest.Profile -> profile(observation, request) is OnlineRequest.Search -> { - streamXLogs(observation, request.query, request.maxPages, request.includeText, output::event) + streamXLogs(observation, request.query, request.maxPages, request.includeText) { output.event(it, timezoneCheck) } 0 } - is OnlineRequest.Capture -> capture(observation, endpoint, request) + is OnlineRequest.Capture -> capture(observation, endpoint, request, timezoneCheck) } } } @@ -60,14 +65,19 @@ class OnlineCommands( return output.result(data, profile.completeness != ProfileCompleteness.TRUNCATED) } - private fun capture(source: ObservationSource, endpoint: Endpoint, request: OnlineRequest.Capture): Int { - val options = CaptureOptions(request.resume, request.maxPages, "${endpoint.host}:${endpoint.port}", request.includeText) + private fun capture(source: ObservationSource, endpoint: Endpoint, request: OnlineRequest.Capture, + timezoneCheck: QueryTimezoneCheck?): Int { return if (request.includeProfiles) { - val manifest = captureEvidence(source, request.query, options, evidence(request.directory)) - output.result(json.encodeToJsonElement(manifest.toDto()), manifest.complete, xlogCapture = true) + val manifest = CaptureCancellation().use { cancellation -> + val options = CaptureOptions(request.resume, request.maxPages, "${endpoint.host}:${endpoint.port}", + request.includeText, request.maxTotalPages, cancellation::cancelled) + captureEvidence(source, request.query, options, evidence(request.directory)) + } + output.result(json.encodeToJsonElement(manifest.toDto()), manifest.complete, xlogCapture = true, timezoneCheck = timezoneCheck) } else { + val options = CaptureOptions(request.resume, request.maxPages, "${endpoint.host}:${endpoint.port}", request.includeText) val manifest = capture(source, request.query, options, captures(request.directory)) - output.result(json.encodeToJsonElement(manifest.toDto()), manifest.executionComplete, xlogCapture = true) + output.result(json.encodeToJsonElement(manifest.toDto()), manifest.executionComplete, xlogCapture = true, timezoneCheck = timezoneCheck) } } } diff --git a/src/main/kotlin/dev/scouterctl/core/AgentDiagnostics.kt b/src/main/kotlin/dev/scouterctl/core/AgentDiagnostics.kt new file mode 100644 index 0000000..ffe5435 --- /dev/null +++ b/src/main/kotlin/dev/scouterctl/core/AgentDiagnostics.kt @@ -0,0 +1,12 @@ +package dev.scouterctl.core + +data class ActiveRequest( + val txid: String, val threadId: String, val state: String, val elapsedMs: Long, + val threadCpuTimeMs: String?, val service: String?, val threadName: String?, +) + +data class ActiveRequests( + val agent: AgentIdentity, val requests: List, val complete: Boolean, val textIncluded: Boolean, +) + +data class AgentThreadDump(val agent: AgentIdentity, val lines: List) diff --git a/src/main/kotlin/dev/scouterctl/core/CollectorTimezone.kt b/src/main/kotlin/dev/scouterctl/core/CollectorTimezone.kt index c93153b..007052e 100644 --- a/src/main/kotlin/dev/scouterctl/core/CollectorTimezone.kt +++ b/src/main/kotlin/dev/scouterctl/core/CollectorTimezone.kt @@ -1,3 +1,5 @@ package dev.scouterctl.core data class CollectorTimezone(val id: String, val utcOffsetSeconds: Int, val referenceTimeEpochMs: Long) + +data class QueryTimezoneCheck(val collectorTimezone: CollectorTimezone?) diff --git a/src/test/kotlin/dev/scouterctl/AgentDiagnosticsCheck.kt b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsCheck.kt new file mode 100644 index 0000000..f6f420b --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsCheck.kt @@ -0,0 +1,90 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.collector.CollectorSource +import dev.scouterctl.bootstrap.BuildInfo +import dev.scouterctl.core.* +import kotlinx.serialization.json.* +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.util.concurrent.TimeUnit + +internal fun verifyAgentDiagnostics(work: Path, endpoint: Endpoint, password: String, objHash: Int, httpPort: Int) { + val cli = AgentCli(work, endpoint.port, password) + val target = listOf("--object-hash", objHash.toString()) + val before = listOf("conf/scouter.conf", "conf/account.xml", "agent.conf").associateWith { Files.readAllBytes(work.resolve(it)) } + val empty = cli.run("active-empty", listOf("agent", "active-requests") + target).single()["data"]!!.jsonObject + check(empty["requests"]!!.jsonArray.isEmpty()) + HttpClient.newHttpClient().use { http -> + fun request(path: String) = HttpRequest.newBuilder(URI("http://127.0.0.1:$httpPort$path")) + .timeout(Duration.ofSeconds(50)).build() + val started = System.currentTimeMillis() + val pending = http.sendAsync(request("/poc/hold"), HttpResponse.BodyHandlers.ofString()) + try { + eventually("instrumented request entered diagnostic hold") { + http.send(request("/diagnostics/hold-state"), HttpResponse.BodyHandlers.ofString()).body() == "waiting" + } + val structural = cli.run("active-structural", listOf("agent", "active-requests") + target).single()["data"]!!.jsonObject + val active = structural["requests"]!!.jsonArray.single().jsonObject + check(active["service"] == JsonNull && active["threadName"] == JsonNull) + check(active["state"] == JsonPrimitive("TIMED_WAITING")) + val txid = active["txid"]!!.jsonPrimitive.content + check(txid.toLong() != 0L && active["threadId"]!!.jsonPrimitive.content.toLong() > 0) + val named = cli.run("active-names", listOf("agent", "active-requests") + target + "--include-text") + .single()["data"]!!.jsonObject["requests"]!!.jsonArray.single().jsonObject + check(named["txid"]!!.jsonPrimitive.content == txid) + check(named["service"]!!.jsonPrimitive.content == "/poc/hold") + check(named["threadName"]!!.jsonPrimitive.content.isNotEmpty()) + val dump = cli.run("thread-dump", listOf("agent", "thread-dump") + target).single()["data"]!!.jsonObject + val lines = dump["lines"]!!.jsonArray.map { it.jsonPrimitive.content } + check(lines.any { it.contains("dev.scouterctl.fixture.TestService.execute") }) + check(lines.any { it.contains("java.util.concurrent.CountDownLatch.await") }) + check(dump["completeness"] == JsonPrimitive("unknown")) + check(http.send(request("/diagnostics/release"), HttpResponse.BodyHandlers.ofString()).statusCode() == 200) + check(pending.get(10, TimeUnit.SECONDS).statusCode() == 200) + val ended = System.currentTimeMillis() + 1 + val date = Instant.ofEpochMilli(started).atZone(ZoneOffset.UTC).toLocalDate().format(DateTimeFormatter.BASIC_ISO_DATE) + val query = Query(date, started, ended, "UTC", objHash, 100) + var stored: XLogRecord? = null + eventually("completed held request stored as the same XLog transaction") { + CollectorSource.connect(endpoint, password, BuildInfo.version).use { source -> + val currentQuery = query.copy(to = System.currentTimeMillis() + 1) + stored = source.page(currentQuery, Cursor()).records.singleOrNull { it.txid == txid } + Files.writeString(work.resolve("diagnostics-xlog-query.json"), buildJsonObject { + put("from", started); put("httpCompletedToExclusive", ended); put("queryToExclusive", currentQuery.to) + put("expectedTxid", txid); put("storedTxid", stored?.txid); put("storedEndTime", stored?.endTime) + }.toString()) + stored != null + } + } + val after = cli.run("active-after", listOf("agent", "active-requests") + target).single()["data"]!!.jsonObject + check(after["requests"]!!.jsonArray.isEmpty()) + cli.run("diagnostics-missing", listOf("agent", "thread-dump", "--object-hash", Int.MIN_VALUE.toString()), 7) + AgentCli(work, endpoint.port, "wrong-$password").run("diagnostics-auth", listOf("agent", "active-requests") + target, 3) + for ((file, bytes) in before) check(Files.readAllBytes(work.resolve(file)).contentEquals(bytes)) + Files.writeString(work.resolve("diagnostics-summary.json"), buildJsonObject { + put("collectorVersion", structural["collector"]!!.jsonObject["version"]!!) + put("agentVersion", structural["agent"]!!.jsonObject["version"]!!) + put("agentJvm", Files.readString(work.resolve("http-port.java-version"))) + put("txid", txid); put("activeRequestMatchedStoredXLog", true) + put("storedEndTime", checkNotNull(stored).endTime) + put("activeBefore", 0); put("activeDuring", 1); put("activeAfter", 0) + put("structuralNamesHidden", true); put("explicitNamesReturned", true) + put("threadDumpContainsHeldRequest", true); put("threadDumpLines", lines.size) + put("threadDumpCompleteness", "unknown"); put("configurationUnchanged", true) + put("missingTargetExit", 7); put("invalidCredentialsExit", 3) + }.toString()) + println("PASS: live active request, stable transaction ID through stored XLog, held-request thread dump, empty snapshots and unchanged configuration") + } finally { + http.send(request("/diagnostics/release"), HttpResponse.BodyHandlers.discarding()) + pending.cancel(true) + } + } +} diff --git a/src/test/kotlin/dev/scouterctl/AgentDiagnosticsDecoderTest.kt b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsDecoderTest.kt new file mode 100644 index 0000000..dfe88cd --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsDecoderTest.kt @@ -0,0 +1,93 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.collector.decodeActiveRequests +import dev.scouterctl.adapters.collector.decodeThreadDump +import dev.scouterctl.core.AgentIdentity +import dev.scouterctl.core.ErrorCode +import dev.scouterctl.core.OperationFailure +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldNotContain +import scouter.lang.pack.MapPack +import scouter.lang.pack.Pack +import scouter.lang.pack.ObjectPack +import scouter.lang.value.DecimalValue +import scouter.lang.value.NullValue +import scouter.lang.value.TextValue +import scouter.util.Hexa32 + +private val diagnosticAgent = AgentIdentity(42, "fixture", "java", true, "2.21.3") + +private fun activeReply() = MapPack().apply { + put("objHash", 42L) + put("complete", true) + newList("id").apply { add(91L); add(0L) } + newList("elapsed").apply { add(12L); add(100L) } + newList("stat").apply { add("RUNNABLE"); add("n/a") } + newList("cpu").apply { add(5L); add(-1L) } + newList("txid").apply { add(Hexa32.toString32(Long.MAX_VALUE)); add("z1") } + newList("service").apply { add("/first"); add(NullValue()) } + newList("name").apply { add("worker"); add(NullValue()) } +} + +class AgentDiagnosticsDecoderTest : FunSpec({ + test("active requests preserve deferred rows, nullable names, unavailable CPU and descending elapsed order") { + val rows = decodeActiveRequests(diagnosticAgent, listOf(activeReply()), true).requests + rows.map { it.txid } shouldBe listOf("-1", Long.MAX_VALUE.toString()) + rows.map { it.threadId } shouldBe listOf("0", "91") + rows.map { it.threadCpuTimeMs } shouldBe listOf(null, "5") + rows.map { it.service } shouldBe listOf(null, "/first") + rows.map { it.threadName } shouldBe listOf(null, "worker") + } + test("active snapshot rejects malformed columns, IDs and identity without leaking values") { + val mutations: List Unit>> = listOf( + "wrong target" to { it.put("objHash", 43L) }, + "text target" to { it.put("objHash", "private-target") }, + "completion type" to { it.put("complete", "private-complete") }, + "missing column" to { it.put("cpu", "private-column") }, + "unequal rows" to { it.getList("id").add(8L) }, + "number type" to { it.getList("id").set(0, TextValue("private-id")) }, + "negative thread" to { it.getList("id").set(0, DecimalValue(-1)) }, + "negative duration" to { it.getList("elapsed").set(0, DecimalValue(-1)) }, + "invalid CPU" to { it.getList("cpu").set(0, DecimalValue(-2)) }, + "unknown state" to { it.getList("stat").set(0, TextValue("private-state")) }, + "state type" to { it.getList("stat").set(0, DecimalValue(1)) }, + "transaction syntax" to { it.getList("txid").set(0, TextValue("private-txid")) }, + "empty transaction" to { it.getList("txid").set(0, TextValue("")) }, + "noncanonical transaction" to { it.getList("txid").set(0, TextValue("x01")) }, + "duplicate transaction" to { it.getList("txid").set(0, TextValue("z1")) }, + "service type" to { it.getList("service").set(0, DecimalValue(1)) }, + "thread name type" to { it.getList("name").set(0, DecimalValue(1)) }, + ) + for ((label, mutate) in mutations) withClue(label) { + val reply = activeReply().also(mutate) + val error = shouldThrow { decodeActiveRequests(diagnosticAgent, listOf(reply), false) } + error.code shouldBe ErrorCode.PROTOCOL_ERROR + error.message.orEmpty() shouldNotContain "private-" + } + } + test("diagnostic response cardinality and empty data distinguish unavailable from malformed") { + val malformed: List> = listOf(listOf(ObjectPack()), listOf(MapPack(), MapPack())) + for (response in malformed) { + shouldThrow { decodeActiveRequests(diagnosticAgent, response, false) }.code shouldBe ErrorCode.PROTOCOL_ERROR + shouldThrow { decodeThreadDump(diagnosticAgent, response) }.code shouldBe ErrorCode.PROTOCOL_ERROR + } + shouldThrow { decodeActiveRequests(diagnosticAgent, emptyList(), false) }.code shouldBe ErrorCode.NOT_FOUND + shouldThrow { decodeThreadDump(diagnosticAgent, emptyList()) }.code shouldBe ErrorCode.NOT_FOUND + shouldThrow { decodeThreadDump(diagnosticAgent, listOf(MapPack().apply { newList("threadDump") })) }.code shouldBe ErrorCode.NOT_FOUND + } + test("thread dumps reject malformed lines and bound decoded content") { + val replies = listOf( + MapPack(), + MapPack().apply { put("error", 7L) }, + MapPack().apply { newList("threadDump").add(7L) }, + MapPack().apply { newList("threadDump").apply { repeat(100001) { add("") } } }, + MapPack().apply { newList("threadDump").add("x".repeat(8_388_609)) }, + ) + for ((index, reply) in replies.withIndex()) withClue("malformed dump $index") { + shouldThrow { decodeThreadDump(diagnosticAgent, listOf(reply)) }.code shouldBe ErrorCode.PROTOCOL_ERROR + } + } +}) diff --git a/src/test/kotlin/dev/scouterctl/AgentDiagnosticsTest.kt b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsTest.kt new file mode 100644 index 0000000..04d90eb --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/AgentDiagnosticsTest.kt @@ -0,0 +1,76 @@ +package dev.scouterctl + +import io.kotest.core.spec.style.FunSpec +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldNotContain +import kotlinx.serialization.json.* + +class AgentDiagnosticsTest : FunSpec({ + test("active request CLI preserves IDs and reveals only opted-in names") { + val cli = ContractCli(tempdir().toPath()) + WireFixture().use { server -> + for (includeText in listOf(false, true)) { + val result = cli.run(server, listOf("agent", "active-requests", "--object-hash", "42") + + if (includeText) listOf("--include-text") else emptyList()) + result.exit shouldBe 0 + val request = result.data()["requests"]!!.jsonArray.single().jsonObject + request["txid"] shouldBe JsonPrimitive(Long.MIN_VALUE.toString()) + request["threadId"] shouldBe JsonPrimitive("9007199254740993") + request["state"] shouldBe JsonPrimitive("WAITING") + request["elapsedMs"] shouldBe JsonPrimitive(1250) + request["threadCpuTimeMs"] shouldBe JsonPrimitive("37") + request["service"] shouldBe if (includeText) JsonPrimitive("/diagnostic/service") else JsonNull + request["threadName"] shouldBe if (includeText) JsonPrimitive("diagnostic-worker") else JsonNull + result.output shouldNotContain "private-" + } + } + } + test("empty active snapshot succeeds while incomplete snapshots exit partial") { + val cli = ContractCli(tempdir().toPath()) + for ((fault, size, exit) in listOf(Triple(Fault.ACTIVE_EMPTY, 0, 0), Triple(Fault.ACTIVE_PARTIAL, 1, 8))) { + WireFixture(fault).use { server -> + val result = cli.run(server, listOf("agent", "active-requests", "--object-hash", "42")) + result.exit shouldBe exit + result.data()["requests"]!!.jsonArray.size shouldBe size + result.events.single()["executionComplete"] shouldBe JsonPrimitive(exit == 0) + } + } + } + test("thread dump CLI emits Agent lines without asserting upstream completeness") { + WireFixture().use { server -> + val result = ContractCli(tempdir().toPath()).run(server, listOf("agent", "thread-dump", "--object-hash", "42")) + result.exit shouldBe 0 + result.data()["lines"]!!.jsonArray.map { it.jsonPrimitive.content } shouldBe listOf( + "\"diagnostic-worker\" #42 WAITING", " at java.lang.Object.wait(Native Method)", + " at example.Worker.await(Worker.java:12)", + ) + result.data()["completeness"] shouldBe JsonPrimitive("unknown") + } + } + test("live diagnostics reject unavailable Agents and do not disclose attachment errors") { + val cli = ContractCli(tempdir().toPath()) + for ((fault, command) in listOf( + Fault.AGENT_OFFLINE to "active-requests", Fault.AGENT_OFFLINE to "thread-dump", + Fault.AGENT_NO_RESPONSE to "active-requests", Fault.AGENT_NO_RESPONSE to "thread-dump", + Fault.DIAGNOSTIC_ERROR to "thread-dump", Fault.EMPTY_OBJECTS to "active-requests", + )) { + WireFixture(fault).use { server -> + val result = cli.run(server, listOf("agent", command, "--object-hash", "42")) + result.exit shouldBe 7 + result.events.single().containsKey("data") shouldBe false + result.output shouldNotContain "private-" + } + } + } + test("diagnostic commands require one nonzero target and reject unrelated options") { + val cli = ContractCli(tempdir().toPath()) + for (args in listOf( + listOf("agent", "active-requests"), listOf("agent", "thread-dump"), + listOf("agent", "active-requests", "--object-hash", "0"), + listOf("agent", "thread-dump", "--object-hash", "0"), + listOf("agent", "active-requests", "--object-hash", "42", "--key", "secret"), + listOf("agent", "thread-dump", "--object-hash", "42", "--include-text"), + )) WireFixture().use { server -> cli.run(server, args).exit shouldBe 2 } + } +}) diff --git a/src/test/kotlin/dev/scouterctl/AgentSmoke.kt b/src/test/kotlin/dev/scouterctl/AgentSmoke.kt index c62d47f..4bea40f 100644 --- a/src/test/kotlin/dev/scouterctl/AgentSmoke.kt +++ b/src/test/kotlin/dev/scouterctl/AgentSmoke.kt @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit fun main(args: Array) { val upstream = Path.of(args.first()).toAbsolutePath().normalize() val hostCounters = args.drop(1) == listOf("--host-counters") + val diagnostics = args.drop(1) == listOf("--diagnostics-only") val work = Files.createTempDirectory("scouterctl-agent-smoke-") Files.createDirectory(work.resolve("conf")) val port = ServerSocket(0).use { it.localPort } @@ -97,12 +98,13 @@ fun main(args: Array) { "-cp", hostJar, "scouter.boot.Boot", home.resolve("lib").toString()) .directory(home.toFile()).redirectErrorStream(true).redirectOutput(work.resolve("host.log").toFile()).start() } - val agent = System.getenv("SCOUTER_AGENT_JAR")?.let(Path::of) + val agentSource = System.getenv("SCOUTER_AGENT_JAR")?.let(Path::of) ?: upstream.resolve("scouter.agent.java/target/scouter.agent.jar") - check(Files.isRegularFile(agent)) { "Build Java 21 agent first: $agent" } + check(Files.isRegularFile(agentSource)) { "Build Java 21 agent first: $agentSource" } + val agent = Files.copy(agentSource, work.resolve("scouter.agent.jar")) val serviceJava = System.getenv("SCOUTER_SERVICE_JAVA") ?: java val serviceClasspath = System.getenv("SCOUTER_SERVICE_CLASSPATH") ?: System.getProperty("java.class.path") - service = ProcessBuilder(serviceJava, "-Xmx128m", "-Duser.timezone=UTC", "-javaagent:$agent", + service = ProcessBuilder(serviceJava, "-Xmx128m", "-Duser.timezone=UTC", "-Djdk.attach.allowAttachSelf=$diagnostics", "-javaagent:$agent", "-Dscouter.config=${work.resolve("agent.conf")}", "-cp", serviceClasspath, "dev.scouterctl.fixture.TestServiceKt", work.resolve("http-port").toString()) .directory(work.toFile()).redirectErrorStream(true).redirectOutput(work.resolve("service.log").toFile()).start() @@ -152,6 +154,10 @@ fun main(args: Array) { verifyAgentInspection(work, endpoint, password, objectHash) return } + if (diagnostics) { + verifyAgentDiagnostics(work, endpoint, password, objectHash, httpPort) + return + } val from = System.currentTimeMillis() val expected = linkedMapOf("/poc/ok" to 18, "/poc/slow" to 6, "/poc/error" to 6, "/poc/sql" to 6) val requests = expected.flatMap { (route, count) -> List(count) { route } }.map { route -> @@ -240,7 +246,7 @@ fun main(args: Array) { } } -private fun eventually(label: String, predicate: () -> Boolean) { +internal fun eventually(label: String, predicate: () -> Boolean) { val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(45) var lastFailure: Exception? = null while (System.nanoTime() < deadline) { diff --git a/src/test/kotlin/dev/scouterctl/CaptureContinuationCheck.kt b/src/test/kotlin/dev/scouterctl/CaptureContinuationCheck.kt new file mode 100644 index 0000000..2ff1aaf --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/CaptureContinuationCheck.kt @@ -0,0 +1,79 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.files.EvidenceStore +import dev.scouterctl.adapters.serialization.json +import dev.scouterctl.application.CaptureOptions +import dev.scouterctl.application.captureEvidence +import dev.scouterctl.core.* +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import java.lang.management.ManagementFactory +import java.lang.management.MemoryType +import java.nio.file.Files +import java.nio.file.Path + +@Serializable +private data class ContinuationSample( + val mode: String, val totalPages: Int, val recordsPerPage: Int, val invocations: Int, + val existingReadBytes: Long, val elapsedNanos: Long, val profileRequests: Int, + val heapPoolPeaks: Map, val heapUsedAfterBytes: Long, +) + +@Serializable +private data class ContinuationReport( + val javaVersion: String, val os: String, val maximumHeapBytes: Long, + val implementationSha256: String, val samples: List, +) + +fun main(args: Array) { + require(args.size == 1) { "Use a new result directory" } + val root = Files.createDirectory(Path.of(args.single()).toAbsolutePath()) + val samples = mutableListOf() + for (pages in listOf(120, 400)) for (continued in listOf(false, true)) { + val directory = root.resolve("pages-$pages-" + if (continued) "continued" else "separate") + val store = EvidenceStore(directory) + val pageSize = 100 + var profiles = 0 + val query = portQuery.copy(to = portQuery.from + 3_600_000, pageSize = pageSize) + val source = MemorySource({ cursor -> + val first = cursor.txid.toLong() + 1 + val last = first + pageSize - 1 + XLogPage((first..last).map { portRow(it) }, Cursor(query.from + last, last.toString()), last < pages.toLong() * pageSize) + }, { id -> + profiles++ + ProfileRecord(id.toString(), List(20) { order -> ProfileStep("sql", order, SqlProfile(123, 1, 0)) }) + }) + var manifest = captureEvidence(source, query, CaptureOptions(false, 40, "memory"), store) + val pools = ManagementFactory.getMemoryPoolMXBeans().filter { it.type == MemoryType.HEAP } + pools.forEach { it.resetPeakUsage() } + var reads = 0L + var elapsed = 0L + var invocations = 0 + while (!manifest.executionComplete) { + val measured = measureResumeReads(directory) { + captureEvidence(source, query, CaptureOptions(true, 40, "memory", + maxTotalPages = if (continued) pages - 40 else null), store) + } + manifest = measured.result + reads += measured.bytes.values.sum() + elapsed += measured.elapsedNanos + invocations++ + Files.move(directory.resolve("reads.jfr"), root.resolve("reads-$pages-$continued-$invocations.jfr")) + } + check(manifest.pages.size == pages && profiles == pages * pageSize) + val peakPools = pools.associate { it.name to it.peakUsage.used } + samples += ContinuationSample(if (continued) "continued" else "separate", pages, pageSize, invocations, + reads, elapsed, profiles, peakPools, ManagementFactory.getMemoryMXBean().heapMemoryUsage.used) + store.validate(manifest) + println("pages=$pages continued=$continued: invocations=$invocations reads=$reads elapsedNanos=$elapsed") + } + for (pair in samples.chunked(2)) { + check(pair[0].existingReadBytes > pair[1].existingReadBytes) + check(pair[1].invocations == 1) + } + val implementation = checkNotNull(EvidenceStore::class.java.getResourceAsStream("/dev/scouterctl/application/EvidenceCaptureKt.class")) + .use { sha256(it.readAllBytes()) } + Files.writeString(root.resolve("report.json"), json.encodeToString(ContinuationReport( + System.getProperty("java.runtime.version"), System.getProperty("os.name"), Runtime.getRuntime().maxMemory(), implementation, samples))) + println("Recorded continuation reads, elapsed time and heap pool peaks in ${root.resolve("report.json")}") +} diff --git a/src/test/kotlin/dev/scouterctl/CaptureContinuationTest.kt b/src/test/kotlin/dev/scouterctl/CaptureContinuationTest.kt new file mode 100644 index 0000000..2388acd --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/CaptureContinuationTest.kt @@ -0,0 +1,123 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.files.EvidenceStore +import dev.scouterctl.application.CaptureOptions +import dev.scouterctl.application.captureEvidence +import dev.scouterctl.core.* +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.engine.spec.tempdir +import io.kotest.matchers.shouldBe +import java.nio.file.Files + +class CaptureContinuationTest : FunSpec({ + test("continued profile capture validates its existing prefix once and retains IDs across the total budget") { + val directory = tempdir().toPath().resolve("capture") + val store = EvidenceStore(directory) + val profiles = mutableListOf() + val source = MemorySource({ cursor -> + val first = cursor.txid.toLong() + 1 + val last = first + 1 + XLogPage(listOf(portRow(first), portRow(last)), Cursor(portQuery.from + last, last.toString()), last < 16) + }, { id -> profiles += id; ProfileRecord(id.toString(), emptyList()) }) + val checkpoint = captureEvidence(source, portQuery, CaptureOptions(false, 2, "memory"), store) + val expectedReads = checkpoint.pages.associate { directory.resolve(it.name) to Files.size(directory.resolve(it.name)) } + + val measured = measureResumeReads(directory) { + captureEvidence(source, portQuery, CaptureOptions(true, 2, "memory", maxTotalPages = 6), store) + } + + measured.bytes shouldBe expectedReads + measured.result.executionComplete shouldBe true + measured.result.pages.size shouldBe 8 + profiles shouldBe (1L..16L).toList() + } + + test("total budget stops after its exact number of new pages and leaves a resumable checkpoint") { + val store = EvidenceStore(tempdir().toPath().resolve("capture")) + val source = MemorySource({ cursor -> + val id = cursor.txid.toLong() + 1 + XLogPage(listOf(portRow(id)), Cursor(portQuery.from + id, id.toString()), id < 5) + }) + + val result = captureEvidence(source, portQuery, CaptureOptions(false, 1, "memory", maxTotalPages = 3), store) + + result.executionComplete shouldBe false + store.load().pages.size shouldBe 3 + store.load().cursor.txid shouldBe "3" + captureEvidence(source, portQuery, CaptureOptions(true, 1, "memory", maxTotalPages = 3), store).executionComplete shouldBe true + } + + test("cancellation during a profile page preserves the previous cursor and resume completes without duplicate records") { + val store = EvidenceStore(tempdir().toPath().resolve("capture")) + var cancelled = false + val source = MemorySource({ cursor -> + val id = cursor.txid.toLong() + 1 + XLogPage(listOf(portRow(id)), Cursor(portQuery.from + id, id.toString()), id < 4) + }, { id -> + if (id == 3L) cancelled = true + ProfileRecord(id.toString(), emptyList()) + }) + + shouldThrow { + captureEvidence(source, portQuery, CaptureOptions(false, 1, "memory", maxTotalPages = 4, cancelled = { cancelled }), store) + }.code shouldBe ErrorCode.PARTIAL_RESULT + + val checkpoint = store.load() + checkpoint.pages.size shouldBe 2 + checkpoint.cursor.txid shouldBe "2" + val resumed = captureEvidence(source, portQuery, CaptureOptions(true, 1, "memory", maxTotalPages = 4), store) + resumed.executionComplete shouldBe true + resumed.pages.flatMapIndexed { index, _ -> store.readPage(resumed, index) }.map { it.xlog.txid } shouldBe listOf("1", "2", "3", "4") + } + + test("a competing profile writer is rejected and the lock is released when the owner finishes") { + val directory = tempdir().toPath().resolve("capture") + val source = MemorySource({ + shouldThrow { + captureEvidence(MemorySource({ error("Competing writer reached the Collector") }), portQuery, + CaptureOptions(true, 1, "memory"), EvidenceStore(directory)) + }.message shouldBe "Profile capture writer lock is unavailable; stop other writers and check write permissions" + XLogPage(listOf(portRow(1)), Cursor(portQuery.from + 1, "1"), false) + }) + + captureEvidence(source, portQuery, CaptureOptions(false, 1, "memory"), EvidenceStore(directory)).executionComplete shouldBe true + captureEvidence(MemorySource({ error("Completed capture fetched a new page") }), portQuery, + CaptureOptions(true, 1, "memory"), EvidenceStore(directory)).executionComplete shouldBe true + } + + test("an externally changed manifest is preserved instead of overwritten at the next checkpoint") { + val directory = tempdir().toPath().resolve("capture") + val source = MemorySource({ + Files.writeString(directory.resolve("manifest.json"), "external change") + XLogPage(listOf(portRow(1)), Cursor(portQuery.from + 1, "1"), false) + }) + + shouldThrow { + captureEvidence(source, portQuery, CaptureOptions(false, 1, "memory", maxTotalPages = 2), EvidenceStore(directory)) + }.code shouldBe ErrorCode.IO_ERROR + + Files.readString(directory.resolve("manifest.json")) shouldBe "external change" + } + + for (budget in listOf(0, 10001)) { + test("invalid total budget $budget is rejected before a capture directory is created") { + val directory = tempdir().toPath().resolve("capture") + shouldThrow { + captureEvidence(MemorySource({ error("Invalid budget queried the Collector") }), portQuery, + CaptureOptions(false, 1, "memory", maxTotalPages = budget), EvidenceStore(directory)) + }.code shouldBe ErrorCode.INVALID_ARGUMENT + Files.exists(directory) shouldBe false + } + } + + test("packaged CLI extends the profile capture budget only when explicitly requested") { + val work = tempdir().toPath() + WireFixture().use { server -> + val result = ContractCli(work).run(server, listOf("capture", "--include-profiles", "--max-pages", "1", + "--max-total-pages", "3", "--directory", work.resolve("capture").toString()) + evidenceWindow) + result.exit shouldBe 0 + EvidenceStore(work.resolve("capture")).load().executionComplete shouldBe true + } + } +}) diff --git a/src/test/kotlin/dev/scouterctl/CollectorTimezoneTest.kt b/src/test/kotlin/dev/scouterctl/CollectorTimezoneTest.kt index ac0a306..f006e4c 100644 --- a/src/test/kotlin/dev/scouterctl/CollectorTimezoneTest.kt +++ b/src/test/kotlin/dev/scouterctl/CollectorTimezoneTest.kt @@ -40,6 +40,11 @@ class CollectorTimezoneTest : FunSpec({ alias.exit shouldBe 0 alias.events.count { it["type"] == JsonPrimitive("record") } shouldBe 3 alias.events.first()["storageDateVerification"] shouldBe JsonPrimitive("unverified") + for (event in listOf(alias.events.first(), alias.events.last())) { + val check = event["timezoneCheck"]!!.jsonObject + check["status"] shouldBe JsonPrimitive("reported-offset-match") + check["collectorTimezone"]!!.jsonObject["id"] shouldBe JsonPrimitive("Asia/Seoul") + } } } @@ -86,6 +91,40 @@ class CollectorTimezoneTest : FunSpec({ search.exit shouldBe 0 search.events.count { it["type"] == JsonPrimitive("record") } shouldBe 3 search.events.first()["storageDateVerification"] shouldBe JsonPrimitive("unverified") + for (event in listOf(search.events.first(), search.events.last())) { + val check = event["timezoneCheck"]!!.jsonObject + check["status"] shouldBe JsonPrimitive("unavailable") + check["collectorTimezone"] shouldBe JsonNull + event["storageDateVerification"] shouldBe JsonPrimitive("unverified") + } + } + } + } + + for (command in listOf("capture", "profile capture", "counter history", "counter export")) { + test("$command reports unavailable timezone verification without changing saved formats") { + val work = tempdir().toPath() + val destination = work.resolve("capture") + WireFixture(Fault.TIMEZONE_MISSING).use { server -> + val window = listOf("--date", "20260101", "--from", "2026-01-01T00:00:00Z", + "--to", "2026-01-01T00:01:00Z", "--timezone", "UTC", "--object-hash", "42") + val args = when (command) { + "capture" -> listOf("capture", "--directory", destination.toString()) + evidenceWindow + "profile capture" -> listOf("capture", "--include-profiles", "--directory", destination.toString()) + evidenceWindow + "counter history" -> listOf("counter", "history", "--counter", "HeapUsed") + window + else -> listOf("counter", "export", "--counter", "HeapUsed", "--directory", destination.toString()) + window.drop(2) + } + val result = ContractCli(work).run(server, args) + result.exit shouldBe 0 + for (event in if (command == "counter history") listOf(result.events.first(), result.events.last()) else result.events) { + event["timezoneCheck"]!!.jsonObject["status"] shouldBe JsonPrimitive("unavailable") + event["storageDateVerification"] shouldBe JsonPrimitive("unverified") + } + if (command != "counter history") { + val manifest = Json.parseToJsonElement(Files.readString(destination.resolve("manifest.json"))).jsonObject + manifest.containsKey("timezoneCheck") shouldBe false + manifest["timezoneVerification"] shouldBe JsonPrimitive("operator-declared") + } } } } diff --git a/src/test/kotlin/dev/scouterctl/MemoryPorts.kt b/src/test/kotlin/dev/scouterctl/MemoryPorts.kt index ede9ba2..4e0b53a 100644 --- a/src/test/kotlin/dev/scouterctl/MemoryPorts.kt +++ b/src/test/kotlin/dev/scouterctl/MemoryPorts.kt @@ -45,6 +45,8 @@ internal class MemorySource( override fun settings(): List = emptyList() override fun agentStatus(objHash: Int): AgentStatus = throw OperationFailure(ErrorCode.NOT_FOUND, "No Agent in memory fixture") override fun agentSettings(objHash: Int): AgentConfiguration = throw OperationFailure(ErrorCode.NOT_FOUND, "No Agent in memory fixture") + override fun activeRequests(objHash: Int, includeText: Boolean): ActiveRequests = throw OperationFailure(ErrorCode.NOT_FOUND, "No Agent in memory fixture") + override fun threadDump(objHash: Int): AgentThreadDump = throw OperationFailure(ErrorCode.NOT_FOUND, "No Agent in memory fixture") override fun counterHistory(query: CounterHistoryQuery): CounterHistory = throw OperationFailure(ErrorCode.NOT_FOUND, "No history in memory fixture") override fun realtimeCounters(objHash: Int): RealtimeCounters = throw OperationFailure(ErrorCode.NOT_FOUND, "No Agent in memory fixture") override fun objects(date: String?): List = emptyList() diff --git a/src/test/kotlin/dev/scouterctl/TimezoneStorageCheck.kt b/src/test/kotlin/dev/scouterctl/TimezoneStorageCheck.kt new file mode 100644 index 0000000..1e4482a --- /dev/null +++ b/src/test/kotlin/dev/scouterctl/TimezoneStorageCheck.kt @@ -0,0 +1,144 @@ +package dev.scouterctl + +import dev.scouterctl.adapters.collector.CollectorSource +import dev.scouterctl.adapters.serialization.json +import dev.scouterctl.bootstrap.BuildInfo +import dev.scouterctl.core.* +import kotlinx.serialization.json.* +import scouter.io.DataOutputX +import scouter.lang.pack.XLogPack +import scouter.net.NetCafe +import scouter.util.CipherUtil +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.InetAddress +import java.net.ServerSocket +import java.nio.file.Files +import java.nio.file.Path +import java.time.Instant +import java.util.UUID +import java.util.concurrent.TimeUnit + +fun main(args: Array) { + require(args.size == 2) { "Expected a built Scouter source directory and a new evidence directory" } + val upstream = Path.of(args[0]).toAbsolutePath().normalize() + val evidence = Path.of(args[1]).toAbsolutePath().normalize() + Files.createDirectory(evidence) + val collectorJava = requireNotNull(System.getenv("SCOUTER_COLLECTOR_JAVA")) + val classpath = listOf(upstream.resolve("scouter.server/target/scouter-server-2.21.3.jar"), + upstream.resolve("scouter.common/target/scouter-common-2.21.3.jar"), + upstream.resolve("scouter.server/target/lib/*")).joinToString(java.io.File.pathSeparator) + val outcomes = listOf("UTC", "America/Chicago").map { zone -> + verifyStorageCalendar(evidence.resolve(zone.replace('/', '-')), zone, collectorJava, classpath) + } + Files.writeString(evidence.resolve("summary.json"), JsonArray(outcomes).toString()) + println("PASS: real Collector UTC control and Chicago summer storage-date divergence; evidence=$evidence") +} + +private fun verifyStorageCalendar(work: Path, zone: String, collectorJava: String, classpath: String): JsonObject { + Files.createDirectories(work.resolve("conf")) + Files.createDirectory(work.resolve("tmp")) + val tcp = ServerSocket(0).use { it.localPort } + val udp = DatagramSocket(0).use { it.localPort } + val password = UUID.randomUUID().toString() + val account = work.resolve("conf/account.xml") + Files.writeString(account, "") + Files.writeString(work.resolve("conf/scouter.conf"), """ + net_tcp_listen_ip=127.0.0.1 + net_tcp_listen_port=$tcp + net_udp_listen_ip=127.0.0.1 + net_udp_listen_port=$udp + net_http_server_enabled=false + net_http_api_enabled=false + server_id=timezone-storage-check + mgr_purge_enabled=false + db_dir=${work.resolve("database")} + log_dir=${work.resolve("logs")} + """.trimIndent()) + val collector = ProcessBuilder(collectorJava, "-XX:-UsePerfData", "-Xmx128m", "-Duser.timezone=$zone", + "-Djava.io.tmpdir=${work.resolve("tmp")}", "-cp", classpath, "scouter.server.Main") + .directory(work.toFile()).redirectErrorStream(true).redirectOutput(work.resolve("collector.log").toFile()).start() + try { + val endpoint = Endpoint("127.0.0.1", tcp, "timezone-check", 2000) + awaitTimezoneCollector("login", collector) { + CollectorSource.connect(endpoint, password, BuildInfo.version).use { it.info.serverId == "timezone-storage-check" } + } + val instant = Instant.parse("2026-07-01T05:30:00Z") + val from = instant.toEpochMilli() + val storageDate = if (zone == "UTC") "20260701" else "20260630" + val storageZone = if (zone == "UTC") "UTC" else "-06:00" + DatagramSocket().use { socket -> + val pack = XLogPack().apply { objHash = 42; txid = 57001; gxid = 57001; endTime = from; service = 10; elapsed = 30 } + val bytes = DataOutputX().apply { write(NetCafe.CAFE); writePack(pack) }.toByteArray() + socket.send(DatagramPacket(bytes, bytes.size, InetAddress.getLoopbackAddress(), udp)) + } + val storedQuery = Query(storageDate, from, from + 1000, storageZone, 42, 10) + awaitTimezoneCollector("stored transaction", collector) { + CollectorSource.connect(endpoint, password, BuildInfo.version).use { + it.page(storedQuery, Cursor()).records.map(XLogRecord::txid) == listOf("57001") + } + } + val stored = CollectorSource.connect(endpoint, password, BuildInfo.version).use { + it.page(storedQuery, Cursor()).records.map(XLogRecord::txid) + } + Files.writeString(work.resolve("stored-partition.json"), buildJsonObject { + put("date", storageDate); put("from", from); put("to", from + 1000) + put("txids", JsonArray(stored.map(::JsonPrimitive))) + }.toString()) + fun cli(label: String, arguments: List, exit: Int): List { + val output = work.resolve("$label.jsonl") + val errors = work.resolve("$label.stderr") + val command = listOf(Path.of(System.getProperty("java.home"), "bin/java").toString(), "-XX:-UsePerfData", + "-Djava.io.tmpdir=${work.resolve("tmp")}", "-jar", System.getProperty("cli.jar")) + arguments + + listOf("--port", tcp.toString(), "--user", "timezone-check", "--no-input") + Files.writeString(work.resolve("$label.command.json"), JsonArray(command.map(::JsonPrimitive)).toString()) + val child = ProcessBuilder(command).directory(work.toFile()).redirectOutput(output.toFile()) + .redirectError(errors.toFile()).apply { + environment()["SCOUTERCTL_PASSWORD"] = password + environment()["SCOUTERCTL_CONFIG"] = work.resolve("config.json").toString() + }.start() + child.outputStream.close() + try { + check(child.waitFor(15, TimeUnit.SECONDS)) { "$label timed out" } + check(child.exitValue() == exit) { "$label: ${Files.readString(output)}" } + check(Files.readString(errors).isEmpty()) + return Files.readAllLines(output).map { json.parseToJsonElement(it).jsonObject } + } finally { if (child.isAlive) child.destroyForcibly().waitFor(5, TimeUnit.SECONDS) } + } + val doctor = cli("doctor", listOf("doctor"), 0).single() + check(doctor["data"]!!.jsonObject["timezone"]!!.jsonObject["id"] == JsonPrimitive(zone)) + val window = listOf("--date", "20260701", "--from", instant.toString(), "--to", instant.plusSeconds(1).toString(), + "--timezone", zone, "--object-hash", "42") + val events = cli("civil-query", listOf("xlog", "search") + window, 0) + val rows = events.count { it["type"] == JsonPrimitive("record") } + check(rows == if (zone == "UTC") 1 else 0) + for (event in listOf(events.first(), events.last())) { + check(event["timezoneCheck"]!!.jsonObject["status"] == JsonPrimitive("reported-offset-match")) + check(event["storageDateVerification"] == JsonPrimitive("unverified")) + } + if (zone != "UTC") { + val failure = cli("mismatch", listOf("xlog", "search") + window.map { if (it == zone) "UTC" else it }, 2).single() + check(failure["error"]!!.jsonObject["code"] == JsonPrimitive("INVALID_ARGUMENT")) + } + return buildJsonObject { + put("collectorZone", zone); put("instant", instant.toString()); put("civilDate", "20260701") + put("storageDate", storageDate); put("storedRecords", stored.size); put("cliRecords", rows) + put("timezoneCheck", "reported-offset-match"); put("storageDateVerification", "unverified") + } + } finally { + collector.destroy() + if (!collector.waitFor(10, TimeUnit.SECONDS)) collector.destroyForcibly().waitFor(5, TimeUnit.SECONDS) + Files.deleteIfExists(account) + Files.writeString(work.resolve("teardown.json"), "{\"collectorStopped\":${!collector.isAlive},\"accountRemoved\":${!Files.exists(account)}}") + check(!collector.isAlive) + } +} + +private fun awaitTimezoneCollector(label: String, collector: Process, ready: () -> Boolean) { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30) + while (System.nanoTime() < deadline && collector.isAlive) { + try { if (ready()) return } catch (_: OperationFailure) { } + Thread.sleep(100) + } + error("Collector $label did not become ready") +} diff --git a/src/test/kotlin/dev/scouterctl/WireFixture.kt b/src/test/kotlin/dev/scouterctl/WireFixture.kt index 7d2e08f..9bbf5a7 100644 --- a/src/test/kotlin/dev/scouterctl/WireFixture.kt +++ b/src/test/kotlin/dev/scouterctl/WireFixture.kt @@ -25,6 +25,7 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit enum class Fault { + ACTIVE_EMPTY, ACTIVE_PARTIAL, DIAGNOSTIC_ERROR, TIMEZONE_SEOUL, TIMEZONE_MISSING, TIMEZONE_INVALID, TIMEZONE_TYPE, TIMEZONE_TIME_TYPE, COUNTER_EXPORT, HISTORY_EMPTY, HISTORY_MALFORMED, HISTORY_TIME_TYPE, HISTORY_OUTSIDE, HISTORY_NAN, HISTORY_TARGET_DUPLICATE, @@ -203,6 +204,40 @@ class WireFixture(private val fault: Fault = Fault.NONE, private val expectedPas } }) } + RequestCmd.OBJECT_ACTIVE_SERVICE_LIST -> { + check(input.readMapPack().getInt("objHash") == 42) + check(fault != Fault.AGENT_OFFLINE) + send(output, MapPack().apply { + put("objHash", 42L) + if (fault != Fault.AGENT_NO_RESPONSE) { + val fields = listOf("id", "elapsed", "stat", "cpu", "txid", "service", "name", "ip", "sql", "login", "desc") + .associateWith { newList(it) } + if (fault != Fault.ACTIVE_EMPTY) { + fields.getValue("id").add(9007199254740993L) + fields.getValue("elapsed").add(1250L) + fields.getValue("stat").add("WAITING") + fields.getValue("cpu").add(37L) + fields.getValue("txid").add(scouter.util.Hexa32.toString32(Long.MIN_VALUE)) + fields.getValue("service").add("/diagnostic/service") + fields.getValue("name").add("diagnostic-worker") + for (key in listOf("ip", "sql", "login", "desc")) fields.getValue(key).add("private-$key") + } + put("complete", fault != Fault.ACTIVE_PARTIAL) + } + }) + } + RequestCmd.OBJECT_THREAD_DUMP -> { + check(input.readMapPack().getInt("objHash") == 42) + check(fault != Fault.AGENT_OFFLINE) + if (fault != Fault.AGENT_NO_RESPONSE) send(output, MapPack().apply { + if (fault == Fault.DIAGNOSTIC_ERROR) put("error", "private-attachment-error") + else newList("threadDump").apply { + add("\"diagnostic-worker\" #42 WAITING") + add(" at java.lang.Object.wait(Native Method)") + add(" at example.Worker.await(Worker.java:12)") + } + }) + } RequestCmd.OBJECT_ENV -> { check(input.readMapPack().getInt("objHash") == 42) check(fault != Fault.AGENT_OFFLINE) diff --git a/src/test/kotlin/dev/scouterctl/fixture/TestService.kt b/src/test/kotlin/dev/scouterctl/fixture/TestService.kt index 2845227..6f21d7e 100644 --- a/src/test/kotlin/dev/scouterctl/fixture/TestService.kt +++ b/src/test/kotlin/dev/scouterctl/fixture/TestService.kt @@ -6,6 +6,8 @@ import java.nio.file.Files import java.nio.file.Paths import java.sql.DriverManager import java.util.concurrent.Executors +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit /** Real HTTP service. Scouter instruments execute/work/JDBC; no telemetry packs are constructed here. */ fun main(args: Array) { @@ -15,6 +17,8 @@ fun main(args: Array) { server.createContext("/") { exchange -> val (status, body) = try { if (exchange.requestURI.path == "/health") 200 to "ready" + else if (exchange.requestURI.path == "/diagnostics/hold-state") 200 to service.holdState() + else if (exchange.requestURI.path == "/diagnostics/release") { service.releaseHold(); 200 to "released" } else 200 to service.execute(exchange.requestURI.path) } catch (_: IllegalStateException) { 500 to "expected test failure" @@ -30,11 +34,21 @@ fun main(args: Array) { } class TestService { + private val holding = CountDownLatch(1) + private val released = CountDownLatch(1) + fun holdState(): String = if (holding.count == 0L) "waiting" else "not-started" + fun releaseHold() { released.countDown() } + fun execute(name: String): String { val result = TestWork().work() return when (name) { "/poc/ok" -> "ok:$result" "/poc/slow" -> { Thread.sleep(180); "slow:$result" } + "/poc/hold" -> { + holding.countDown() + check(released.await(45, TimeUnit.SECONDS)) { "Diagnostic hold timed out" } + "released:$result" + } "/poc/error" -> throw IllegalStateException("fixture-error-marker") "/poc/sql" -> DriverManager.getConnection("jdbc:h2:mem:poc;DB_CLOSE_DELAY=-1").use { connection -> connection.prepareStatement("select ? as marker").use { statement -> From 3f9ab8c8ff9ab7aab6804549663623c51cabe72e Mon Sep 17 00:00:00 2001 From: oxspan <36991763+oxspan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:59:33 +0900 Subject: [PATCH 5/5] =?UTF-8?q?test:=20=E2=9C=85=20Filter=20capture=20page?= =?UTF-8?q?s=20before=20reading=20recovery=20snapshots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/kotlin/dev/scouterctl/SpringRecoveryCheck.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/kotlin/dev/scouterctl/SpringRecoveryCheck.kt b/src/test/kotlin/dev/scouterctl/SpringRecoveryCheck.kt index bc47489..f60c13b 100644 --- a/src/test/kotlin/dev/scouterctl/SpringRecoveryCheck.kt +++ b/src/test/kotlin/dev/scouterctl/SpringRecoveryCheck.kt @@ -114,8 +114,10 @@ internal fun readRecoveryCapture(directory: Path, withProfiles: Boolean): Recove RecoveryCapture(rows, manifest.executionComplete, manifest.cursor) } -internal fun snapshotRecoveryPages(directory: Path): Map = snapshotRecoveryFiles(directory) - .filterKeys { it.endsWith(".jsonl") } +internal fun snapshotRecoveryPages(directory: Path): Map = Files.list(directory).use { files -> + files.filter { it.fileName.toString().endsWith(".jsonl") && Files.isRegularFile(it) }.toList() + .associate { it.fileName.toString() to Files.readString(it) } +} internal fun snapshotRecoveryFiles(directory: Path): Map = Files.list(directory).use { files -> files.filter { Files.isRegularFile(it) }.toList().associate { it.fileName.toString() to Files.readString(it) }