From 1ac7f50274c905bdd13aed1666c33e7a62eb3bba Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Fri, 25 Sep 2026 21:28:59 -0400 Subject: [PATCH 1/4] feat(dppo): make the log-probability clamp configurable DPPO clamps every element's log-probability to [-5, 2] before forming the ratio, as its reference does. With dppo-policy the upper bound never binds (every step's deviation is at least sampling_noise_level; at 0.1 the log-density is at most 1.38). fpo-policy's flow steps are narrower - 0.1 down to about 0.01 at the same level - and 50.7% of its elements land above 2 (up to 93% at the last step), where they carry no gradient and a ratio of exactly 1. The bounds are now logprob_clamp_min/max, defaulting to the reference's, so the change is inert until someone lifts them. Tests written first (5 of 5 failing before): the defaults, the fraction on a fresh fpo-policy at E28's level, a fully clamped batch has zero actor gradient, lifting the bounds restores it, and the flags parse (-inf needs the equals form). --- src/plugrl_server/algorithm/dppo/dppo.py | 5 +- .../algorithm/dppo/dppo_config.py | 11 ++ tests/test_dppo_logprob_clamp.py | 112 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 tests/test_dppo_logprob_clamp.py diff --git a/src/plugrl_server/algorithm/dppo/dppo.py b/src/plugrl_server/algorithm/dppo/dppo.py index 618afd6..3ef9d15 100644 --- a/src/plugrl_server/algorithm/dppo/dppo.py +++ b/src/plugrl_server/algorithm/dppo/dppo.py @@ -230,13 +230,14 @@ def _compute_loss(self, obs, action, oldlogprob, value, advantage, ret): sampling_noise_level=self.config.logprob_noise_level, ) + lo, hi = self.config.logprob_clamp_min, self.config.logprob_clamp_max newlogprob = ( - newlogprob.clamp(min=-5, max=2) + newlogprob.clamp(min=lo, max=hi) .mean(dim=(-1, -2)) .reshape(batch_size, ft_denoising_steps) ) oldlogprob = ( - oldlogprob.clamp(min=-5, max=2) + oldlogprob.clamp(min=lo, max=hi) .mean(dim=(-1, -2)) .reshape(batch_size, ft_denoising_steps) ) diff --git a/src/plugrl_server/algorithm/dppo/dppo_config.py b/src/plugrl_server/algorithm/dppo/dppo_config.py index 84a778d..bf8e156 100644 --- a/src/plugrl_server/algorithm/dppo/dppo_config.py +++ b/src/plugrl_server/algorithm/dppo/dppo_config.py @@ -40,6 +40,17 @@ class DPPOAlgoConfig(BaseAlgoConfig): logprob_noise_level: float = 0.01 sampling_noise_level: float = 0.01 + # Every element's log-probability is clamped to these bounds before the + # ratio is formed - DPPO's reference values. With DPPO's own policy the + # upper bound never binds: each of its denoising steps has a deviation of + # at least `sampling_noise_level`, and a normal with deviation 0.1 has a + # log-density of at most 1.38. A flow policy's later steps are narrower; + # `fpo-policy`'s run down to about 0.01 at level 0.1, and about half its + # elements land above 2, where they carry no gradient and a ratio of 1. + # `inf` lifts a bound; `--algo.logprob-clamp-min=-inf` needs the equals + # sign, or argparse reads `-inf` as a flag. + logprob_clamp_min: float = -5.0 + logprob_clamp_max: float = 2.0 clip_advantage_lower_quantile: float = 0 clip_advantage_upper_quantile: float = 1 n_critic_warmup_itrs: int = 0 diff --git a/tests/test_dppo_logprob_clamp.py b/tests/test_dppo_logprob_clamp.py new file mode 100644 index 0000000..1a33100 --- /dev/null +++ b/tests/test_dppo_logprob_clamp.py @@ -0,0 +1,112 @@ +"""DPPO's log-probability clamp, and what it does to a narrow policy. + +DPPO's loss clamps every element's log-probability to [-5, 2] before forming +the ratio, as DPPO's reference does. With DPPO's own policy, whose every +denoising step has a standard deviation of at least `sampling_noise_level` +(0.1 in the MuJoCo variants), the upper bound never binds: the density of a +normal with deviation 0.1 is at most e^1.38. A flow policy's later steps are +narrower - `fpo-policy`'s run from 0.1 down to about 0.01 at the same level - +and there about half the elements land above 2. A clamped element carries +no gradient and a ratio of exactly 1. + +The bounds are now configuration, defaulting to the reference's. +""" + +from __future__ import annotations + +import math + +import numpy as np +import torch +import tyro + +from plugrl_server.algorithm.dppo.dppo import DPPOAlgorithm +from plugrl_server.algorithm.dppo.dppo_config import DPPOAlgoConfig +from plugrl_server.policy.fpo.fpo_policy import FPOPolicy, FPOPolicyConfig + +B = 64 + + +def _algo(**overrides) -> DPPOAlgorithm: + config = DPPOAlgoConfig( + buffer_size=B, + train_itrs=20, + batch_size=B, + sampling_noise_level=0.1, + logprob_noise_level=0.1, + **overrides, + ) + torch.manual_seed(0) + policy = FPOPolicy(FPOPolicyConfig(obs_dim=11, action_dim=3, device="cpu")) + return DPPOAlgorithm(config, policy) + + +def _policy_gradient(algo: DPPOAlgorithm) -> tuple[float, float]: + """The actor's gradient norm from the policy loss on one minibatch, and the + fraction of log-probability elements above the upper bound.""" + obs = { + "states": { + "obs": np.random.default_rng(0).standard_normal((B, 11)).astype(np.float32) + } + } + with torch.no_grad(): + _, state = algo.policy.get_action_and_runtime_state( + obs, sampling_noise_level=algo.config.sampling_noise_level + ) + model_obs = {"x": state.obs.x, "t": state.obs.t, "cond": state.obs.cond} + advantage = torch.linspace(-1.0, 1.0, B) + zeros = torch.zeros(B) + + algo.policy.actor.zero_grad() + pg_loss, *_ = algo._compute_loss( + model_obs, state.action, state.logprob, zeros, advantage, zeros + ) + pg_loss.backward() + norm = math.sqrt( + sum( + float(p.grad.pow(2).sum()) + for p in algo.policy.actor.parameters() + if p.grad is not None + ) + ) + high = float((state.logprob > algo.config.logprob_clamp_max).float().mean()) + return norm, high + + +def test_the_defaults_are_the_references_bounds(): + config = DPPOAlgoConfig() + assert (config.logprob_clamp_min, config.logprob_clamp_max) == (-5.0, 2.0) + + +def test_the_upper_bound_binds_on_about_half_of_a_flow_policys_elements(): + """The finding itself, at E28's noise level.""" + _, high = _policy_gradient(_algo()) + assert 0.35 < high < 0.65 + + +def test_a_clamped_element_carries_no_gradient(): + """Clamp everything and the policy loss cannot move the actor at all.""" + norm, high = _policy_gradient( + _algo(logprob_clamp_min=-20.0, logprob_clamp_max=-10.0) + ) + assert high > 0.99 + assert norm == 0.0 + + +def test_lifting_the_bounds_restores_the_gradient_the_clamp_removed(): + clamped, _ = _policy_gradient(_algo()) + free, high = _policy_gradient( + _algo(logprob_clamp_min=-math.inf, logprob_clamp_max=math.inf) + ) + assert high == 0.0 + assert free > clamped > 0.0 + + +def test_the_bounds_parse_from_the_command_line(): + """`inf` parses; `-inf` needs the equals form, or argparse reads it as a flag.""" + config = tyro.cli( + DPPOAlgoConfig, + args=["--logprob-clamp-min=-inf", "--logprob-clamp-max", "inf"], + ) + assert config.logprob_clamp_min == -math.inf + assert config.logprob_clamp_max == math.inf From 0ecfd1d0e42671c75295934fa7d35a0ff72854c5 Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Fri, 25 Sep 2026 21:30:43 -0400 Subject: [PATCH 2/4] exp(e29): the clamp-or-noise harness, before the pilot E24's run_cell.sh plus EXTRA_ALGO_ARGS; run.sh runs E28's fpodppo-hopper cell as three arms that differ only in those flags: control, noclamp (--algo.logprob-clamp-max inf, #64) and wide (noise level 1.0 for sampling and log-probability). --- experiments/e29-clamp-or-noise/.gitignore | 3 + experiments/e29-clamp-or-noise/run.sh | 54 ++++++++ experiments/e29-clamp-or-noise/run_cell.sh | 152 +++++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 experiments/e29-clamp-or-noise/.gitignore create mode 100644 experiments/e29-clamp-or-noise/run.sh create mode 100644 experiments/e29-clamp-or-noise/run_cell.sh diff --git a/experiments/e29-clamp-or-noise/.gitignore b/experiments/e29-clamp-or-noise/.gitignore new file mode 100644 index 0000000..76410a3 --- /dev/null +++ b/experiments/e29-clamp-or-noise/.gitignore @@ -0,0 +1,3 @@ +# Checkpoints and tensorboard event files; summarise.py reads out what they hold. +results/*/dppo/ +pilot/*/dppo/ diff --git a/experiments/e29-clamp-or-noise/run.sh b/experiments/e29-clamp-or-noise/run.sh new file mode 100644 index 0000000..9a5e8bd --- /dev/null +++ b/experiments/e29-clamp-or-noise/run.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# E29: fpo-policy under DPPO on Hopper, three arms at once. See PROTOCOL.md. +# +# bash run.sh # the registered run: 100 iterations, seeds 0-2 +# ITERS=11 SEEDS=0 OUT=pilot bash run.sh # a pilot +# +# Every arm is E28's fpodppo-hopper cell - fpo-policy, `dppo hopper`, +# Hopper-v5, buffer 4,096, the variant's minibatch - and differs only in the +# extra algorithm flags below. The server runs this checkout's code through +# PYTHONPATH; the Pythons are the venvs beside it on guangzhao. +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +R="$HERE/${OUT:-results}" +CELL="$HERE/run_cell.sh" +ITERS="${ITERS:-100}" +export SEEDS="${SEEDS:-0 1 2}" +mkdir -p "$R" + +export OMP_NUM_THREADS=1 +export PYTHONPATH="$ROOT/src" +export PLUGRL_ENV_CLIENT="${PLUGRL_ENV_CLIENT:-$ROOT/../plugrl-env-client}" +export PLUGRL_SERVER_PYTHON="${PLUGRL_SERVER_PYTHON:-$ROOT/../plugrl-server/.venv/bin/python}" +export PLUGRL_CLIENT_PYTHON="${PLUGRL_CLIENT_PYTHON:-$PLUGRL_ENV_CLIENT/.venv/bin/python}" +for py in "$PLUGRL_SERVER_PYTHON" "$PLUGRL_CLIENT_PYTHON"; do + [ -x "$py" ] || { echo "error: no Python at $py" >&2; exit 1; } +done + +date '+start %F %T' +echo "code: $(git -C "$ROOT" rev-parse --short HEAD) (src $PYTHONPATH)" +echo "client: $(git -C "$PLUGRL_ENV_CLIENT" rev-parse --short HEAD)" +echo "iters: $ITERS seeds: $SEEDS out: $R" + +arm() { # NAME PORT_BASE EXTRA_ALGO_ARGS + PORT_BASE="$2" EXTRA_ALGO_ARGS="$3" bash "$CELL" "$1" fpo-policy default dppo hopper \ + Hopper-v5 "$ITERS" "$R/$1" > "$R/$1.out" 2>&1 +} + +arm control 9600 "" & +A=$! +sleep 20 +arm noclamp 9610 "--algo.logprob-clamp-max inf" & +B=$! +sleep 20 +arm wide 9620 "--algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0" & +C=$! +wait "$A" "$B" "$C" + +date '+end %F %T' +for a in control noclamp wide; do + printf '%-8s %s\n' "$a" "$(grep -E 'failed seeds' "$R/$a.out" 2>/dev/null)" +done +echo E29_DONE diff --git a/experiments/e29-clamp-or-noise/run_cell.sh b/experiments/e29-clamp-or-noise/run_cell.sh new file mode 100644 index 0000000..3fb82a7 --- /dev/null +++ b/experiments/e29-clamp-or-noise/run_cell.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# One arm of E29 - E24's run_cell.sh with one addition: EXTRA_ALGO_ARGS, word +# split and appended to the algorithm's flags, which is all E29's arms differ +# in. E24's description follows. +# +# One cell of E24's coverage matrix: a policy, an algorithm and a MuJoCo task, +# run on SEEDS concurrently, each seed a server and an env client. +# +# bash run_cell.sh CELL POLICY POLICY_VARIANT ALGO ALGO_VARIANT ENV ITERS [OUT_DIR] +# +# CELL a name for the directory and the checkpoints +# POLICY fpo-policy | dppo-policy +# ALGO fpo | dppo +# ENV a gymnasium MuJoCo task, e.g. Walker2d-v5 +# ITERS iterations of BUFFER entries +# +# Settings are E17-E23's wherever they apply: buffer 4,096, save every 20 +# iterations, server and client seeds matched. What differs by combination: +# +# * fpo-policy takes the task's dimensions on the command line; dppo-policy +# reads them, and its D4RL normalisation, from the config its variant names. +# * FPO counts --algo.global-steps; DPPO derives it from --algo.train-itrs. +# * dppo-policy returns a chunk of 4 actions, and DPPO's own setting executes +# the whole chunk before the next inference, so its client replans every 4 +# steps. fpo-policy's chunk is 1. +# * The client's episode budget is an upper bound large enough never to bind: +# the server ends every run. +# +# BUFFER (default 4096) and BATCH (the algorithm variant's own default when +# unset) come from the environment, so run.sh can set them per cell. +set -uo pipefail + +CELL="$1" +POLICY="$2" +POLICY_VARIANT="$3" +ALGO="$4" +ALGO_VARIANT="$5" +ENV_NAME="$6" +ITERS="$7" +HERE="$(cd "$(dirname "$0")" && pwd)" +OUT="${8:-$HERE/results/$CELL}" +case "$OUT" in /*) ;; *) OUT="$(pwd)/$OUT" ;; esac + +SEEDS="${SEEDS:-0 1 2}" +PORT_BASE="${PORT_BASE:-9300}" +BUFFER="${BUFFER:-4096}" +SERVER_DIR="$(cd "$HERE/../.." && pwd)" +mkdir -p "$OUT" + +case "$ENV_NAME" in + HalfCheetah-v5) DIMS=(17 6) ;; + Hopper-v5) DIMS=(11 3) ;; + Walker2d-v5) DIMS=(17 6) ;; + *) echo "error: no dimensions recorded for $ENV_NAME" >&2; exit 1 ;; +esac + +POLICY_ARGS=() +REPLAN=1 +if [ "$POLICY" = "fpo-policy" ]; then + POLICY_ARGS=(--policy.obs-dim "${DIMS[0]}" --policy.action-dim "${DIMS[1]}") +elif [ "$POLICY" = "dppo-policy" ]; then + REPLAN=4 +fi + +ALGO_ARGS=(--algo.buffer-size "$BUFFER") +if [ -n "${BATCH:-}" ]; then + ALGO_ARGS+=(--algo.batch-size "$BATCH") +fi +if [ "$ALGO" = "fpo" ]; then + ALGO_ARGS+=(--algo.global-steps $((BUFFER * ITERS))) +else + ALGO_ARGS+=(--algo.train-itrs "$ITERS") +fi +if [ -n "${EXTRA_ALGO_ARGS:-}" ]; then + read -r -a EXTRA <<< "$EXTRA_ALGO_ARGS" + ALGO_ARGS+=("${EXTRA[@]}") +fi + +find_client_dir() { + local candidate + for candidate in \ + "${PLUGRL_ENV_CLIENT:-}" \ + "$SERVER_DIR/../plugrl-env-client" \ + "$SERVER_DIR/../../../../plugrl-env-client"; do + [ -n "$candidate" ] && [ -d "$candidate" ] && (cd "$candidate" && pwd) && return + done + return 1 +} +CLIENT_DIR="$(find_client_dir)" || { + echo "error: no plugrl-env-client found. Set PLUGRL_ENV_CLIENT." >&2 + exit 1 +} + +pick_python() { + local repo="$1" override="${2:-}" candidate + if [ -n "$override" ]; then echo "$override"; return; fi + for candidate in "$repo/.venv/Scripts/python.exe" "$repo/.venv/bin/python"; do + [ -x "$candidate" ] && { echo "$candidate"; return; } + done + command -v python3 || command -v python +} +SERVER_PY="$(pick_python "$SERVER_DIR" "${PLUGRL_SERVER_PYTHON:-}")" +CLIENT_PY="$(pick_python "$CLIENT_DIR" "${PLUGRL_CLIENT_PYTHON:-}")" + +echo "cell: $CELL = $POLICY/$POLICY_VARIANT x $ALGO/$ALGO_VARIANT x $ENV_NAME" +echo "iters: $ITERS x $BUFFER batch: ${BATCH:-variant default} replan: $REPLAN seeds: $SEEDS" +echo "extra: ${EXTRA_ALGO_ARGS:-none}" +echo "out: $OUT" +date '+start %F %T' + +run_seed() { + local seed="$1" port=$((PORT_BASE + $1)) + (cd "$SERVER_DIR" && "$SERVER_PY" -m plugrl_server.cli \ + "$POLICY" "$POLICY_VARIANT" "$ALGO" "$ALGO_VARIANT" \ + --port "$port" --seed "$seed" --policy.device cpu \ + "${POLICY_ARGS[@]}" "${ALGO_ARGS[@]}" \ + --algo.save-interval 20 \ + --no-show-progress-bar --no-show-metric-table \ + --checkpoint-base-dir "$OUT" --exp-name "$CELL-seed$seed" --overwrite \ + > "$OUT/server-seed$seed.log" 2>&1 &) + + local waited=0 + until grep -q "is listening on" "$OUT/server-seed$seed.log" 2>/dev/null; do + sleep 1 + waited=$((waited + 1)) + if [ "$waited" -ge 300 ]; then + echo "seed $seed: server never listened; see $OUT/server-seed$seed.log" >&2 + return 1 + fi + done + + (cd "$CLIENT_DIR" && "$CLIENT_PY" -m plugrl_env_client.cli mujoco-v1 \ + --server-port "$port" --server-host 127.0.0.1 \ + --num-envs 1 --num-episodes $((BUFFER * ITERS * 4)) \ + --env.name "$ENV_NAME" \ + --runner.replan-steps "$REPLAN" --runner.seed "$seed" \ + > "$OUT/client-seed$seed.log" 2>&1) + echo "seed $seed finished rc=$? at $(date '+%T')" +} + +PIDS=() +for seed in $SEEDS; do + run_seed "$seed" & + PIDS+=($!) + sleep 5 +done +FAILED=0 +for pid in "${PIDS[@]}"; do wait "$pid" || FAILED=$((FAILED + 1)); done + +date '+end %F %T' +echo "failed seeds: $FAILED" +echo "CELL_DONE $CELL" From 20484bf76170a446376230b56c8692dca19fd386 Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Fri, 25 Sep 2026 21:36:58 -0400 Subject: [PATCH 3/4] exp: pre-register E29 - does the logprob clamp or the narrow noise stop fpo-policy under DPPO Three arms of E28's fpodppo-hopper: control, noclamp (upper bound lifted, #64), wide (noise level 1.0). V1: logged entropy -1.920 / +0.383. P2 the control stays within +50; P3 noclamp and P4 wide gain >= +100 on 2 of 3, both expected falsified after the pilot. Pilot, the clamp probe and the actor movement of E28's cells recorded before any registered run. --- experiments/e29-clamp-or-noise/PROTOCOL.md | 154 +++++++++++ experiments/e29-clamp-or-noise/clamp_probe.py | 45 ++++ .../e29-clamp-or-noise/results/pilot.txt | 30 +++ experiments/e29-clamp-or-noise/summarise.py | 253 ++++++++++++++++++ 4 files changed, 482 insertions(+) create mode 100644 experiments/e29-clamp-or-noise/PROTOCOL.md create mode 100644 experiments/e29-clamp-or-noise/clamp_probe.py create mode 100644 experiments/e29-clamp-or-noise/results/pilot.txt create mode 100644 experiments/e29-clamp-or-noise/summarise.py diff --git a/experiments/e29-clamp-or-noise/PROTOCOL.md b/experiments/e29-clamp-or-noise/PROTOCOL.md new file mode 100644 index 0000000..d874de0 --- /dev/null +++ b/experiments/e29-clamp-or-noise/PROTOCOL.md @@ -0,0 +1,154 @@ +# E29 measurement protocol (pre-registered) + +**Written 2026-09-25, after the pilot in `results/pilot.txt` and before the +registered run.** + +This file must not be edited after the first registered data point. Anything +learned afterwards goes in `AMENDMENT.md`, dated. + +--- + +## The question + +`fpo-policy` learns HalfCheetah, Hopper and Walker2d under FPO (E6, E16, E23, +E24) and none of them under DPPO (E17, E18, E28), where DPPO's own +`dppo-policy` improves all three (E28). By the project's rule that is a +defect of ours. Reading the code for where the two policies meet DPPO's loss +differently found two concrete differences: + +1. **The clamp.** DPPO clamps every element's log-probability to [-5, 2] + before forming the ratio. At the MuJoCo variants' noise level, 0.1, the + upper bound binds on **50.7%** of `fpo-policy`'s elements - none at its + first three steps, 93% at its last - and on none of `dppo-policy`'s. A + clamped element carries no gradient and a ratio of exactly 1. #64 makes + the bounds configurable. +2. **The noise.** `dppo-policy` floors every denoising step's deviation at + the level; `fpo-policy`'s flow steps run from 0.1 down to about 0.01. Per + element, the logged entropy is −1.920 for `fpo-policy` and 0.312 for + `dppo-policy` - about a tenth of the deviation. + +**Which of these stops `fpo-policy` learning under DPPO - either, both, or +neither?** + +### What this cannot settle + +* Any cause beyond these two. If neither arm learns, the remaining + differences - network, chunk, number of denoising steps, output scale - + are the next question, not this one's. +* Other tasks: E29 runs Hopper only, where E28's contrast is sharpest + (`dppo-policy` +260 to +417, `fpo-policy` +5 to +11). + +--- + +## Declared in advance: what was already known + +1. **E28's `fpodppo-hopper`**, this experiment's cell unchanged: gain (mean of + iterations 91-100 minus the first) +10.6, +4.9, +9.7; `approx_kl` around + 2.5 × 10⁻⁷ over the first ten iterations; entropy −1.920. +2. **The probe** (`clamp_probe.py`, `results/pilot.txt`): the clamp's reach at + level 0.1 as above; at level 1.0, **no** `fpo-policy` element above 2 + and 0.19% below −5 (`dppo-policy` at 0.1: 0.17%), entropy 0.383. +3. **How far each actor moved in E28**, iteration 20 to 100, relative + distance of the actor's weights, seeds 0 / 1 / 2: `fpo-policy` **22.5, + 20.0, 21.7%**, `dppo-policy` 11.9, 12.0, 11.6%. `fpo-policy`'s weights do move - further + than `dppo-policy`'s - while its action distribution barely changes. +4. **The pilot** (`results/pilot.txt`), eleven iterations, seed 0: every arm + ran; each server logged the configuration intended; entropies −1.9198, + −1.9198, +0.3828. Against `control`, `noclamp` raised the mean + `approx_kl` 2.4-fold (5.9 × 10⁻⁷) and the actor's gradient norm 1.5-fold; + `wide` lowered `approx_kl` tenfold (2.7 × 10⁻⁸) - at an entropy close to + `dppo-policy`'s, whose `approx_kl` is about 10⁻⁵. No arm's return moved + in eleven iterations. + +--- + +## Design + +| arm | extra algorithm flags | the clamp | the noise | +| --- | --- | --- | --- | +| `control` | none | binds on 50.7% | level 0.1 | +| `noclamp` | `--algo.logprob-clamp-max inf` | lifted | level 0.1 | +| `wide` | `--algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0` | cannot bind | level 1.0, entropy ≈ `dppo-policy`'s | + +Every arm is E28's `fpodppo-hopper` cell: `fpo-policy`, `dppo hopper`, +Hopper-v5, buffer 4,096, the variant's minibatch of 2,048, 100 iterations, +seeds 0, 1, 2, checkpoints every 20 iterations. `run_cell.sh` is E24's with +one addition, `EXTRA_ALGO_ARGS`. `wide` raises the level for both sampling +and the log-probability, so the ratio is taken under the distribution the +actions were drawn from. Only the upper bound is lifted in `noclamp`: it is +the one that binds. + +`wide` changes the noise and, as a consequence, removes the clamp's reach +(known item 2); `noclamp` removes the clamp's reach alone. The design cannot +add noise while keeping the clamp's reach, so "the noise alone" is read +from the two together, below. + +Machine `guangzhao`, CPU, one thread per process, the three arms at once. +Code: `main` plus #64, run from this branch's worktree through `PYTHONPATH`; +client plugrl-env-client at 931ab56. + +--- + +## Checks + +* **V1 - the manipulation took.** Each arm's logged `losses/entropy`, which + depends only on the noise schedule, is within 0.005 of **−1.920** + (`control`, `noclamp`) and **−1.920 + ln 10 = +0.383** (`wide`) at every + iteration. An arm that fails V1 is not read. + +--- + +## Predictions, and what falsifies each + +The gain is the mean return of iterations 91-100 minus the first +iteration's. + +**P1 - every arm runs end to end**: 100 iterations logged, the checkpoint at +100, no traceback, client exit 0, on every seed. + +**P2 - the control reproduces E28**: `control`'s gain is at most **+50** on +every seed. + +> E28: +5 to +11. Falsified: the control moved, and nothing below is read. + +**P3 - lifting the clamp alone lets `fpo-policy` learn**: `noclamp`'s gain is +at least **+100** on at least 2 of 3 seeds. + +**P4 - ten times the noise lets it learn**: `wide`'s gain is at least **+100** +on at least 2 of 3 seeds. + +> +100 is about nine times the control's largest gain in E28 and under half +> of `dppo-policy`'s smallest (+260). + +**What I expect, after the pilot: both P3 and P4 falsified.** In eleven +iterations `noclamp` changed the update little and `wide` made the ratio +move less, not more. The hypotheses are registered as the claims they would +support, so that a falsification rules each one out. + +**Reading P3 and P4 together:** + +| P3 `noclamp` | P4 `wide` | reading | +| --- | --- | --- | +| holds | holds | the clamp is enough to stop it; the noise need not change | +| holds | falsified | the clamp stops it, and ten times the noise hurts in some other way | +| falsified | holds | the clamp alone is not it; the narrow noise is - with or without the clamp | +| falsified | falsified | neither: both are ruled out as sufficient causes | + +**Reported, not predicted:** `approx_kl` and `clipfrac` over the first and +last ten iterations; the actor's relative movement from iteration 20 to 100, +beside E28's; each run's return and episode length. + +--- + +## Declared deviations allowed in advance + +1. One restart of any run that dies for a reason outside the experiment, + recorded in `AMENDMENT.md`. +2. A failure caused by how an arm was launched is fixed and the arm rerun + once, recorded in `AMENDMENT.md`. + +--- + +## Reading order + +P1, V1, P2, then P3 and P4 and the table, then the reported figures. diff --git a/experiments/e29-clamp-or-noise/clamp_probe.py b/experiments/e29-clamp-or-noise/clamp_probe.py new file mode 100644 index 0000000..17bb178 --- /dev/null +++ b/experiments/e29-clamp-or-noise/clamp_probe.py @@ -0,0 +1,45 @@ +"""How often does DPPO's logprob clamp [-5, 2] bind, per denoising step? + +Builds each policy fresh (random initialisation, as every E28 cell starts), +samples a batch of actions with the Hopper variant's sampling_noise_level, +and counts the per-element logprobs the loss would clamp. A clamped element +contributes no gradient and a ratio of exactly 1. +""" + +import sys + +import numpy as np +import torch + +from plugrl_server.policy.dppo.dppo_policy import DPPOPolicy, DPPOPolicyConfigHopper +from plugrl_server.policy.fpo.fpo_policy import FPOPolicy, FPOPolicyConfig + +LEVEL = float(sys.argv[1]) if len(sys.argv) > 1 else 0.1 +B = 4096 +torch.manual_seed(0) +obs = { + "states": { + "obs": np.random.default_rng(0).standard_normal((B, 11)).astype(np.float32) + } +} + +policies = { + "fpo-policy": FPOPolicy(FPOPolicyConfig(obs_dim=11, action_dim=3, device="cpu")), + "dppo-policy": DPPOPolicy(DPPOPolicyConfigHopper(device="cpu")), +} +print(f"sampling_noise_level {LEVEL}; fraction of logprob elements outside [-5, 2]") +for name, policy in policies.items(): + with torch.no_grad(): + _, state = policy.get_action_and_runtime_state(obs, sampling_noise_level=LEVEL) + lp = state.logprob # [B, steps, horizon, dim] + high = (lp > 2).float().mean(dim=(0, 2, 3)) + low = (lp < -5).float().mean(dim=(0, 2, 3)) + t = state.obs.t[0] + print( + f"{name}: {lp.shape[1]} steps, overall above 2: {float((lp > 2).float().mean()):.3f}, " + f"below -5: {float((lp < -5).float().mean()):.4f}, mean entropy {float(state.entropy.mean()):.3f}" + ) + for i in range(lp.shape[1]): + print( + f" step {i:2d} t={float(t[i]):6.3f} above 2: {float(high[i]):.3f} below -5: {float(low[i]):.4f}" + ) diff --git a/experiments/e29-clamp-or-noise/results/pilot.txt b/experiments/e29-clamp-or-noise/results/pilot.txt new file mode 100644 index 0000000..fd8cd57 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/pilot.txt @@ -0,0 +1,30 @@ +# E29 pilot, 2026-09-25 21:30-21:33, guangzhao, before PROTOCOL.md was written. +# run.sh with ITERS=11 SEEDS=0 OUT=pilot: the three arms at once, seed 0, +# 11 iterations (DPPO's scheduler refuses fewer than its 10-iteration warmup). +# Code 0ecfd1d (main + #64), client 931ab56. + +All three arms: seed 0 finished rc=0, no traceback in the server log. + +Configuration, as each server logged it: + control logprob_clamp_max=2.0 sampling_noise_level=0.1 logprob_noise_level=0.1 + noclamp logprob_clamp_max=inf sampling_noise_level=0.1 logprob_noise_level=0.1 + wide logprob_clamp_max=2.0 sampling_noise_level=1.0 logprob_noise_level=1.0 + +Tensorboard, iterations 1-11 (entropy min..max; the rest means): + + arm entropy approx_kl clipfrac actor grad norm return + control -1.9198..-1.9198 2.5e-07 4.4e-07 2.5e-03 14.9, 17.3, 14.3, 13.7, 18.0, 14.8, 14.4, 16.1, 18.1, 13.9, 16.3 + noclamp -1.9198..-1.9198 5.9e-07 3.6e-06 3.8e-03 14.9, 17.5, 14.7, 13.7, 16.7, 14.4, 14.6, 16.3, 17.5, 14.9, 15.5 + wide 0.3828.. 0.3828 2.7e-08 0.0 2.6e-03 16.7, 16.0, 17.0, 16.7, 16.1, 18.5, 17.7, 15.4, 16.5, 16.6, 18.5 + +control's first return is E28's fpodppo-hopper seed 0's, 14.9. The logged +entropies are the probe's (-1.920 at level 0.1; +0.383 at 1.0, against +-1.920 + ln 10 = +0.3826). + +Before the pilot, a probe of freshly initialised policies (clamp_probe.py, +4,096 Hopper-sized samples, run with `PYTHONPATH=src python clamp_probe.py LEVEL`): + + level 0.1 fpo-policy above 2: 50.7% (0, 0, 0, 36.9, 59.1, 70.3, 77.4, 82.9, 88.1, 92.7% by step) + below -5: 0.02% entropy -1.920 + dppo-policy above 2: 0% below -5: 0.17% entropy 0.312 + level 1.0 fpo-policy above 2: 0% below -5: 0.19% entropy 0.383 diff --git a/experiments/e29-clamp-or-noise/summarise.py b/experiments/e29-clamp-or-noise/summarise.py new file mode 100644 index 0000000..ede6509 --- /dev/null +++ b/experiments/e29-clamp-or-noise/summarise.py @@ -0,0 +1,253 @@ +"""Read E29's verdicts from each arm's logs and tensorboards. + + python summarise.py [results-dir] + +P1, then V1, then P2, P3, P4 - PROTOCOL.md's reading order - then the +update diagnostics. The gain is the mean return of iterations 91-100 minus +the first iteration's. Writes `summary.tsv`, one row per arm and seed. +""" + +import datetime +import math +import pathlib +import re +import sys + +from tensorboard.backend.event_processing.event_accumulator import EventAccumulator + +HERE = pathlib.Path(__file__).resolve().parent +RESULTS = pathlib.Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else HERE / "results" +SEEDS = (0, 1, 2) +ITERS = 100 +ARMS = ("control", "noclamp", "wide") +# Every element's entropy depends only on the noise schedule, not the weights: +# E28 logged -1.920 at level 0.1, and scaling the level by k adds ln k. +ENTROPY = {"control": -1.920, "noclamp": -1.920, "wide": -1.920 + math.log(10.0)} +ENTROPY_TOL = 0.005 +CONTROL_MAX = 50.0 +LEARNS = 100.0 + + +def run_dir(arm: str, seed: int) -> pathlib.Path | None: + hits = sorted((RESULTS / arm).glob(f"*/*/{arm}-seed{seed}")) + return hits[0] if hits else None + + +def curve(run: pathlib.Path | None, tag: str) -> list[float]: + if run is None: + return [] + events = sorted((run / "tensorboard").glob("events.*")) + if not events: + return [] + acc = EventAccumulator(str(events[0]), size_guidance={"scalars": 0}) + acc.Reload() + if tag not in acc.Tags()["scalars"]: + return [] + return [e.value for e in acc.Scalars(tag)] + + +def wall_minutes(out: str) -> float | None: + stamps = dict(re.findall(r"^(start|end)\s+(\S+ \S+)$", out, flags=re.M)) + if "start" not in stamps or "end" not in stamps: + return None + fmt = "%Y-%m-%d %H:%M:%S" + delta = datetime.datetime.strptime(stamps["end"], fmt) - datetime.datetime.strptime( + stamps["start"], fmt + ) + return delta.total_seconds() / 60 + + +def mean(xs: list[float]) -> float: + return sum(xs) / len(xs) if xs else float("nan") + + +def actor_movement(run: pathlib.Path | None) -> float: + """Relative distance of the actor's weights, first checkpoint to last + (iterations 20 and 100): how far eighty iterations of DPPO moved it.""" + from safetensors import safe_open + + saves = ( + sorted( + (p for p in run.iterdir() if p.name.isdigit()), key=lambda p: int(p.name) + ) + if run + else [] + ) + if len(saves) < 2: + return float("nan") + num = den = 0.0 + with ( + safe_open(str(saves[0] / "model.safetensors"), "pt") as a, + safe_open(str(saves[-1] / "model.safetensors"), "pt") as b, + ): + for key in a.keys(): + if key.startswith("actor."): + x, y = a.get_tensor(key).double(), b.get_tensor(key).double() + num += float((y - x).pow(2).sum()) + den += float(x.pow(2).sum()) + return math.sqrt(num / den) if den else float("nan") + + +def main() -> int: + data = {} + print("P1 every arm runs end to end") + runs_ok = {} + for arm in ARMS: + out_path = RESULTS / f"{arm}.out" + out = ( + out_path.read_text(encoding="utf-8", errors="replace") + if out_path.exists() + else "" + ) + exits = { + int(s): int(rc) for s, rc in re.findall(r"seed (\d) finished rc=(\d+)", out) + } + ok_arm = True + for seed in SEEDS: + run = run_dir(arm, seed) + reward = curve(run, "rollout/reward") + saves = [p for p in run.iterdir() if p.name.isdigit()] if run else [] + log_path = RESULTS / arm / f"server-seed{seed}.log" + log = ( + log_path.read_text(encoding="utf-8", errors="replace") + if log_path.exists() + else "" + ) + ok = ( + len(reward) >= ITERS + and len(saves) >= ITERS // 20 + and bool(log) + and "Traceback" not in log + and exits.get(seed) == 0 + ) + ok_arm &= ok + gain = ( + mean(reward[ITERS - 10 : ITERS]) - reward[0] + if len(reward) >= ITERS + else float("nan") + ) + data[arm, seed] = dict( + ok=ok, + reward=reward, + gain=gain, + entropy=curve(run, "losses/entropy"), + kl=curve(run, "losses/approx_kl"), + clipfrac=curve(run, "losses/clipfrac"), + length=curve(run, "rollout/length"), + ) + runs_ok[arm] = ok_arm + minutes = wall_minutes(out) + print( + f" {arm:8s} {'HOLDS' if ok_arm else 'FAILS'}" + + (f" ({minutes:.0f} min)" if minutes is not None else "") + ) + + print("\nV1 the manipulation: each arm's logged entropy, every iteration") + v1 = {} + for arm in ARMS: + values = [v for s in SEEDS for v in data[arm, s]["entropy"]] + v1[arm] = bool(values) and all( + abs(v - ENTROPY[arm]) <= ENTROPY_TOL for v in values + ) + span = f"{min(values):.4f} to {max(values):.4f}" if values else "not logged" + print( + f" {arm:8s} expected {ENTROPY[arm]:+.4f}, logged {span} " + f"{'PASS' if v1[arm] else 'FAIL'}" + ) + + def gains(arm: str) -> list[float]: + return [data[arm, s]["gain"] for s in SEEDS] + + def show(arm: str) -> str: + return ", ".join(f"{g:+.1f}" for g in gains(arm)) + + def readable(arm: str) -> bool: + return runs_ok[arm] and v1[arm] + + print( + f"\nP2 the control reproduces E28: gain at most +{CONTROL_MAX:.0f} on every seed" + ) + print(f" control {show('control')}") + p2 = readable("control") and all(g <= CONTROL_MAX for g in gains("control")) + print( + f" {'HOLDS' if p2 else 'FALSIFIED or not readable - nothing below is read'}" + ) + + for name, arm, text in ( + ("P3", "noclamp", "lifting the clamp alone lets it learn"), + ("P4", "wide", "ten times the noise lets it learn"), + ): + print(f"\n{name} {text}: gain at least +{LEARNS:.0f} on 2 of 3") + print(f" {arm:8s} {show(arm)}") + if not p2: + print(" not read - P2 does not hold") + elif not readable(arm): + print( + " NOT READ - the arm did not run, or its manipulation did not take" + ) + else: + passed = sum(g >= LEARNS for g in gains(arm)) + print(f" {passed} of 3 {'HOLDS' if passed >= 2 else 'FALSIFIED'}") + + print("\nreported: update diagnostics, mean over iterations 1-10 and 91-100") + for arm in ARMS: + for tag in ("kl", "clipfrac"): + first = mean([v for s in SEEDS for v in data[arm, s][tag][:10]]) + last = mean( + [v for s in SEEDS for v in data[arm, s][tag][ITERS - 10 : ITERS]] + ) + print(f" {arm:8s} {tag:9s} {first:.2e} -> {last:.2e}") + + print("\nreported: the actor's relative movement, iteration 20 to 100, per seed") + for arm in ARMS: + moves = [actor_movement(run_dir(arm, s)) for s in SEEDS] + print(f" {arm:8s} {', '.join(f'{100 * m:.2f}%' for m in moves)}") + + print( + "\nreported: first iteration and mean of the last ten, return and episode length" + ) + lines = [ + "arm\tseed\titerations\truns\tfirst_return\tlast10_return\tgain\tfirst_length\tlast10_length" + ] + for arm in ARMS: + for s in SEEDS: + d = data[arm, s] + reward, length = d["reward"], d["length"] + first = reward[0] if reward else float("nan") + last10 = ( + mean(reward[ITERS - 10 : ITERS]) + if len(reward) >= ITERS + else float("nan") + ) + first_len = length[0] if length else float("nan") + last10_len = ( + mean(length[ITERS - 10 : ITERS]) + if len(length) >= ITERS + else float("nan") + ) + print( + f" {arm:8s} seed {s} return {first:7.1f} -> {last10:7.1f} " + f"length {first_len:6.1f} -> {last10_len:6.1f}" + ) + lines.append( + "\t".join( + [ + arm, + str(s), + str(len(reward)), + str(d["ok"]).lower(), + f"{first:.1f}", + f"{last10:.1f}", + f"{d['gain']:.1f}", + f"{first_len:.1f}", + f"{last10_len:.1f}", + ] + ) + ) + (HERE / "summary.tsv").write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"\nwrote {HERE / 'summary.tsv'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 6c08227b799a0b084dcf8c0a8c15d8afd630cb51 Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Sun, 27 Sep 2026 01:31:36 -0400 Subject: [PATCH 4/4] exp(e29): results - the clamp is ruled out; ten times the noise helps but misses the bar P1 3/3, V1 pass, P2 holds (control +10.6/+4.5/+9.7, E28's to a point). P3 falsified: lifting the clamp gives +7.0/+3.0/+7.8. P4 falsified by its rule: ten times the noise gives +73.6/+84.6/+106.2, one seed past +100, seven to nineteen times the control and still climbing. The pilot's approx_kl was misread: across noise levels it scales as 1/sigma^2. --- experiments/e29-clamp-or-noise/FINDINGS.md | 106 ++++++++++++++++++ .../e29-clamp-or-noise/results/control.out | 11 ++ .../results/control/client-seed0.log | 49 ++++++++ .../results/control/client-seed1.log | 50 +++++++++ .../results/control/client-seed2.log | 49 ++++++++ .../results/control/server-seed0.log | 25 +++++ .../results/control/server-seed1.log | 25 +++++ .../results/control/server-seed2.log | 25 +++++ .../e29-clamp-or-noise/results/noclamp.out | 11 ++ .../results/noclamp/client-seed0.log | 49 ++++++++ .../results/noclamp/client-seed1.log | 50 +++++++++ .../results/noclamp/client-seed2.log | 49 ++++++++ .../results/noclamp/server-seed0.log | 25 +++++ .../results/noclamp/server-seed1.log | 25 +++++ .../results/noclamp/server-seed2.log | 25 +++++ .../e29-clamp-or-noise/results/run.out | 9 ++ .../e29-clamp-or-noise/results/verdicts.txt | 47 ++++++++ .../e29-clamp-or-noise/results/wide.out | 11 ++ .../results/wide/client-seed0.log | 49 ++++++++ .../results/wide/client-seed1.log | 49 ++++++++ .../results/wide/client-seed2.log | 49 ++++++++ .../results/wide/server-seed0.log | 25 +++++ .../results/wide/server-seed1.log | 25 +++++ .../results/wide/server-seed2.log | 25 +++++ experiments/e29-clamp-or-noise/summary.tsv | 10 ++ 25 files changed, 873 insertions(+) create mode 100644 experiments/e29-clamp-or-noise/FINDINGS.md create mode 100644 experiments/e29-clamp-or-noise/results/control.out create mode 100644 experiments/e29-clamp-or-noise/results/control/client-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/control/client-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/control/client-seed2.log create mode 100644 experiments/e29-clamp-or-noise/results/control/server-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/control/server-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/control/server-seed2.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp.out create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/client-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/client-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/client-seed2.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/server-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/server-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/noclamp/server-seed2.log create mode 100644 experiments/e29-clamp-or-noise/results/run.out create mode 100644 experiments/e29-clamp-or-noise/results/verdicts.txt create mode 100644 experiments/e29-clamp-or-noise/results/wide.out create mode 100644 experiments/e29-clamp-or-noise/results/wide/client-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/wide/client-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/wide/client-seed2.log create mode 100644 experiments/e29-clamp-or-noise/results/wide/server-seed0.log create mode 100644 experiments/e29-clamp-or-noise/results/wide/server-seed1.log create mode 100644 experiments/e29-clamp-or-noise/results/wide/server-seed2.log create mode 100644 experiments/e29-clamp-or-noise/summary.tsv diff --git a/experiments/e29-clamp-or-noise/FINDINGS.md b/experiments/e29-clamp-or-noise/FINDINGS.md new file mode 100644 index 0000000..5567240 --- /dev/null +++ b/experiments/e29-clamp-or-noise/FINDINGS.md @@ -0,0 +1,106 @@ +# E29: the log-probability clamp is not why `fpo-policy` stands still under DPPO; its narrow noise is part of it + +2026-09-25 · Linux workstation (`guangzhao`), CPU only · `fpo-policy` · DPPO · +Hopper-v5, three arms, three seeds, 100 iterations · protocol: +[`PROTOCOL.md`](PROTOCOL.md) (`20484bf`, after the pilot in +[`results/pilot.txt`](results/pilot.txt)) + +--- + +## The result + +`fpo-policy` learns Hopper under FPO and not under DPPO (E28), where DPPO's +own policy improves it by +260 to +417. Reading the code found two places +where the two policies meet DPPO's loss differently: the clamp of every +log-probability to [-5, 2], which binds on half of `fpo-policy`'s elements +and none of `dppo-policy`'s, and the per-step noise, about a tenth of +`dppo-policy`'s. E29 changed each, one arm apiece, on E28's cell. + +| arm | change | gain, seeds 0 / 1 / 2 | verdict | +| --- | --- | --- | --- | +| `control` | none | +10.6 / +4.5 / +9.7 | P2 holds: E28 was +10.6 / +4.9 / +9.7 | +| `noclamp` | upper bound lifted | **+7.0 / +3.0 / +7.8** | **P3 falsified**: 0 of 3 past +100 | +| `wide` | noise level ×10 | **+73.6 / +84.6 / +106.2** | **P4 falsified**: 1 of 3 past +100 | + +(Gain: mean return of iterations 91-100 minus the first iteration's.) + +* **P1 holds**, 3 of 3 arms, 9 of 9 seeds; 20 minutes each. +* **V1 passes**: the logged entropy is −1.9198 in `control` and `noclamp` + and +0.3828 in `wide`, at every iteration - the expected −1.920 and + −1.920 + ln 10 = +0.383. + +**The clamp is ruled out.** Lifting it changes nothing measurable: the gains +sit inside the control's, and the actor moves as much as the control's +(15-20% from iteration 20 to 100, against 20-23%). + +**The noise is part of it, and not all of it.** By the registered bar - +100 +on two seeds of three - P4 is falsified, and the protocol's reading for two +falsifications, "neither is sufficient", is what is claimed. Described +beside it: every `wide` seed gained seven to nineteen times its control +seed, and every one ended at least 60 above its control seed's best +iteration. +Mean return per ten iterations: + +| arm, seed | 1-10 | 21-30 | 41-50 | 61-70 | 81-90 | 91-100 | +| --- | --- | --- | --- | --- | --- | --- | +| `wide` 0 | 17 | 19 | 25 | 56 | 77 | 90 | +| `wide` 1 | 21 | 23 | 30 | 54 | 89 | 106 | +| `wide` 2 | 19 | 22 | 28 | 52 | 97 | 126 | +| `control` 0-2 | 16-18 | 17-19 | 19 | 19-22 | 22-24 | 24-27 | + +`wide` barely moves for forty iterations, then climbs, and is still climbing +at the end; episodes grew from about 23 steps to 58-76. At an entropy about +`dppo-policy`'s, it gains about a quarter of what `dppo-policy` gained in +E28. + +--- + +## What I expected, and what the pilot misled me about + +After the pilot I registered both as expected to fail. By the rule both did; +about `wide` I was wrong in substance. In eleven iterations `wide` had made +`approx_kl` ten times *smaller* than the control's, and I read that as the +ratio moving less. It is not comparable across noise levels: the KL between +two normals for the same shift of the mean scales as 1/σ², so ten times the +deviation divides it by a hundred for the same change. The actor moved +**40-43%** in `wide` against the control's 20-23%. + +--- + +## What this does and does not support + +**Supported:** + +* The log-probability clamp does not stop `fpo-policy` learning under DPPO. + #64 keeps it configurable and its default the reference's; E29 gives no + reason to change that default. +* Ten times the flow policy's noise, for sampling and for the ratio alike, + lets DPPO move `fpo-policy`'s return on Hopper by +74 to +106 in 409,600 + steps, where the unchanged policy moves +5 to +11. + +**Not supported:** + +* That `fpo-policy` learns Hopper under DPPO at any noise level: the bar was + not met, and one level was tried. +* That the noise is the whole difference from `dppo-policy`. At matched + entropy the gap is still about fourfold. The two policies also differ in + network (`fpo-policy`'s four hidden layers of 32 against DPPO's own), + action chunk (1 against 4), denoising steps (10 against 20) and + `fpo-policy`'s output scale of 0.25; none is tested here. +* Anything about how the noise helps - through exploration, through the + variance of the policy gradient, or through the ratio. E29 varies the + level once and does not separate them. + +--- + +## Reproducing + +```bash +bash run.sh # three arms at once; about 20 minutes on 24 cores +python summarise.py # P1, V1, P2-P4, the diagnostics; writes summary.tsv +PYTHONPATH=src python clamp_probe.py 0.1 # the clamp's reach on fresh policies, per step +``` + +`results/verdicts.txt` is `summarise.py`'s output. The pilot's own logs stay +on `guangzhao`; what they showed is in `results/pilot.txt`. Checkpoints and +tensorboard files stay there too (`.gitignore`). diff --git a/experiments/e29-clamp-or-noise/results/control.out b/experiments/e29-clamp-or-noise/results/control.out new file mode 100644 index 0000000..8a36852 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control.out @@ -0,0 +1,11 @@ +cell: control = fpo-policy/default x dppo/hopper x Hopper-v5 +iters: 100 x 4096 batch: variant default replan: 1 seeds: 0 1 2 +extra: none +out: /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control +start 2026-09-25 21:36:26 +seed 0 finished rc=0 at 21:56:23 +seed 1 finished rc=0 at 21:56:35 +seed 2 finished rc=0 at 21:56:36 +end 2026-09-25 21:56:36 +failed seeds: 0 +CELL_DONE control diff --git a/experiments/e29-clamp-or-noise/results/control/client-seed0.log b/experiments/e29-clamp-or-noise/results/control/client-seed0.log new file mode 100644 index 0000000..a066983 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/client-seed0.log @@ -0,0 +1,49 @@ +2026-09-25 21:36:28.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:28.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:28.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:28.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:28.342 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:28.378 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213628-9c3d0bcf output_dir=runs/mujoco-v1-nenv1-20260925-213628-9c3d0bcf recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:28.411 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9600... +2026-09-25 21:36:28.413 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:36:58.415 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10107 infer_calls=10107 feedback_calls=10107 infer_wait=27.288s infer_obs_pack=0.140s env_step=1.365s feedback_total=0.695s feedback_obs_pack=0.180s feedback_info_pack=0.036s effective_fps=342.75 +2026-09-25 21:37:28.417 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20205 infer_calls=20205 feedback_calls=20205 infer_wait=54.363s infer_obs_pack=0.312s env_step=2.847s feedback_total=1.445s feedback_obs_pack=0.381s feedback_info_pack=0.079s effective_fps=342.64 +2026-09-25 21:37:58.418 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30388 infer_calls=30388 feedback_calls=30388 infer_wait=81.222s infer_obs_pack=0.472s env_step=4.455s feedback_total=2.252s feedback_obs_pack=0.596s feedback_info_pack=0.124s effective_fps=343.75 +2026-09-25 21:38:28.418 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40589 infer_calls=40589 feedback_calls=40589 infer_wait=108.034s infer_obs_pack=0.636s env_step=6.080s feedback_total=3.074s feedback_obs_pack=0.822s feedback_info_pack=0.171s effective_fps=344.49 +2026-09-25 21:38:58.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50866 infer_calls=50866 feedback_calls=50866 infer_wait=134.846s infer_obs_pack=0.800s env_step=7.701s feedback_total=3.899s feedback_obs_pack=1.039s feedback_info_pack=0.216s effective_fps=345.45 +2026-09-25 21:39:28.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61172 infer_calls=61172 feedback_calls=61172 infer_wait=161.684s infer_obs_pack=0.961s env_step=9.309s feedback_total=4.722s feedback_obs_pack=1.257s feedback_info_pack=0.264s effective_fps=346.24 +2026-09-25 21:39:58.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71497 infer_calls=71497 feedback_calls=71497 infer_wait=188.434s infer_obs_pack=1.130s env_step=10.975s feedback_total=5.555s feedback_obs_pack=1.480s feedback_info_pack=0.309s effective_fps=346.91 +2026-09-25 21:40:28.423 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81831 infer_calls=81831 feedback_calls=81831 infer_wait=215.186s infer_obs_pack=1.299s env_step=12.631s feedback_total=6.401s feedback_obs_pack=1.707s feedback_info_pack=0.356s effective_fps=347.45 +2026-09-25 21:40:58.423 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=91957 infer_calls=91957 feedback_calls=91957 infer_wait=242.061s infer_obs_pack=1.462s env_step=14.234s feedback_total=7.208s feedback_obs_pack=1.925s feedback_info_pack=0.400s effective_fps=347.05 +2026-09-25 21:41:28.839 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=102402 infer_calls=102402 feedback_calls=102402 infer_wait=269.235s infer_obs_pack=1.627s env_step=15.886s feedback_total=8.048s feedback_obs_pack=2.145s feedback_info_pack=0.446s effective_fps=347.37 +2026-09-25 21:41:58.842 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112792 infer_calls=112792 feedback_calls=112792 infer_wait=296.015s infer_obs_pack=1.797s env_step=17.544s feedback_total=8.878s feedback_obs_pack=2.363s feedback_info_pack=0.492s effective_fps=347.87 +2026-09-25 21:42:28.845 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122914 infer_calls=122914 feedback_calls=122914 infer_wait=322.829s infer_obs_pack=1.968s env_step=19.173s feedback_total=9.694s feedback_obs_pack=2.582s feedback_info_pack=0.537s effective_fps=347.54 +2026-09-25 21:42:58.847 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=133376 infer_calls=133376 feedback_calls=133376 infer_wait=349.569s infer_obs_pack=2.137s env_step=20.841s feedback_total=10.528s feedback_obs_pack=2.805s feedback_info_pack=0.583s effective_fps=348.17 +2026-09-25 21:43:28.848 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=143597 infer_calls=143597 feedback_calls=143597 infer_wait=376.366s infer_obs_pack=2.302s env_step=22.488s feedback_total=11.344s feedback_obs_pack=3.027s feedback_info_pack=0.630s effective_fps=348.11 +2026-09-25 21:43:58.849 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=154019 infer_calls=154019 feedback_calls=154019 infer_wait=403.096s infer_obs_pack=2.470s env_step=24.181s feedback_total=12.172s feedback_obs_pack=3.254s feedback_info_pack=0.677s effective_fps=348.52 +2026-09-25 21:44:28.852 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=164370 infer_calls=164370 feedback_calls=164370 infer_wait=429.869s infer_obs_pack=2.637s env_step=25.840s feedback_total=13.002s feedback_obs_pack=3.473s feedback_info_pack=0.723s effective_fps=348.72 +2026-09-25 21:44:58.852 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=174826 infer_calls=174826 feedback_calls=174826 infer_wait=456.579s infer_obs_pack=2.809s env_step=27.533s feedback_total=13.834s feedback_obs_pack=3.697s feedback_info_pack=0.771s effective_fps=349.12 +2026-09-25 21:45:28.858 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=185089 infer_calls=185089 feedback_calls=185089 infer_wait=483.390s infer_obs_pack=2.978s env_step=29.179s feedback_total=14.647s feedback_obs_pack=3.917s feedback_info_pack=0.816s effective_fps=349.10 +2026-09-25 21:45:58.861 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=195377 infer_calls=195377 feedback_calls=195377 infer_wait=510.126s infer_obs_pack=3.144s env_step=30.853s feedback_total=15.495s feedback_obs_pack=4.143s feedback_info_pack=0.866s effective_fps=349.13 +2026-09-25 21:46:28.863 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=205617 infer_calls=205617 feedback_calls=205617 infer_wait=536.943s infer_obs_pack=3.308s env_step=32.480s feedback_total=16.324s feedback_obs_pack=4.362s feedback_info_pack=0.913s effective_fps=349.06 +2026-09-25 21:46:58.864 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=216001 infer_calls=216001 feedback_calls=216001 infer_wait=563.655s infer_obs_pack=3.477s env_step=34.176s feedback_total=17.160s feedback_obs_pack=4.589s feedback_info_pack=0.960s effective_fps=349.25 +2026-09-25 21:47:28.864 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=226142 infer_calls=226142 feedback_calls=226142 infer_wait=590.499s infer_obs_pack=3.639s env_step=35.804s feedback_total=17.968s feedback_obs_pack=4.809s feedback_info_pack=1.009s effective_fps=349.03 +2026-09-25 21:47:58.866 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=236476 infer_calls=236476 feedback_calls=236476 infer_wait=617.273s infer_obs_pack=3.805s env_step=37.462s feedback_total=18.796s feedback_obs_pack=5.030s feedback_info_pack=1.057s effective_fps=349.13 +2026-09-25 21:48:28.867 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=246674 infer_calls=246674 feedback_calls=246674 infer_wait=644.120s infer_obs_pack=3.969s env_step=39.085s feedback_total=19.601s feedback_obs_pack=5.241s feedback_info_pack=1.102s effective_fps=349.01 +2026-09-25 21:48:58.869 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=257070 infer_calls=257070 feedback_calls=257070 infer_wait=670.873s infer_obs_pack=4.142s env_step=40.750s feedback_total=20.429s feedback_obs_pack=5.465s feedback_info_pack=1.151s effective_fps=349.19 +2026-09-25 21:49:28.871 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=267263 infer_calls=267263 feedback_calls=267263 infer_wait=697.671s infer_obs_pack=4.307s env_step=42.397s feedback_total=21.243s feedback_obs_pack=5.690s feedback_info_pack=1.197s effective_fps=349.08 +2026-09-25 21:49:58.872 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=277705 infer_calls=277705 feedback_calls=277705 infer_wait=724.416s infer_obs_pack=4.476s env_step=44.072s feedback_total=22.080s feedback_obs_pack=5.912s feedback_info_pack=1.243s effective_fps=349.30 +2026-09-25 21:50:28.874 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=288048 infer_calls=288048 feedback_calls=288048 infer_wait=751.199s infer_obs_pack=4.640s env_step=45.716s feedback_total=22.918s feedback_obs_pack=6.134s feedback_info_pack=1.289s effective_fps=349.37 +2026-09-25 21:50:58.875 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=298496 infer_calls=298496 feedback_calls=298496 infer_wait=777.944s infer_obs_pack=4.806s env_step=47.387s feedback_total=23.755s feedback_obs_pack=6.361s feedback_info_pack=1.336s effective_fps=349.57 +2026-09-25 21:51:28.877 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=308709 infer_calls=308709 feedback_calls=308709 infer_wait=804.706s infer_obs_pack=4.979s env_step=49.045s feedback_total=24.585s feedback_obs_pack=6.581s feedback_info_pack=1.383s effective_fps=349.49 +2026-09-25 21:51:58.880 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=319040 infer_calls=319040 feedback_calls=319040 infer_wait=831.420s infer_obs_pack=5.152s env_step=50.747s feedback_total=25.424s feedback_obs_pack=6.809s feedback_info_pack=1.429s effective_fps=349.54 +2026-09-25 21:52:28.880 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=329269 infer_calls=329269 feedback_calls=329269 infer_wait=858.221s infer_obs_pack=5.315s env_step=52.403s feedback_total=26.244s feedback_obs_pack=7.034s feedback_info_pack=1.475s effective_fps=349.47 +2026-09-25 21:52:58.882 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=339627 infer_calls=339627 feedback_calls=339627 infer_wait=884.992s infer_obs_pack=5.488s env_step=54.051s feedback_total=27.077s feedback_obs_pack=7.259s feedback_info_pack=1.521s effective_fps=349.55 +2026-09-25 21:53:28.882 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=349790 infer_calls=349790 feedback_calls=349790 infer_wait=911.844s infer_obs_pack=5.656s env_step=55.661s feedback_total=27.896s feedback_obs_pack=7.483s feedback_info_pack=1.570s effective_fps=349.42 +2026-09-25 21:53:58.884 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=360202 infer_calls=360202 feedback_calls=360202 infer_wait=938.610s infer_obs_pack=5.821s env_step=57.321s feedback_total=28.739s feedback_obs_pack=7.698s feedback_info_pack=1.619s effective_fps=349.54 +2026-09-25 21:54:28.885 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=370381 infer_calls=370381 feedback_calls=370381 infer_wait=965.464s infer_obs_pack=5.985s env_step=58.943s feedback_total=29.543s feedback_obs_pack=7.916s feedback_info_pack=1.664s effective_fps=349.44 +2026-09-25 21:54:58.885 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=380832 infer_calls=380832 feedback_calls=380832 infer_wait=992.205s infer_obs_pack=6.150s env_step=60.618s feedback_total=30.388s feedback_obs_pack=8.137s feedback_info_pack=1.710s effective_fps=349.59 +2026-09-25 21:55:28.886 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=390948 infer_calls=390948 feedback_calls=390948 infer_wait=1019.024s infer_obs_pack=6.324s env_step=62.258s feedback_total=31.194s feedback_obs_pack=8.364s feedback_info_pack=1.756s effective_fps=349.44 +2026-09-25 21:55:58.887 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=401332 infer_calls=401332 feedback_calls=401332 infer_wait=1045.779s infer_obs_pack=6.489s env_step=63.932s feedback_total=32.030s feedback_obs_pack=8.584s feedback_info_pack=1.802s effective_fps=349.52 +2026-09-25 21:56:23.516 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1067.296s infer_obs_pack=6.623s env_step=65.286s feedback_total=32.690s feedback_obs_pack=8.764s feedback_info_pack=1.839s effective_fps=349.52 +2026-09-25 21:56:23.517 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 17616/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/control/client-seed1.log b/experiments/e29-clamp-or-noise/results/control/client-seed1.log new file mode 100644 index 0000000..de3fa08 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/client-seed1.log @@ -0,0 +1,50 @@ +2026-09-25 21:36:33.340 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:33.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:33.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:33.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:33.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:33.379 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213633-e48e08a1 output_dir=runs/mujoco-v1-nenv1-20260925-213633-e48e08a1 recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:33.412 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9601... +2026-09-25 21:36:33.414 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:03.418 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=9902 infer_calls=9902 feedback_calls=9902 infer_wait=27.272s infer_obs_pack=0.141s env_step=1.384s feedback_total=0.696s feedback_obs_pack=0.193s feedback_info_pack=0.037s effective_fps=335.74 +2026-09-25 21:37:33.418 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=19969 infer_calls=19969 feedback_calls=19969 infer_wait=54.292s infer_obs_pack=0.308s env_step=2.898s feedback_total=1.458s feedback_obs_pack=0.399s feedback_info_pack=0.077s effective_fps=338.71 +2026-09-25 21:38:03.419 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30115 infer_calls=30115 feedback_calls=30115 infer_wait=81.175s infer_obs_pack=0.470s env_step=4.475s feedback_total=2.278s feedback_obs_pack=0.620s feedback_info_pack=0.124s effective_fps=340.67 +2026-09-25 21:38:33.422 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40362 infer_calls=40362 feedback_calls=40362 infer_wait=107.951s infer_obs_pack=0.640s env_step=6.123s feedback_total=3.114s feedback_obs_pack=0.854s feedback_info_pack=0.171s effective_fps=342.55 +2026-09-25 21:39:03.423 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50485 infer_calls=50485 feedback_calls=50485 infer_wait=134.781s infer_obs_pack=0.802s env_step=7.752s feedback_total=3.918s feedback_obs_pack=1.075s feedback_info_pack=0.218s effective_fps=342.84 +2026-09-25 21:39:33.424 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=60874 infer_calls=60874 feedback_calls=60874 infer_wait=161.569s infer_obs_pack=0.973s env_step=9.382s feedback_total=4.752s feedback_obs_pack=1.295s feedback_info_pack=0.266s effective_fps=344.55 +2026-09-25 21:40:03.424 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71112 infer_calls=71112 feedback_calls=71112 infer_wait=188.412s infer_obs_pack=1.138s env_step=10.986s feedback_total=5.582s feedback_obs_pack=1.513s feedback_info_pack=0.314s effective_fps=345.01 +2026-09-25 21:40:33.427 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81382 infer_calls=81382 feedback_calls=81382 infer_wait=215.222s infer_obs_pack=1.305s env_step=12.612s feedback_total=6.411s feedback_obs_pack=1.737s feedback_info_pack=0.359s effective_fps=345.50 +2026-09-25 21:41:03.429 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=91377 infer_calls=91377 feedback_calls=91377 infer_wait=242.084s infer_obs_pack=1.470s env_step=14.227s feedback_total=7.212s feedback_obs_pack=1.956s feedback_info_pack=0.405s effective_fps=344.83 +2026-09-25 21:41:33.432 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=101662 infer_calls=101662 feedback_calls=101662 infer_wait=268.866s infer_obs_pack=1.639s env_step=15.875s feedback_total=8.042s feedback_obs_pack=2.183s feedback_info_pack=0.453s effective_fps=345.29 +2026-09-25 21:42:03.435 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=111790 infer_calls=111790 feedback_calls=111790 infer_wait=295.760s infer_obs_pack=1.809s env_step=17.464s feedback_total=8.846s feedback_obs_pack=2.400s feedback_info_pack=0.499s effective_fps=345.16 +2026-09-25 21:42:33.438 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122176 infer_calls=122176 feedback_calls=122176 infer_wait=322.467s infer_obs_pack=1.981s env_step=19.160s feedback_total=9.688s feedback_obs_pack=2.637s feedback_info_pack=0.548s effective_fps=345.82 +2026-09-25 21:43:03.441 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=132468 infer_calls=132468 feedback_calls=132468 infer_wait=349.269s infer_obs_pack=2.148s env_step=20.785s feedback_total=10.525s feedback_obs_pack=2.856s feedback_info_pack=0.598s effective_fps=346.12 +2026-09-25 21:43:33.442 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=142875 infer_calls=142875 feedback_calls=142875 infer_wait=376.050s infer_obs_pack=2.314s env_step=22.431s feedback_total=11.355s feedback_obs_pack=3.076s feedback_info_pack=0.645s effective_fps=346.66 +2026-09-25 21:44:03.443 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=152961 infer_calls=152961 feedback_calls=152961 infer_wait=402.868s infer_obs_pack=2.482s env_step=24.059s feedback_total=12.176s feedback_obs_pack=3.301s feedback_info_pack=0.693s effective_fps=346.39 +2026-09-25 21:44:33.447 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=163279 infer_calls=163279 feedback_calls=163279 infer_wait=429.600s infer_obs_pack=2.652s env_step=25.738s feedback_total=13.011s feedback_obs_pack=3.528s feedback_info_pack=0.740s effective_fps=346.66 +2026-09-25 21:45:03.447 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=173472 infer_calls=173472 feedback_calls=173472 infer_wait=456.386s infer_obs_pack=2.819s env_step=27.390s feedback_total=13.836s feedback_obs_pack=3.753s feedback_info_pack=0.787s effective_fps=346.65 +2026-09-25 21:45:33.449 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=183845 infer_calls=183845 feedback_calls=183845 infer_wait=483.109s infer_obs_pack=2.989s env_step=29.075s feedback_total=14.669s feedback_obs_pack=3.984s feedback_info_pack=0.834s effective_fps=346.98 +2026-09-25 21:46:03.450 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=193927 infer_calls=193927 feedback_calls=193927 infer_wait=509.932s infer_obs_pack=3.156s env_step=30.719s feedback_total=15.472s feedback_obs_pack=4.202s feedback_info_pack=0.878s effective_fps=346.74 +2026-09-25 21:46:33.453 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=204347 infer_calls=204347 feedback_calls=204347 infer_wait=536.619s infer_obs_pack=3.328s env_step=32.417s feedback_total=16.333s feedback_obs_pack=4.433s feedback_info_pack=0.927s effective_fps=347.12 +2026-09-25 21:47:03.456 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=214584 infer_calls=214584 feedback_calls=214584 infer_wait=563.473s infer_obs_pack=3.491s env_step=34.042s feedback_total=17.137s feedback_obs_pack=4.653s feedback_info_pack=0.975s effective_fps=347.14 +2026-09-25 21:47:33.457 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=224795 infer_calls=224795 feedback_calls=224795 infer_wait=590.298s infer_obs_pack=3.656s env_step=35.671s feedback_total=17.956s feedback_obs_pack=4.876s feedback_info_pack=1.022s effective_fps=347.13 +2026-09-25 21:48:03.457 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=234880 infer_calls=234880 feedback_calls=234880 infer_wait=617.138s infer_obs_pack=3.817s env_step=37.298s feedback_total=18.765s feedback_obs_pack=5.104s feedback_info_pack=1.069s effective_fps=346.93 +2026-09-25 21:48:33.460 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=245210 infer_calls=245210 feedback_calls=245210 infer_wait=643.850s infer_obs_pack=3.984s env_step=38.992s feedback_total=19.610s feedback_obs_pack=5.335s feedback_info_pack=1.119s effective_fps=347.11 +2026-09-25 21:49:03.462 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=255396 infer_calls=255396 feedback_calls=255396 infer_wait=670.644s infer_obs_pack=4.149s env_step=40.638s feedback_total=20.435s feedback_obs_pack=5.560s feedback_info_pack=1.168s effective_fps=347.07 +2026-09-25 21:49:33.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=265755 infer_calls=265755 feedback_calls=265755 infer_wait=697.391s infer_obs_pack=4.319s env_step=42.307s feedback_total=21.274s feedback_obs_pack=5.787s feedback_info_pack=1.215s effective_fps=347.26 +2026-09-25 21:50:03.467 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=275972 infer_calls=275972 feedback_calls=275972 infer_wait=724.182s infer_obs_pack=4.486s env_step=43.947s feedback_total=22.114s feedback_obs_pack=6.007s feedback_info_pack=1.262s effective_fps=347.25 +2026-09-25 21:50:33.470 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=286473 infer_calls=286473 feedback_calls=286473 infer_wait=750.851s infer_obs_pack=4.661s env_step=45.656s feedback_total=22.985s feedback_obs_pack=6.246s feedback_info_pack=1.310s effective_fps=347.60 +2026-09-25 21:51:03.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=296630 infer_calls=296630 feedback_calls=296630 infer_wait=777.644s infer_obs_pack=4.834s env_step=47.299s feedback_total=23.814s feedback_obs_pack=6.466s feedback_info_pack=1.357s effective_fps=347.51 +2026-09-25 21:51:33.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=306850 infer_calls=306850 feedback_calls=306850 infer_wait=804.362s infer_obs_pack=5.005s env_step=48.990s feedback_total=24.654s feedback_obs_pack=6.695s feedback_info_pack=1.409s effective_fps=347.50 +2026-09-25 21:52:03.472 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=317007 infer_calls=317007 feedback_calls=317007 infer_wait=831.109s infer_obs_pack=5.183s env_step=50.658s feedback_total=25.487s feedback_obs_pack=6.916s feedback_info_pack=1.456s effective_fps=347.43 +2026-09-25 21:52:33.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=327381 infer_calls=327381 feedback_calls=327381 infer_wait=857.848s infer_obs_pack=5.359s env_step=52.336s feedback_total=26.312s feedback_obs_pack=7.141s feedback_info_pack=1.504s effective_fps=347.59 +2026-09-25 21:53:03.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=337503 infer_calls=337503 feedback_calls=337503 infer_wait=884.715s infer_obs_pack=5.522s env_step=53.942s feedback_total=27.118s feedback_obs_pack=7.359s feedback_info_pack=1.549s effective_fps=347.48 +2026-09-25 21:53:33.476 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=347780 infer_calls=347780 feedback_calls=347780 infer_wait=911.515s infer_obs_pack=5.692s env_step=55.571s feedback_total=27.958s feedback_obs_pack=7.582s feedback_info_pack=1.599s effective_fps=347.52 +2026-09-25 21:54:03.477 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=358087 infer_calls=358087 feedback_calls=358087 infer_wait=938.278s infer_obs_pack=5.858s env_step=57.235s feedback_total=28.801s feedback_obs_pack=7.808s feedback_info_pack=1.647s effective_fps=347.60 +2026-09-25 21:54:33.479 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=368438 infer_calls=368438 feedback_calls=368438 infer_wait=965.001s infer_obs_pack=6.031s env_step=58.918s feedback_total=29.648s feedback_obs_pack=8.039s feedback_info_pack=1.696s effective_fps=347.71 +2026-09-25 21:55:03.480 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=378464 infer_calls=378464 feedback_calls=378464 infer_wait=991.810s infer_obs_pack=6.198s env_step=60.559s feedback_total=30.470s feedback_obs_pack=8.263s feedback_info_pack=1.740s effective_fps=347.52 +2026-09-25 21:55:33.481 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=388693 infer_calls=388693 feedback_calls=388693 infer_wait=1018.574s infer_obs_pack=6.366s env_step=62.232s feedback_total=31.297s feedback_obs_pack=8.487s feedback_info_pack=1.787s effective_fps=347.52 +2026-09-25 21:56:03.485 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=398723 infer_calls=398723 feedback_calls=398723 infer_wait=1045.416s infer_obs_pack=6.532s env_step=63.870s feedback_total=32.100s feedback_obs_pack=8.706s feedback_info_pack=1.836s effective_fps=347.34 +2026-09-25 21:56:33.487 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=409105 infer_calls=409105 feedback_calls=409105 infer_wait=1072.195s infer_obs_pack=6.705s env_step=65.515s feedback_total=32.936s feedback_obs_pack=8.929s feedback_info_pack=1.882s effective_fps=347.48 +2026-09-25 21:56:35.513 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1073.439s infer_obs_pack=6.713s env_step=65.596s feedback_total=32.975s feedback_obs_pack=8.940s feedback_info_pack=1.885s effective_fps=347.50 +2026-09-25 21:56:35.514 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 17885/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/control/client-seed2.log b/experiments/e29-clamp-or-noise/results/control/client-seed2.log new file mode 100644 index 0000000..42d6d17 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/client-seed2.log @@ -0,0 +1,49 @@ +2026-09-25 21:36:38.343 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:38.343 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:38.343 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:38.344 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:38.344 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:38.383 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213638-5cee620b output_dir=runs/mujoco-v1-nenv1-20260925-213638-5cee620b recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:38.417 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9602... +2026-09-25 21:36:38.420 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:08.422 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=9994 infer_calls=9994 feedback_calls=9994 infer_wait=27.249s infer_obs_pack=0.141s env_step=1.401s feedback_total=0.689s feedback_obs_pack=0.187s feedback_info_pack=0.037s effective_fps=339.02 +2026-09-25 21:37:38.423 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20152 infer_calls=20152 feedback_calls=20152 infer_wait=54.196s infer_obs_pack=0.307s env_step=2.960s feedback_total=1.458s feedback_obs_pack=0.391s feedback_info_pack=0.077s effective_fps=342.02 +2026-09-25 21:38:08.427 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30218 infer_calls=30218 feedback_calls=30218 infer_wait=81.042s infer_obs_pack=0.470s env_step=4.592s feedback_total=2.255s feedback_obs_pack=0.614s feedback_info_pack=0.123s effective_fps=341.99 +2026-09-25 21:38:38.428 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40375 infer_calls=40375 feedback_calls=40375 infer_wait=107.853s infer_obs_pack=0.632s env_step=6.246s feedback_total=3.063s feedback_obs_pack=0.836s feedback_info_pack=0.168s effective_fps=342.76 +2026-09-25 21:39:08.429 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50485 infer_calls=50485 feedback_calls=50485 infer_wait=134.711s infer_obs_pack=0.795s env_step=7.850s feedback_total=3.878s feedback_obs_pack=1.058s feedback_info_pack=0.214s effective_fps=342.89 +2026-09-25 21:39:38.431 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=60874 infer_calls=60874 feedback_calls=60874 infer_wait=161.516s infer_obs_pack=0.960s env_step=9.475s feedback_total=4.716s feedback_obs_pack=1.279s feedback_info_pack=0.262s effective_fps=344.57 +2026-09-25 21:40:08.432 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71043 infer_calls=71043 feedback_calls=71043 infer_wait=188.408s infer_obs_pack=1.126s env_step=11.076s feedback_total=5.506s feedback_obs_pack=1.489s feedback_info_pack=0.304s effective_fps=344.68 +2026-09-25 21:40:38.435 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81341 infer_calls=81341 feedback_calls=81341 infer_wait=215.171s infer_obs_pack=1.292s env_step=12.748s feedback_total=6.339s feedback_obs_pack=1.712s feedback_info_pack=0.351s effective_fps=345.33 +2026-09-25 21:41:08.435 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=91403 infer_calls=91403 feedback_calls=91403 infer_wait=242.063s infer_obs_pack=1.453s env_step=14.349s feedback_total=7.129s feedback_obs_pack=1.930s feedback_info_pack=0.393s effective_fps=344.93 +2026-09-25 21:41:38.438 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=101886 infer_calls=101886 feedback_calls=101886 infer_wait=268.815s infer_obs_pack=1.622s env_step=16.004s feedback_total=7.987s feedback_obs_pack=2.152s feedback_info_pack=0.440s effective_fps=346.05 +2026-09-25 21:42:08.440 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=111862 infer_calls=111862 feedback_calls=111862 infer_wait=295.721s infer_obs_pack=1.782s env_step=17.598s feedback_total=8.764s feedback_obs_pack=2.369s feedback_info_pack=0.481s effective_fps=345.40 +2026-09-25 21:42:38.444 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122255 infer_calls=122255 feedback_calls=122255 infer_wait=322.505s infer_obs_pack=1.951s env_step=19.241s feedback_total=9.607s feedback_obs_pack=2.586s feedback_info_pack=0.526s effective_fps=346.03 +2026-09-25 21:43:08.446 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=132464 infer_calls=132464 feedback_calls=132464 infer_wait=349.350s infer_obs_pack=2.124s env_step=20.857s feedback_total=10.412s feedback_obs_pack=2.800s feedback_info_pack=0.569s effective_fps=346.09 +2026-09-25 21:43:38.447 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=142857 infer_calls=142857 feedback_calls=142857 infer_wait=376.116s infer_obs_pack=2.291s env_step=22.528s feedback_total=11.234s feedback_obs_pack=3.022s feedback_info_pack=0.613s effective_fps=346.60 +2026-09-25 21:44:08.449 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=153017 infer_calls=153017 feedback_calls=153017 infer_wait=402.932s infer_obs_pack=2.454s env_step=24.163s feedback_total=12.043s feedback_obs_pack=3.243s feedback_info_pack=0.659s effective_fps=346.51 +2026-09-25 21:44:38.451 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=163459 infer_calls=163459 feedback_calls=163459 infer_wait=429.668s infer_obs_pack=2.625s env_step=25.840s feedback_total=12.877s feedback_obs_pack=3.473s feedback_info_pack=0.704s effective_fps=347.04 +2026-09-25 21:45:08.453 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=173733 infer_calls=173733 feedback_calls=173733 infer_wait=456.511s infer_obs_pack=2.796s env_step=27.470s feedback_total=13.674s feedback_obs_pack=3.688s feedback_info_pack=0.747s effective_fps=347.15 +2026-09-25 21:45:38.457 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184069 infer_calls=184069 feedback_calls=184069 infer_wait=483.254s infer_obs_pack=2.967s env_step=29.144s feedback_total=14.500s feedback_obs_pack=3.914s feedback_info_pack=0.791s effective_fps=347.39 +2026-09-25 21:46:08.459 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=194280 infer_calls=194280 feedback_calls=194280 infer_wait=510.091s infer_obs_pack=3.127s env_step=30.772s feedback_total=15.307s feedback_obs_pack=4.136s feedback_info_pack=0.836s effective_fps=347.37 +2026-09-25 21:46:38.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=204743 infer_calls=204743 feedback_calls=204743 infer_wait=536.868s infer_obs_pack=3.297s env_step=32.423s feedback_total=16.144s feedback_obs_pack=4.357s feedback_info_pack=0.880s effective_fps=347.77 +2026-09-25 21:47:08.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=215032 infer_calls=215032 feedback_calls=215032 infer_wait=563.659s infer_obs_pack=3.460s env_step=34.067s feedback_total=16.976s feedback_obs_pack=4.575s feedback_info_pack=0.925s effective_fps=347.86 +2026-09-25 21:47:38.854 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=225282 infer_calls=225282 feedback_calls=225282 infer_wait=590.854s infer_obs_pack=3.631s env_step=35.705s feedback_total=17.792s feedback_obs_pack=4.795s feedback_info_pack=0.969s effective_fps=347.67 +2026-09-25 21:48:08.857 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=235693 infer_calls=235693 feedback_calls=235693 infer_wait=617.656s infer_obs_pack=3.796s env_step=37.349s feedback_total=18.620s feedback_obs_pack=5.013s feedback_info_pack=1.013s effective_fps=347.93 +2026-09-25 21:48:38.858 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=245944 infer_calls=245944 feedback_calls=245944 infer_wait=644.503s infer_obs_pack=3.968s env_step=38.950s feedback_total=19.447s feedback_obs_pack=5.230s feedback_info_pack=1.055s effective_fps=347.93 +2026-09-25 21:49:08.858 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=256403 infer_calls=256403 feedback_calls=256403 infer_wait=671.225s infer_obs_pack=4.138s env_step=40.639s feedback_total=20.283s feedback_obs_pack=5.457s feedback_info_pack=1.099s effective_fps=348.24 +2026-09-25 21:49:38.859 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=266639 infer_calls=266639 feedback_calls=266639 infer_wait=698.044s infer_obs_pack=4.303s env_step=42.258s feedback_total=21.111s feedback_obs_pack=5.673s feedback_info_pack=1.147s effective_fps=348.22 +2026-09-25 21:50:08.861 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=277127 infer_calls=277127 feedback_calls=277127 infer_wait=724.760s infer_obs_pack=4.474s env_step=43.944s feedback_total=21.948s feedback_obs_pack=5.903s feedback_info_pack=1.192s effective_fps=348.53 +2026-09-25 21:50:38.862 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=287408 infer_calls=287408 feedback_calls=287408 infer_wait=751.606s infer_obs_pack=4.643s env_step=45.557s feedback_total=22.758s feedback_obs_pack=6.115s feedback_info_pack=1.237s effective_fps=348.56 +2026-09-25 21:51:08.864 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=297833 infer_calls=297833 feedback_calls=297833 infer_wait=778.348s infer_obs_pack=4.812s env_step=47.243s feedback_total=23.588s feedback_obs_pack=6.340s feedback_info_pack=1.280s effective_fps=348.75 +2026-09-25 21:51:38.865 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=308026 infer_calls=308026 feedback_calls=308026 infer_wait=805.149s infer_obs_pack=4.975s env_step=48.900s feedback_total=24.402s feedback_obs_pack=6.557s feedback_info_pack=1.323s effective_fps=348.67 +2026-09-25 21:52:08.866 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=318477 infer_calls=318477 feedback_calls=318477 infer_wait=831.913s infer_obs_pack=5.149s env_step=50.561s feedback_total=25.223s feedback_obs_pack=6.776s feedback_info_pack=1.369s effective_fps=348.88 +2026-09-25 21:52:38.866 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=328755 infer_calls=328755 feedback_calls=328755 infer_wait=858.729s infer_obs_pack=5.320s env_step=52.188s feedback_total=26.054s feedback_obs_pack=6.992s feedback_info_pack=1.412s effective_fps=348.89 +2026-09-25 21:53:08.867 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=339046 infer_calls=339046 feedback_calls=339046 infer_wait=885.552s infer_obs_pack=5.489s env_step=53.824s feedback_total=26.871s feedback_obs_pack=7.214s feedback_info_pack=1.457s effective_fps=348.91 +2026-09-25 21:53:38.868 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=349097 infer_calls=349097 feedback_calls=349097 infer_wait=912.417s infer_obs_pack=5.662s env_step=55.450s feedback_total=27.659s feedback_obs_pack=7.431s feedback_info_pack=1.499s effective_fps=348.68 +2026-09-25 21:54:08.869 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=359477 infer_calls=359477 feedback_calls=359477 infer_wait=939.198s infer_obs_pack=5.826s env_step=57.109s feedback_total=28.481s feedback_obs_pack=7.655s feedback_info_pack=1.544s effective_fps=348.80 +2026-09-25 21:54:38.873 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=369614 infer_calls=369614 feedback_calls=369614 infer_wait=966.061s infer_obs_pack=5.996s env_step=58.738s feedback_total=29.279s feedback_obs_pack=7.873s feedback_info_pack=1.586s effective_fps=348.67 +2026-09-25 21:55:08.875 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=379847 infer_calls=379847 feedback_calls=379847 infer_wait=992.846s infer_obs_pack=6.168s env_step=60.400s feedback_total=30.090s feedback_obs_pack=8.091s feedback_info_pack=1.629s effective_fps=348.64 +2026-09-25 21:55:38.877 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=389843 infer_calls=389843 feedback_calls=389843 infer_wait=1019.716s infer_obs_pack=6.342s env_step=62.023s feedback_total=30.873s feedback_obs_pack=8.310s feedback_info_pack=1.675s effective_fps=348.40 +2026-09-25 21:56:08.877 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=400184 infer_calls=400184 feedback_calls=400184 infer_wait=1046.477s infer_obs_pack=6.508s env_step=63.703s feedback_total=31.701s feedback_obs_pack=8.541s feedback_info_pack=1.719s effective_fps=348.47 +2026-09-25 21:56:36.524 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1070.726s infer_obs_pack=6.657s env_step=65.186s feedback_total=32.448s feedback_obs_pack=8.738s feedback_info_pack=1.759s effective_fps=348.59 +2026-09-25 21:56:36.525 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 17579/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/control/server-seed0.log b/experiments/e29-clamp-or-noise/results/control/server-seed0.log new file mode 100644 index 0000000..75c618a --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/server-seed0.log @@ -0,0 +1,25 @@ +21:36:27|INFO|plugrl_server version: 0.1.0 +21:36:27|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:27|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:27|INFO|Seeded python, numpy and torch with 0. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:27|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0 +21:36:27|INFO|Policy created... +21:36:27|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:27|INFO|Algorithm created: + +21:36:27|INFO|Agent Server is listening on 0.0.0.0:9600 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:29|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0/81921 +21:44:27|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0/163841 +21:48:26|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0/245761 +21:52:24|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0/327681 +21:56:23|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed0/409601 +21:56:23|INFO|Stopping server as the algorithm signaled to stop. +21:56:23|INFO|Shutdown started: aborting pending infer requests. +21:56:23|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:23|INFO|Shutdown closing 1 websocket connection(s). +21:56:23|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 42726). +21:56:23|INFO|WebSocket server closed. +21:56:23|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/control/server-seed1.log b/experiments/e29-clamp-or-noise/results/control/server-seed1.log new file mode 100644 index 0000000..d2d5194 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/server-seed1.log @@ -0,0 +1,25 @@ +21:36:32|INFO|plugrl_server version: 0.1.0 +21:36:32|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:32|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:32|INFO|Seeded python, numpy and torch with 1. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:32|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1 +21:36:32|INFO|Policy created... +21:36:32|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:32|INFO|Algorithm created: + +21:36:32|INFO|Agent Server is listening on 0.0.0.0:9601 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:35|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1/81921 +21:44:35|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1/163841 +21:48:35|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1/245761 +21:52:34|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1/327681 +21:56:35|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed1/409601 +21:56:35|INFO|Stopping server as the algorithm signaled to stop. +21:56:35|INFO|Shutdown started: aborting pending infer requests. +21:56:35|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:35|INFO|Shutdown closing 1 websocket connection(s). +21:56:35|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 55398). +21:56:35|INFO|WebSocket server closed. +21:56:35|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/control/server-seed2.log b/experiments/e29-clamp-or-noise/results/control/server-seed2.log new file mode 100644 index 0000000..f6e4aeb --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/control/server-seed2.log @@ -0,0 +1,25 @@ +21:36:37|INFO|plugrl_server version: 0.1.0 +21:36:37|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:37|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:37|INFO|Seeded python, numpy and torch with 2. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:37|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2 +21:36:37|INFO|Policy created... +21:36:38|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:38|INFO|Algorithm created: + +21:36:38|INFO|Agent Server is listening on 0.0.0.0:9602 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:40|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2/81921 +21:44:40|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2/163841 +21:48:38|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2/245761 +21:52:35|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2/327681 +21:56:36|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/control/dppo/fpo-policy/control-seed2/409601 +21:56:36|INFO|Stopping server as the algorithm signaled to stop. +21:56:36|INFO|Shutdown started: aborting pending infer requests. +21:56:36|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:36|INFO|Shutdown closing 1 websocket connection(s). +21:56:36|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 45654). +21:56:36|INFO|WebSocket server closed. +21:56:36|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/noclamp.out b/experiments/e29-clamp-or-noise/results/noclamp.out new file mode 100644 index 0000000..b5330ad --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp.out @@ -0,0 +1,11 @@ +cell: noclamp = fpo-policy/default x dppo/hopper x Hopper-v5 +iters: 100 x 4096 batch: variant default replan: 1 seeds: 0 1 2 +extra: --algo.logprob-clamp-max inf +out: /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp +start 2026-09-25 21:36:46 +seed 0 finished rc=0 at 21:56:42 +seed 2 finished rc=0 at 21:56:53 +seed 1 finished rc=0 at 21:56:55 +end 2026-09-25 21:56:55 +failed seeds: 0 +CELL_DONE noclamp diff --git a/experiments/e29-clamp-or-noise/results/noclamp/client-seed0.log b/experiments/e29-clamp-or-noise/results/noclamp/client-seed0.log new file mode 100644 index 0000000..424b09e --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/client-seed0.log @@ -0,0 +1,49 @@ +2026-09-25 21:36:48.336 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:48.336 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:48.336 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:48.337 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:48.337 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:48.377 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213648-90c0d935 output_dir=runs/mujoco-v1-nenv1-20260925-213648-90c0d935 recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:48.416 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9610... +2026-09-25 21:36:48.418 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:18.420 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10080 infer_calls=10080 feedback_calls=10080 infer_wait=27.133s infer_obs_pack=0.152s env_step=1.437s feedback_total=0.746s feedback_obs_pack=0.194s feedback_info_pack=0.044s effective_fps=342.06 +2026-09-25 21:37:48.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20305 infer_calls=20305 feedback_calls=20305 infer_wait=54.029s infer_obs_pack=0.324s env_step=3.026s feedback_total=1.524s feedback_obs_pack=0.407s feedback_info_pack=0.088s effective_fps=344.72 +2026-09-25 21:38:18.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30450 infer_calls=30450 feedback_calls=30450 infer_wait=80.878s infer_obs_pack=0.489s env_step=4.640s feedback_total=2.324s feedback_obs_pack=0.623s feedback_info_pack=0.136s effective_fps=344.73 +2026-09-25 21:38:48.423 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40805 infer_calls=40805 feedback_calls=40805 infer_wait=107.671s infer_obs_pack=0.661s env_step=6.269s feedback_total=3.151s feedback_obs_pack=0.846s feedback_info_pack=0.181s effective_fps=346.54 +2026-09-25 21:39:18.425 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50964 infer_calls=50964 feedback_calls=50964 infer_wait=134.495s infer_obs_pack=0.825s env_step=7.886s feedback_total=3.978s feedback_obs_pack=1.071s feedback_info_pack=0.230s effective_fps=346.26 +2026-09-25 21:39:48.425 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61430 infer_calls=61430 feedback_calls=61430 infer_wait=161.224s infer_obs_pack=1.002s env_step=9.548s feedback_total=4.830s feedback_obs_pack=1.294s feedback_info_pack=0.279s effective_fps=347.84 +2026-09-25 21:40:18.429 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71566 infer_calls=71566 feedback_calls=71566 infer_wait=188.022s infer_obs_pack=1.168s env_step=11.208s feedback_total=5.642s feedback_obs_pack=1.517s feedback_info_pack=0.328s effective_fps=347.34 +2026-09-25 21:40:48.430 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81905 infer_calls=81905 feedback_calls=81905 infer_wait=214.825s infer_obs_pack=1.333s env_step=12.839s feedback_total=6.473s feedback_obs_pack=1.746s feedback_info_pack=0.377s effective_fps=347.84 +2026-09-25 21:41:18.431 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=92029 infer_calls=92029 feedback_calls=92029 infer_wait=241.678s infer_obs_pack=1.498s env_step=14.445s feedback_total=7.281s feedback_obs_pack=1.962s feedback_info_pack=0.422s effective_fps=347.41 +2026-09-25 21:41:48.726 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=102402 infer_calls=102402 feedback_calls=102402 infer_wait=268.713s infer_obs_pack=1.668s env_step=16.129s feedback_total=8.101s feedback_obs_pack=2.184s feedback_info_pack=0.473s effective_fps=347.58 +2026-09-25 21:42:18.728 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112704 infer_calls=112704 feedback_calls=112704 infer_wait=295.517s infer_obs_pack=1.834s env_step=17.768s feedback_total=8.925s feedback_obs_pack=2.401s feedback_info_pack=0.523s effective_fps=347.80 +2026-09-25 21:42:48.729 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=123055 infer_calls=123055 feedback_calls=123055 infer_wait=322.307s infer_obs_pack=2.002s env_step=19.408s feedback_total=9.759s feedback_obs_pack=2.619s feedback_info_pack=0.570s effective_fps=348.13 +2026-09-25 21:43:18.734 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=133427 infer_calls=133427 feedback_calls=133427 infer_wait=349.078s infer_obs_pack=2.173s env_step=21.064s feedback_total=10.588s feedback_obs_pack=2.843s feedback_info_pack=0.618s effective_fps=348.46 +2026-09-25 21:43:48.735 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=143644 infer_calls=143644 feedback_calls=143644 infer_wait=375.829s infer_obs_pack=2.340s env_step=22.721s feedback_total=11.429s feedback_obs_pack=3.071s feedback_info_pack=0.668s effective_fps=348.38 +2026-09-25 21:44:18.737 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=154076 infer_calls=154076 feedback_calls=154076 infer_wait=402.550s infer_obs_pack=2.510s env_step=24.403s feedback_total=12.266s feedback_obs_pack=3.295s feedback_info_pack=0.718s effective_fps=348.80 +2026-09-25 21:44:48.737 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=164437 infer_calls=164437 feedback_calls=164437 infer_wait=429.314s infer_obs_pack=2.680s env_step=26.068s feedback_total=13.086s feedback_obs_pack=3.517s feedback_info_pack=0.766s effective_fps=349.01 +2026-09-25 21:45:18.737 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=174916 infer_calls=174916 feedback_calls=174916 infer_wait=456.055s infer_obs_pack=2.852s env_step=27.752s feedback_total=13.903s feedback_obs_pack=3.740s feedback_info_pack=0.816s effective_fps=349.44 +2026-09-25 21:45:48.739 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184993 infer_calls=184993 feedback_calls=184993 infer_wait=482.880s infer_obs_pack=3.019s env_step=29.390s feedback_total=14.704s feedback_obs_pack=3.957s feedback_info_pack=0.864s effective_fps=349.05 +2026-09-25 21:46:18.741 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=195271 infer_calls=195271 feedback_calls=195271 infer_wait=509.631s infer_obs_pack=3.185s env_step=31.076s feedback_total=15.519s feedback_obs_pack=4.178s feedback_info_pack=0.911s effective_fps=349.07 +2026-09-25 21:46:48.743 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=205504 infer_calls=205504 feedback_calls=205504 infer_wait=536.423s infer_obs_pack=3.351s env_step=32.719s feedback_total=16.343s feedback_obs_pack=4.404s feedback_info_pack=0.962s effective_fps=349.00 +2026-09-25 21:47:18.744 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=215919 infer_calls=215919 feedback_calls=215919 infer_wait=563.183s infer_obs_pack=3.522s env_step=34.385s feedback_total=17.171s feedback_obs_pack=4.624s feedback_info_pack=1.014s effective_fps=349.24 +2026-09-25 21:47:48.745 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=226038 infer_calls=226038 feedback_calls=226038 infer_wait=590.041s infer_obs_pack=3.687s env_step=35.993s feedback_total=17.974s feedback_obs_pack=4.841s feedback_info_pack=1.063s effective_fps=348.99 +2026-09-25 21:48:18.747 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=236458 infer_calls=236458 feedback_calls=236458 infer_wait=616.795s infer_obs_pack=3.854s env_step=37.651s feedback_total=18.814s feedback_obs_pack=5.067s feedback_info_pack=1.112s effective_fps=349.21 +2026-09-25 21:48:48.749 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=246716 infer_calls=246716 feedback_calls=246716 infer_wait=643.583s infer_obs_pack=4.029s env_step=39.308s feedback_total=19.633s feedback_obs_pack=5.289s feedback_info_pack=1.163s effective_fps=349.18 +2026-09-25 21:49:18.750 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=257132 infer_calls=257132 feedback_calls=257132 infer_wait=670.300s infer_obs_pack=4.201s env_step=40.996s feedback_total=20.473s feedback_obs_pack=5.516s feedback_info_pack=1.211s effective_fps=349.38 +2026-09-25 21:49:48.751 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=267306 infer_calls=267306 feedback_calls=267306 infer_wait=697.123s infer_obs_pack=4.375s env_step=42.624s feedback_total=21.274s feedback_obs_pack=5.739s feedback_info_pack=1.259s effective_fps=349.24 +2026-09-25 21:50:18.753 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=277723 infer_calls=277723 feedback_calls=277723 infer_wait=723.855s infer_obs_pack=4.546s env_step=44.301s feedback_total=22.107s feedback_obs_pack=5.969s feedback_info_pack=1.308s effective_fps=349.42 +2026-09-25 21:50:48.754 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=288071 infer_calls=288071 feedback_calls=288071 infer_wait=750.615s infer_obs_pack=4.714s env_step=45.967s feedback_total=22.933s feedback_obs_pack=6.196s feedback_info_pack=1.358s effective_fps=349.50 +2026-09-25 21:51:18.757 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=298401 infer_calls=298401 feedback_calls=298401 infer_wait=777.356s infer_obs_pack=4.885s env_step=47.652s feedback_total=23.758s feedback_obs_pack=6.421s feedback_info_pack=1.405s effective_fps=349.56 +2026-09-25 21:51:48.761 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=308564 infer_calls=308564 feedback_calls=308564 infer_wait=804.145s infer_obs_pack=5.057s env_step=49.315s feedback_total=24.570s feedback_obs_pack=6.639s feedback_info_pack=1.454s effective_fps=349.42 +2026-09-25 21:52:18.761 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=319122 infer_calls=319122 feedback_calls=319122 infer_wait=830.879s infer_obs_pack=5.230s env_step=50.981s feedback_total=25.412s feedback_obs_pack=6.863s feedback_info_pack=1.506s effective_fps=349.72 +2026-09-25 21:52:48.763 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=329317 infer_calls=329317 feedback_calls=329317 infer_wait=857.713s infer_obs_pack=5.401s env_step=52.610s feedback_total=26.215s feedback_obs_pack=7.083s feedback_info_pack=1.552s effective_fps=349.62 +2026-09-25 21:53:18.763 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=339585 infer_calls=339585 feedback_calls=339585 infer_wait=884.518s infer_obs_pack=5.576s env_step=54.247s feedback_total=27.025s feedback_obs_pack=7.303s feedback_info_pack=1.598s effective_fps=349.60 +2026-09-25 21:53:48.765 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=349824 infer_calls=349824 feedback_calls=349824 infer_wait=911.330s infer_obs_pack=5.743s env_step=55.893s feedback_total=27.837s feedback_obs_pack=7.533s feedback_info_pack=1.644s effective_fps=349.54 +2026-09-25 21:54:18.765 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=360283 infer_calls=360283 feedback_calls=360283 infer_wait=938.076s infer_obs_pack=5.913s env_step=57.567s feedback_total=28.669s feedback_obs_pack=7.762s feedback_info_pack=1.693s effective_fps=349.71 +2026-09-25 21:54:48.768 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=370619 infer_calls=370619 feedback_calls=370619 infer_wait=964.888s infer_obs_pack=6.081s env_step=59.203s feedback_total=29.492s feedback_obs_pack=7.982s feedback_info_pack=1.741s effective_fps=349.75 +2026-09-25 21:55:19.101 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=380930 infer_calls=380930 feedback_calls=380930 infer_wait=992.031s infer_obs_pack=6.245s env_step=60.843s feedback_total=30.310s feedback_obs_pack=8.201s feedback_info_pack=1.786s effective_fps=349.66 +2026-09-25 21:55:49.105 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=391369 infer_calls=391369 feedback_calls=391369 infer_wait=1018.790s infer_obs_pack=6.417s env_step=62.522s feedback_total=31.132s feedback_obs_pack=8.424s feedback_info_pack=1.834s effective_fps=349.79 +2026-09-25 21:56:19.106 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=401583 infer_calls=401583 feedback_calls=401583 infer_wait=1045.572s infer_obs_pack=6.582s env_step=64.183s feedback_total=31.953s feedback_obs_pack=8.649s feedback_info_pack=1.882s effective_fps=349.72 +2026-09-25 21:56:42.533 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1066.052s infer_obs_pack=6.708s env_step=65.440s feedback_total=32.574s feedback_obs_pack=8.819s feedback_info_pack=1.917s effective_fps=349.85 +2026-09-25 21:56:42.533 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 17916/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/noclamp/client-seed1.log b/experiments/e29-clamp-or-noise/results/noclamp/client-seed1.log new file mode 100644 index 0000000..e6fa4ce --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/client-seed1.log @@ -0,0 +1,50 @@ +2026-09-25 21:36:53.339 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:53.340 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:53.340 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:53.340 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:53.341 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:53.381 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213653-082d2b2c output_dir=runs/mujoco-v1-nenv1-20260925-213653-082d2b2c recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:53.416 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9611... +2026-09-25 21:36:53.419 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:23.421 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10116 infer_calls=10116 feedback_calls=10116 infer_wait=27.127s infer_obs_pack=0.154s env_step=1.449s feedback_total=0.738s feedback_obs_pack=0.197s feedback_info_pack=0.046s effective_fps=343.28 +2026-09-25 21:37:53.422 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20394 infer_calls=20394 feedback_calls=20394 infer_wait=53.962s infer_obs_pack=0.328s env_step=3.051s feedback_total=1.561s feedback_obs_pack=0.415s feedback_info_pack=0.098s effective_fps=346.24 +2026-09-25 21:38:23.424 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30539 infer_calls=30539 feedback_calls=30539 infer_wait=80.809s infer_obs_pack=0.501s env_step=4.646s feedback_total=2.379s feedback_obs_pack=0.631s feedback_info_pack=0.148s effective_fps=345.71 +2026-09-25 21:38:53.425 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40821 infer_calls=40821 feedback_calls=40821 infer_wait=107.623s infer_obs_pack=0.669s env_step=6.267s feedback_total=3.209s feedback_obs_pack=0.852s feedback_info_pack=0.200s effective_fps=346.62 +2026-09-25 21:39:23.425 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50868 infer_calls=50868 feedback_calls=50868 infer_wait=134.513s infer_obs_pack=0.833s env_step=7.868s feedback_total=3.998s feedback_obs_pack=1.066s feedback_info_pack=0.249s effective_fps=345.54 +2026-09-25 21:39:53.426 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61149 infer_calls=61149 feedback_calls=61149 infer_wait=161.289s infer_obs_pack=0.996s env_step=9.519s feedback_total=4.819s feedback_obs_pack=1.288s feedback_info_pack=0.304s effective_fps=346.21 +2026-09-25 21:40:23.427 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71347 infer_calls=71347 feedback_calls=71347 infer_wait=188.080s infer_obs_pack=1.157s env_step=11.144s feedback_total=5.661s feedback_obs_pack=1.510s feedback_info_pack=0.355s effective_fps=346.27 +2026-09-25 21:40:53.429 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81681 infer_calls=81681 feedback_calls=81681 infer_wait=214.843s infer_obs_pack=1.331s env_step=12.780s feedback_total=6.507s feedback_obs_pack=1.735s feedback_info_pack=0.408s effective_fps=346.90 +2026-09-25 21:41:23.432 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=91750 infer_calls=91750 feedback_calls=91750 infer_wait=241.687s infer_obs_pack=1.498s env_step=14.401s feedback_total=7.319s feedback_obs_pack=1.957s feedback_info_pack=0.460s effective_fps=346.35 +2026-09-25 21:41:53.433 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=102089 infer_calls=102089 feedback_calls=102089 infer_wait=268.450s infer_obs_pack=1.668s env_step=16.063s feedback_total=8.147s feedback_obs_pack=2.185s feedback_info_pack=0.512s effective_fps=346.86 +2026-09-25 21:42:23.437 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112234 infer_calls=112234 feedback_calls=112234 infer_wait=295.263s infer_obs_pack=1.834s env_step=17.695s feedback_total=8.961s feedback_obs_pack=2.407s feedback_info_pack=0.563s effective_fps=346.67 +2026-09-25 21:42:53.438 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122661 infer_calls=122661 feedback_calls=122661 infer_wait=321.997s infer_obs_pack=2.008s env_step=19.357s feedback_total=9.808s feedback_obs_pack=2.633s feedback_info_pack=0.616s effective_fps=347.31 +2026-09-25 21:43:23.439 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=132837 infer_calls=132837 feedback_calls=132837 infer_wait=348.741s infer_obs_pack=2.176s env_step=21.021s feedback_total=10.641s feedback_obs_pack=2.856s feedback_info_pack=0.667s effective_fps=347.21 +2026-09-25 21:43:53.441 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=143218 infer_calls=143218 feedback_calls=143218 infer_wait=375.437s infer_obs_pack=2.357s env_step=22.709s feedback_total=11.488s feedback_obs_pack=3.089s feedback_info_pack=0.720s effective_fps=347.62 +2026-09-25 21:44:23.442 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=153402 infer_calls=153402 feedback_calls=153402 infer_wait=402.214s infer_obs_pack=2.536s env_step=24.345s feedback_total=12.322s feedback_obs_pack=3.312s feedback_info_pack=0.772s effective_fps=347.52 +2026-09-25 21:44:53.443 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=163723 infer_calls=163723 feedback_calls=163723 infer_wait=428.953s infer_obs_pack=2.710s env_step=26.013s feedback_total=13.164s feedback_obs_pack=3.535s feedback_info_pack=0.828s effective_fps=347.73 +2026-09-25 21:45:23.444 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=173824 infer_calls=173824 feedback_calls=173824 infer_wait=455.769s infer_obs_pack=2.875s env_step=27.645s feedback_total=13.978s feedback_obs_pack=3.761s feedback_info_pack=0.880s effective_fps=347.46 +2026-09-25 21:45:53.446 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184130 infer_calls=184130 feedback_calls=184130 infer_wait=482.538s infer_obs_pack=3.044s env_step=29.291s feedback_total=14.823s feedback_obs_pack=3.988s feedback_info_pack=0.934s effective_fps=347.61 +2026-09-25 21:46:23.449 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=194143 infer_calls=194143 feedback_calls=194143 infer_wait=509.411s infer_obs_pack=3.210s env_step=30.887s feedback_total=15.634s feedback_obs_pack=4.206s feedback_info_pack=0.986s effective_fps=347.22 +2026-09-25 21:46:53.449 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=204396 infer_calls=204396 feedback_calls=204396 infer_wait=536.134s infer_obs_pack=3.380s env_step=32.573s feedback_total=16.472s feedback_obs_pack=4.437s feedback_info_pack=1.040s effective_fps=347.28 +2026-09-25 21:47:23.450 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=214429 infer_calls=214429 feedback_calls=214429 infer_wait=562.968s infer_obs_pack=3.550s env_step=34.185s feedback_total=17.286s feedback_obs_pack=4.652s feedback_info_pack=1.090s effective_fps=346.98 +2026-09-25 21:47:53.452 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=224708 infer_calls=224708 feedback_calls=224708 infer_wait=589.736s infer_obs_pack=3.722s env_step=35.823s feedback_total=18.132s feedback_obs_pack=4.883s feedback_info_pack=1.141s effective_fps=347.09 +2026-09-25 21:48:23.454 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=234774 infer_calls=234774 feedback_calls=234774 infer_wait=616.523s infer_obs_pack=3.890s env_step=37.464s feedback_total=18.966s feedback_obs_pack=5.107s feedback_info_pack=1.199s effective_fps=346.87 +2026-09-25 21:48:53.455 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=245122 infer_calls=245122 feedback_calls=245122 infer_wait=643.213s infer_obs_pack=4.060s env_step=39.148s feedback_total=19.836s feedback_obs_pack=5.334s feedback_info_pack=1.254s effective_fps=347.07 +2026-09-25 21:49:23.457 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=255381 infer_calls=255381 feedback_calls=255381 infer_wait=670.030s infer_obs_pack=4.225s env_step=40.771s feedback_total=20.670s feedback_obs_pack=5.557s feedback_info_pack=1.306s effective_fps=347.13 +2026-09-25 21:49:53.460 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=265851 infer_calls=265851 feedback_calls=265851 infer_wait=696.804s infer_obs_pack=4.392s env_step=42.422s feedback_total=21.511s feedback_obs_pack=5.779s feedback_info_pack=1.358s effective_fps=347.46 +2026-09-25 21:50:23.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=276099 infer_calls=276099 feedback_calls=276099 infer_wait=723.588s infer_obs_pack=4.560s env_step=44.059s feedback_total=22.350s feedback_obs_pack=5.996s feedback_info_pack=1.411s effective_fps=347.49 +2026-09-25 21:50:53.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=286512 infer_calls=286512 feedback_calls=286512 infer_wait=750.303s infer_obs_pack=4.732s env_step=45.740s feedback_total=23.203s feedback_obs_pack=6.219s feedback_info_pack=1.465s effective_fps=347.72 +2026-09-25 21:51:23.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=296652 infer_calls=296652 feedback_calls=296652 infer_wait=777.166s infer_obs_pack=4.896s env_step=47.349s feedback_total=24.012s feedback_obs_pack=6.444s feedback_info_pack=1.515s effective_fps=347.60 +2026-09-25 21:51:53.466 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=306959 infer_calls=306959 feedback_calls=306959 infer_wait=803.889s infer_obs_pack=5.068s env_step=49.029s feedback_total=24.851s feedback_obs_pack=6.671s feedback_info_pack=1.567s effective_fps=347.70 +2026-09-25 21:52:23.467 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=317174 infer_calls=317174 feedback_calls=317174 infer_wait=830.635s infer_obs_pack=5.242s env_step=50.679s feedback_total=25.709s feedback_obs_pack=6.895s feedback_info_pack=1.618s effective_fps=347.68 +2026-09-25 21:52:53.467 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=327460 infer_calls=327460 feedback_calls=327460 infer_wait=857.400s infer_obs_pack=5.411s env_step=52.339s feedback_total=26.532s feedback_obs_pack=7.117s feedback_info_pack=1.671s effective_fps=347.74 +2026-09-25 21:53:23.467 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=337507 infer_calls=337507 feedback_calls=337507 infer_wait=884.219s infer_obs_pack=5.573s env_step=53.979s feedback_total=27.343s feedback_obs_pack=7.343s feedback_info_pack=1.721s effective_fps=347.55 +2026-09-25 21:53:53.469 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=347774 infer_calls=347774 feedback_calls=347774 infer_wait=910.983s infer_obs_pack=5.739s env_step=55.638s feedback_total=28.186s feedback_obs_pack=7.569s feedback_info_pack=1.777s effective_fps=347.58 +2026-09-25 21:54:23.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=357891 infer_calls=357891 feedback_calls=357891 infer_wait=937.792s infer_obs_pack=5.906s env_step=57.269s feedback_total=29.006s feedback_obs_pack=7.791s feedback_info_pack=1.830s effective_fps=347.48 +2026-09-25 21:54:53.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=368228 infer_calls=368228 feedback_calls=368228 infer_wait=964.561s infer_obs_pack=6.081s env_step=58.924s feedback_total=29.846s feedback_obs_pack=8.018s feedback_info_pack=1.884s effective_fps=347.58 +2026-09-25 21:55:23.479 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=378320 infer_calls=378320 feedback_calls=378320 infer_wait=991.418s infer_obs_pack=6.248s env_step=60.522s feedback_total=30.669s feedback_obs_pack=8.237s feedback_info_pack=1.934s effective_fps=347.45 +2026-09-25 21:55:53.481 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=388626 infer_calls=388626 feedback_calls=388626 infer_wait=1018.160s infer_obs_pack=6.419s env_step=62.198s feedback_total=31.505s feedback_obs_pack=8.468s feedback_info_pack=1.987s effective_fps=347.52 +2026-09-25 21:56:23.483 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=398757 infer_calls=398757 feedback_calls=398757 infer_wait=1044.988s infer_obs_pack=6.593s env_step=63.813s feedback_total=32.330s feedback_obs_pack=8.687s feedback_info_pack=2.040s effective_fps=347.43 +2026-09-25 21:56:53.487 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=409067 infer_calls=409067 feedback_calls=409067 infer_wait=1071.908s infer_obs_pack=6.753s env_step=65.378s feedback_total=33.119s feedback_obs_pack=8.899s feedback_info_pack=2.088s effective_fps=347.50 +2026-09-25 21:56:55.494 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1073.278s infer_obs_pack=6.760s env_step=65.457s feedback_total=33.156s feedback_obs_pack=8.909s feedback_info_pack=2.090s effective_fps=347.52 +2026-09-25 21:56:55.494 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 18242/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/noclamp/client-seed2.log b/experiments/e29-clamp-or-noise/results/noclamp/client-seed2.log new file mode 100644 index 0000000..44b1c50 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/client-seed2.log @@ -0,0 +1,49 @@ +2026-09-25 21:36:58.348 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:36:58.348 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:36:58.348 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:36:58.349 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:36:58.349 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:36:58.393 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213658-1e768064 output_dir=runs/mujoco-v1-nenv1-20260925-213658-1e768064 recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:36:58.430 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9612... +2026-09-25 21:36:58.432 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:28.436 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10075 infer_calls=10075 feedback_calls=10075 infer_wait=27.099s infer_obs_pack=0.144s env_step=1.484s feedback_total=0.741s feedback_obs_pack=0.195s feedback_info_pack=0.041s effective_fps=341.90 +2026-09-25 21:37:58.865 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20482 infer_calls=20482 feedback_calls=20482 infer_wait=54.299s infer_obs_pack=0.330s env_step=3.121s feedback_total=1.584s feedback_obs_pack=0.417s feedback_info_pack=0.086s effective_fps=345.20 +2026-09-25 21:38:28.868 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30784 infer_calls=30784 feedback_calls=30784 infer_wait=81.099s infer_obs_pack=0.503s env_step=4.762s feedback_total=2.405s feedback_obs_pack=0.635s feedback_info_pack=0.132s effective_fps=346.79 +2026-09-25 21:38:58.932 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40962 infer_calls=40962 feedback_calls=40962 infer_wait=107.969s infer_obs_pack=0.672s env_step=6.395s feedback_total=3.229s feedback_obs_pack=0.852s feedback_info_pack=0.183s effective_fps=346.35 +2026-09-25 21:39:28.932 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=51318 infer_calls=51318 feedback_calls=51318 infer_wait=134.737s infer_obs_pack=0.841s env_step=8.054s feedback_total=4.065s feedback_obs_pack=1.070s feedback_info_pack=0.229s effective_fps=347.45 +2026-09-25 21:39:58.932 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61470 infer_calls=61470 feedback_calls=61470 infer_wait=161.539s infer_obs_pack=1.010s env_step=9.691s feedback_total=4.895s feedback_obs_pack=1.291s feedback_info_pack=0.276s effective_fps=347.02 +2026-09-25 21:40:28.935 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71804 infer_calls=71804 feedback_calls=71804 infer_wait=188.308s infer_obs_pack=1.181s env_step=11.338s feedback_total=5.738s feedback_obs_pack=1.509s feedback_info_pack=0.323s effective_fps=347.61 +2026-09-25 21:40:59.121 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81922 infer_calls=81922 feedback_calls=81922 infer_wait=215.392s infer_obs_pack=1.346s env_step=12.930s feedback_total=6.533s feedback_obs_pack=1.726s feedback_info_pack=0.370s effective_fps=346.83 +2026-09-25 21:41:29.123 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=92214 infer_calls=92214 feedback_calls=92214 infer_wait=242.112s infer_obs_pack=1.517s env_step=14.607s feedback_total=7.391s feedback_obs_pack=1.955s feedback_info_pack=0.418s effective_fps=347.16 +2026-09-25 21:41:59.237 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=102402 infer_calls=102402 feedback_calls=102402 infer_wait=269.068s infer_obs_pack=1.679s env_step=16.226s feedback_total=8.206s feedback_obs_pack=2.171s feedback_info_pack=0.466s effective_fps=346.92 +2026-09-25 21:42:29.240 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112731 infer_calls=112731 feedback_calls=112731 infer_wait=295.852s infer_obs_pack=1.849s env_step=17.862s feedback_total=9.046s feedback_obs_pack=2.391s feedback_info_pack=0.512s effective_fps=347.28 +2026-09-25 21:42:59.240 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122967 infer_calls=122967 feedback_calls=122967 infer_wait=322.665s infer_obs_pack=2.012s env_step=19.485s feedback_total=9.884s feedback_obs_pack=2.618s feedback_info_pack=0.561s effective_fps=347.32 +2026-09-25 21:43:29.243 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=133403 infer_calls=133403 feedback_calls=133403 infer_wait=349.429s infer_obs_pack=2.185s env_step=21.145s feedback_total=10.720s feedback_obs_pack=2.835s feedback_info_pack=0.608s effective_fps=347.87 +2026-09-25 21:43:59.243 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=143635 infer_calls=143635 feedback_calls=143635 infer_wait=376.244s infer_obs_pack=2.356s env_step=22.749s feedback_total=11.568s feedback_obs_pack=3.051s feedback_info_pack=0.657s effective_fps=347.85 +2026-09-25 21:44:29.246 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=154015 infer_calls=154015 feedback_calls=154015 infer_wait=403.020s infer_obs_pack=2.528s env_step=24.399s feedback_total=12.398s feedback_obs_pack=3.271s feedback_info_pack=0.704s effective_fps=348.18 +2026-09-25 21:44:59.248 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=164304 infer_calls=164304 feedback_calls=164304 infer_wait=429.806s infer_obs_pack=2.694s env_step=26.045s feedback_total=13.239s feedback_obs_pack=3.488s feedback_info_pack=0.749s effective_fps=348.26 +2026-09-25 21:45:29.251 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=174698 infer_calls=174698 feedback_calls=174698 infer_wait=456.545s infer_obs_pack=2.865s env_step=27.711s feedback_total=14.089s feedback_obs_pack=3.709s feedback_info_pack=0.798s effective_fps=348.55 +2026-09-25 21:45:59.252 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184925 infer_calls=184925 feedback_calls=184925 infer_wait=483.394s infer_obs_pack=3.029s env_step=29.329s feedback_total=14.905s feedback_obs_pack=3.923s feedback_info_pack=0.842s effective_fps=348.48 +2026-09-25 21:46:29.256 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=195333 infer_calls=195333 feedback_calls=195333 infer_wait=510.164s infer_obs_pack=3.196s env_step=30.973s feedback_total=15.747s feedback_obs_pack=4.139s feedback_info_pack=0.891s effective_fps=348.76 +2026-09-25 21:46:59.257 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=205489 infer_calls=205489 feedback_calls=205489 infer_wait=536.973s infer_obs_pack=3.365s env_step=32.610s feedback_total=16.576s feedback_obs_pack=4.362s feedback_info_pack=0.939s effective_fps=348.57 +2026-09-25 21:47:29.259 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=215863 infer_calls=215863 feedback_calls=215863 infer_wait=563.774s infer_obs_pack=3.529s env_step=34.251s feedback_total=17.410s feedback_obs_pack=4.582s feedback_info_pack=0.984s effective_fps=348.75 +2026-09-25 21:47:59.261 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=225968 infer_calls=225968 feedback_calls=225968 infer_wait=590.592s infer_obs_pack=3.693s env_step=35.885s feedback_total=18.236s feedback_obs_pack=4.796s feedback_info_pack=1.029s effective_fps=348.50 +2026-09-25 21:48:29.263 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=236497 infer_calls=236497 feedback_calls=236497 infer_wait=617.319s infer_obs_pack=3.868s env_step=37.543s feedback_total=19.098s feedback_obs_pack=5.022s feedback_info_pack=1.077s effective_fps=348.90 +2026-09-25 21:48:59.266 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=246868 infer_calls=246868 feedback_calls=246868 infer_wait=644.119s infer_obs_pack=4.045s env_step=39.160s feedback_total=19.953s feedback_obs_pack=5.237s feedback_info_pack=1.124s effective_fps=349.04 +2026-09-25 21:49:29.267 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=257264 infer_calls=257264 feedback_calls=257264 infer_wait=670.842s infer_obs_pack=4.224s env_step=40.841s feedback_total=20.789s feedback_obs_pack=5.461s feedback_info_pack=1.171s effective_fps=349.21 +2026-09-25 21:49:59.267 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=267501 infer_calls=267501 feedback_calls=267501 infer_wait=697.666s infer_obs_pack=4.395s env_step=42.463s feedback_total=21.616s feedback_obs_pack=5.678s feedback_info_pack=1.217s effective_fps=349.15 +2026-09-25 21:50:29.270 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=278027 infer_calls=278027 feedback_calls=278027 infer_wait=724.384s infer_obs_pack=4.569s env_step=44.140s feedback_total=22.477s feedback_obs_pack=5.905s feedback_info_pack=1.264s effective_fps=349.47 +2026-09-25 21:50:59.271 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=288274 infer_calls=288274 feedback_calls=288274 infer_wait=751.157s infer_obs_pack=4.739s env_step=45.796s feedback_total=23.312s feedback_obs_pack=6.130s feedback_info_pack=1.312s effective_fps=349.42 +2026-09-25 21:51:29.273 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=298712 infer_calls=298712 feedback_calls=298712 infer_wait=777.860s infer_obs_pack=4.913s env_step=47.496s feedback_total=24.166s feedback_obs_pack=6.357s feedback_info_pack=1.360s effective_fps=349.60 +2026-09-25 21:51:59.275 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=309030 infer_calls=309030 feedback_calls=309030 infer_wait=804.581s infer_obs_pack=5.080s env_step=49.164s feedback_total=25.038s feedback_obs_pack=6.580s feedback_info_pack=1.407s effective_fps=349.64 +2026-09-25 21:52:29.279 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=319418 infer_calls=319418 feedback_calls=319418 infer_wait=831.274s infer_obs_pack=5.248s env_step=50.878s feedback_total=25.878s feedback_obs_pack=6.808s feedback_info_pack=1.455s effective_fps=349.75 +2026-09-25 21:52:59.279 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=329544 infer_calls=329544 feedback_calls=329544 infer_wait=858.121s infer_obs_pack=5.415s env_step=52.494s feedback_total=26.700s feedback_obs_pack=7.026s feedback_info_pack=1.500s effective_fps=349.56 +2026-09-25 21:53:29.281 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=339901 infer_calls=339901 feedback_calls=339901 infer_wait=884.944s infer_obs_pack=5.585s env_step=54.111s feedback_total=27.533s feedback_obs_pack=7.242s feedback_info_pack=1.545s effective_fps=349.63 +2026-09-25 21:53:59.284 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=350099 infer_calls=350099 feedback_calls=350099 infer_wait=911.786s infer_obs_pack=5.745s env_step=55.729s feedback_total=28.372s feedback_obs_pack=7.460s feedback_info_pack=1.593s effective_fps=349.53 +2026-09-25 21:54:29.285 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=360430 infer_calls=360430 feedback_calls=360430 infer_wait=938.550s infer_obs_pack=5.918s env_step=57.400s feedback_total=29.194s feedback_obs_pack=7.679s feedback_info_pack=1.639s effective_fps=349.57 +2026-09-25 21:54:59.287 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=370605 infer_calls=370605 feedback_calls=370605 infer_wait=965.395s infer_obs_pack=6.083s env_step=59.015s feedback_total=30.012s feedback_obs_pack=7.892s feedback_info_pack=1.686s effective_fps=349.46 +2026-09-25 21:55:29.695 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=380930 infer_calls=380930 feedback_calls=380930 infer_wait=992.582s infer_obs_pack=6.259s env_step=60.653s feedback_total=30.852s feedback_obs_pack=8.108s feedback_info_pack=1.732s effective_fps=349.37 +2026-09-25 21:55:59.697 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=391427 infer_calls=391427 feedback_calls=391427 infer_wait=1019.311s infer_obs_pack=6.426s env_step=62.309s feedback_total=31.727s feedback_obs_pack=8.331s feedback_info_pack=1.780s effective_fps=349.56 +2026-09-25 21:56:29.699 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=401599 infer_calls=401599 feedback_calls=401599 infer_wait=1046.127s infer_obs_pack=6.594s env_step=63.942s feedback_total=32.560s feedback_obs_pack=8.552s feedback_info_pack=1.828s effective_fps=349.45 +2026-09-25 21:56:53.396 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1066.965s infer_obs_pack=6.716s env_step=65.150s feedback_total=33.165s feedback_obs_pack=8.710s feedback_info_pack=1.861s effective_fps=349.49 +2026-09-25 21:56:53.396 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 17675/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/noclamp/server-seed0.log b/experiments/e29-clamp-or-noise/results/noclamp/server-seed0.log new file mode 100644 index 0000000..ec13bd2 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/server-seed0.log @@ -0,0 +1,25 @@ +21:36:47|INFO|plugrl_server version: 0.1.0 +21:36:47|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=inf, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:47|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:47|INFO|Seeded python, numpy and torch with 0. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:47|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0 +21:36:48|INFO|Policy created... +21:36:48|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:48|INFO|Algorithm created: + +21:36:48|INFO|Agent Server is listening on 0.0.0.0:9610 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:48|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0/81921 +21:44:47|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0/163841 +21:48:46|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0/245761 +21:52:44|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0/327681 +21:56:42|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed0/409601 +21:56:42|INFO|Stopping server as the algorithm signaled to stop. +21:56:42|INFO|Shutdown started: aborting pending infer requests. +21:56:42|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:42|INFO|Shutdown closing 1 websocket connection(s). +21:56:42|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 59720). +21:56:42|INFO|WebSocket server closed. +21:56:42|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/noclamp/server-seed1.log b/experiments/e29-clamp-or-noise/results/noclamp/server-seed1.log new file mode 100644 index 0000000..a817e3d --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/server-seed1.log @@ -0,0 +1,25 @@ +21:36:53|INFO|plugrl_server version: 0.1.0 +21:36:53|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=inf, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:53|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:53|INFO|Seeded python, numpy and torch with 1. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:53|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1 +21:36:53|INFO|Policy created... +21:36:53|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:53|INFO|Algorithm created: + +21:36:53|INFO|Agent Server is listening on 0.0.0.0:9611 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:54|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1/81921 +21:44:54|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1/163841 +21:48:55|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1/245761 +21:52:54|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1/327681 +21:56:55|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed1/409601 +21:56:55|INFO|Stopping server as the algorithm signaled to stop. +21:56:55|INFO|Shutdown started: aborting pending infer requests. +21:56:55|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:55|INFO|Shutdown closing 1 websocket connection(s). +21:56:55|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 42866). +21:56:55|INFO|WebSocket server closed. +21:56:55|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/noclamp/server-seed2.log b/experiments/e29-clamp-or-noise/results/noclamp/server-seed2.log new file mode 100644 index 0000000..da4f970 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/noclamp/server-seed2.log @@ -0,0 +1,25 @@ +21:36:58|INFO|plugrl_server version: 0.1.0 +21:36:58|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=0.1, sampling_noise_level=0.1, logprob_clamp_min=-5.0, logprob_clamp_max=inf, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:36:58|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:36:58|INFO|Seeded python, numpy and torch with 2. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:36:58|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2 +21:36:58|INFO|Policy created... +21:36:58|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:36:58|INFO|Algorithm created: + +21:36:58|INFO|Agent Server is listening on 0.0.0.0:9612 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:40:59|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2/81921 +21:44:57|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2/163841 +21:48:56|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2/245761 +21:52:54|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2/327681 +21:56:53|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/noclamp/dppo/fpo-policy/noclamp-seed2/409601 +21:56:53|INFO|Stopping server as the algorithm signaled to stop. +21:56:53|INFO|Shutdown started: aborting pending infer requests. +21:56:53|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:56:53|INFO|Shutdown closing 1 websocket connection(s). +21:56:53|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 55598). +21:56:53|INFO|WebSocket server closed. +21:56:53|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/run.out b/experiments/e29-clamp-or-noise/results/run.out new file mode 100644 index 0000000..d71f8a5 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/run.out @@ -0,0 +1,9 @@ +start 2026-09-25 21:36:26 +code: 20484bf (src /home/guangzhao/zuogou/plugrl/e29/src) +client: 931ab56 +iters: 100 seeds: 0 1 2 out: /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results +end 2026-09-25 21:57:18 +control failed seeds: 0 +noclamp failed seeds: 0 +wide failed seeds: 0 +E29_DONE diff --git a/experiments/e29-clamp-or-noise/results/verdicts.txt b/experiments/e29-clamp-or-noise/results/verdicts.txt new file mode 100644 index 0000000..649bb06 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/verdicts.txt @@ -0,0 +1,47 @@ +P1 every arm runs end to end + control HOLDS (20 min) + noclamp HOLDS (20 min) + wide HOLDS (20 min) + +V1 the manipulation: each arm's logged entropy, every iteration + control expected -1.9200, logged -1.9198 to -1.9198 PASS + noclamp expected -1.9200, logged -1.9198 to -1.9198 PASS + wide expected +0.3826, logged 0.3828 to 0.3828 PASS + +P2 the control reproduces E28: gain at most +50 on every seed + control +10.6, +4.5, +9.7 + HOLDS + +P3 lifting the clamp alone lets it learn: gain at least +100 on 2 of 3 + noclamp +7.0, +3.0, +7.8 + 0 of 3 FALSIFIED + +P4 ten times the noise lets it learn: gain at least +100 on 2 of 3 + wide +73.6, +84.6, +106.2 + 1 of 3 FALSIFIED + +reported: update diagnostics, mean over iterations 1-10 and 91-100 + control kl 2.39e-07 -> 2.82e-06 + control clipfrac 1.46e-06 -> 1.79e-03 + noclamp kl 5.13e-07 -> 2.85e-06 + noclamp clipfrac 2.60e-06 -> 1.57e-03 + wide kl 2.50e-08 -> 1.48e-06 + wide clipfrac 0.00e+00 -> 1.02e-03 + +reported: the actor's relative movement, iteration 20 to 100, per seed + control 22.52%, 19.83%, 21.71% + noclamp 19.18%, 15.24%, 20.24% + wide 39.97%, 42.63%, 43.47% + +reported: first iteration and mean of the last ten, return and episode length + control seed 0 return 14.9 -> 25.5 length 20.2 -> 27.4 + control seed 1 return 19.7 -> 24.2 length 22.8 -> 25.3 + control seed 2 return 17.0 -> 26.7 length 21.9 -> 27.4 + noclamp seed 0 return 14.9 -> 21.9 length 20.2 -> 25.3 + noclamp seed 1 return 19.7 -> 22.7 length 22.8 -> 24.2 + noclamp seed 2 return 17.0 -> 24.8 length 21.9 -> 26.3 + wide seed 0 return 16.7 -> 90.3 length 22.7 -> 58.4 + wide seed 1 return 21.2 -> 105.7 length 23.6 -> 61.5 + wide seed 2 return 19.6 -> 125.8 length 23.8 -> 75.9 + +wrote /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/summary.tsv diff --git a/experiments/e29-clamp-or-noise/results/wide.out b/experiments/e29-clamp-or-noise/results/wide.out new file mode 100644 index 0000000..0f7946c --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide.out @@ -0,0 +1,11 @@ +cell: wide = fpo-policy/default x dppo/hopper x Hopper-v5 +iters: 100 x 4096 batch: variant default replan: 1 seeds: 0 1 2 +extra: --algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0 +out: /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide +start 2026-09-25 21:37:06 +seed 1 finished rc=0 at 21:57:02 +seed 0 finished rc=0 at 21:57:08 +seed 2 finished rc=0 at 21:57:18 +end 2026-09-25 21:57:18 +failed seeds: 0 +CELL_DONE wide diff --git a/experiments/e29-clamp-or-noise/results/wide/client-seed0.log b/experiments/e29-clamp-or-noise/results/wide/client-seed0.log new file mode 100644 index 0000000..d5c27f1 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/client-seed0.log @@ -0,0 +1,49 @@ +2026-09-25 21:37:08.346 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:37:08.347 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:37:08.347 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:37:08.347 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:37:08.348 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:37:08.389 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213708-f5deac66 output_dir=runs/mujoco-v1-nenv1-20260925-213708-f5deac66 recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:37:08.425 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9620... +2026-09-25 21:37:08.427 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:38.429 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10126 infer_calls=10126 feedback_calls=10126 infer_wait=26.922s infer_obs_pack=0.152s env_step=1.586s feedback_total=0.782s feedback_obs_pack=0.210s feedback_info_pack=0.042s effective_fps=343.92 +2026-09-25 21:38:08.430 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20397 infer_calls=20397 feedback_calls=20397 infer_wait=53.664s infer_obs_pack=0.328s env_step=3.258s feedback_total=1.600s feedback_obs_pack=0.429s feedback_info_pack=0.092s effective_fps=346.59 +2026-09-25 21:38:38.432 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30548 infer_calls=30548 feedback_calls=30548 infer_wait=80.451s infer_obs_pack=0.492s env_step=4.909s feedback_total=2.418s feedback_obs_pack=0.648s feedback_info_pack=0.139s effective_fps=346.07 +2026-09-25 21:39:08.434 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40719 infer_calls=40719 feedback_calls=40719 infer_wait=107.246s infer_obs_pack=0.660s env_step=6.559s feedback_total=3.230s feedback_obs_pack=0.867s feedback_info_pack=0.187s effective_fps=345.97 +2026-09-25 21:39:38.436 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=50799 infer_calls=50799 feedback_calls=50799 infer_wait=134.061s infer_obs_pack=0.824s env_step=8.196s feedback_total=4.053s feedback_obs_pack=1.086s feedback_info_pack=0.231s effective_fps=345.26 +2026-09-25 21:40:08.438 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61138 infer_calls=61138 feedback_calls=61138 infer_wait=160.773s infer_obs_pack=0.994s env_step=9.877s feedback_total=4.903s feedback_obs_pack=1.312s feedback_info_pack=0.283s effective_fps=346.30 +2026-09-25 21:40:38.439 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71315 infer_calls=71315 feedback_calls=71315 infer_wait=187.576s infer_obs_pack=1.157s env_step=11.520s feedback_total=5.716s feedback_obs_pack=1.530s feedback_info_pack=0.330s effective_fps=346.24 +2026-09-25 21:41:08.440 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81562 infer_calls=81562 feedback_calls=81562 infer_wait=214.347s infer_obs_pack=1.326s env_step=13.180s feedback_total=6.537s feedback_obs_pack=1.747s feedback_info_pack=0.376s effective_fps=346.50 +2026-09-25 21:41:38.441 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=91693 infer_calls=91693 feedback_calls=91693 infer_wait=241.147s infer_obs_pack=1.494s env_step=14.828s feedback_total=7.351s feedback_obs_pack=1.964s feedback_info_pack=0.420s effective_fps=346.25 +2026-09-25 21:42:08.442 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=101962 infer_calls=101962 feedback_calls=101962 infer_wait=267.909s infer_obs_pack=1.665s env_step=16.480s feedback_total=8.184s feedback_obs_pack=2.181s feedback_info_pack=0.465s effective_fps=346.53 +2026-09-25 21:42:38.444 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112068 infer_calls=112068 feedback_calls=112068 infer_wait=294.677s infer_obs_pack=1.829s env_step=18.165s feedback_total=8.990s feedback_obs_pack=2.405s feedback_info_pack=0.509s effective_fps=346.25 +2026-09-25 21:43:08.445 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122432 infer_calls=122432 feedback_calls=122432 infer_wait=321.421s infer_obs_pack=1.993s env_step=19.847s feedback_total=9.819s feedback_obs_pack=2.625s feedback_info_pack=0.562s effective_fps=346.75 +2026-09-25 21:43:38.448 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=132544 infer_calls=132544 feedback_calls=132544 infer_wait=348.226s infer_obs_pack=2.161s env_step=21.504s feedback_total=10.623s feedback_obs_pack=2.843s feedback_info_pack=0.607s effective_fps=346.51 +2026-09-25 21:44:08.449 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=142873 infer_calls=142873 feedback_calls=142873 infer_wait=374.955s infer_obs_pack=2.330s env_step=23.199s feedback_total=11.451s feedback_obs_pack=3.066s feedback_info_pack=0.655s effective_fps=346.83 +2026-09-25 21:44:38.450 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=153064 infer_calls=153064 feedback_calls=153064 infer_wait=401.790s infer_obs_pack=2.492s env_step=24.823s feedback_total=12.273s feedback_obs_pack=3.277s feedback_info_pack=0.698s effective_fps=346.79 +2026-09-25 21:45:08.456 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=163562 infer_calls=163562 feedback_calls=163562 infer_wait=428.565s infer_obs_pack=2.660s env_step=26.486s feedback_total=13.109s feedback_obs_pack=3.497s feedback_info_pack=0.745s effective_fps=347.40 +2026-09-25 21:45:38.456 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=173720 infer_calls=173720 feedback_calls=173720 infer_wait=455.407s infer_obs_pack=2.829s env_step=28.122s feedback_total=13.903s feedback_obs_pack=3.710s feedback_info_pack=0.789s effective_fps=347.26 +2026-09-25 21:46:08.458 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184065 infer_calls=184065 feedback_calls=184065 infer_wait=482.171s infer_obs_pack=2.996s env_step=29.801s feedback_total=14.730s feedback_obs_pack=3.926s feedback_info_pack=0.835s effective_fps=347.49 +2026-09-25 21:46:38.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=194205 infer_calls=194205 feedback_calls=194205 infer_wait=508.936s infer_obs_pack=3.166s env_step=31.472s feedback_total=15.562s feedback_obs_pack=4.146s feedback_info_pack=0.885s effective_fps=347.33 +2026-09-25 21:47:08.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=204588 infer_calls=204588 feedback_calls=204588 infer_wait=535.661s infer_obs_pack=3.337s env_step=33.163s feedback_total=16.395s feedback_obs_pack=4.363s feedback_info_pack=0.932s effective_fps=347.61 +2026-09-25 21:47:38.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=214652 infer_calls=214652 feedback_calls=214652 infer_wait=562.502s infer_obs_pack=3.498s env_step=34.801s feedback_total=17.210s feedback_obs_pack=4.583s feedback_info_pack=0.979s effective_fps=347.33 +2026-09-25 21:48:08.465 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=224940 infer_calls=224940 feedback_calls=224940 infer_wait=589.252s infer_obs_pack=3.664s env_step=36.504s feedback_total=18.020s feedback_obs_pack=4.808s feedback_info_pack=1.025s effective_fps=347.43 +2026-09-25 21:48:38.467 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=234997 infer_calls=234997 feedback_calls=234997 infer_wait=616.097s infer_obs_pack=3.829s env_step=38.157s feedback_total=18.809s feedback_obs_pack=5.022s feedback_info_pack=1.072s effective_fps=347.17 +2026-09-25 21:49:08.468 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=245365 infer_calls=245365 feedback_calls=245365 infer_wait=642.820s infer_obs_pack=3.998s env_step=39.865s feedback_total=19.649s feedback_obs_pack=5.243s feedback_info_pack=1.121s effective_fps=347.38 +2026-09-25 21:49:38.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=255519 infer_calls=255519 feedback_calls=255519 infer_wait=669.599s infer_obs_pack=4.167s env_step=41.546s feedback_total=20.466s feedback_obs_pack=5.460s feedback_info_pack=1.167s effective_fps=347.28 +2026-09-25 21:50:08.473 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=265930 infer_calls=265930 feedback_calls=265930 infer_wait=696.332s infer_obs_pack=4.330s env_step=43.253s feedback_total=21.305s feedback_obs_pack=5.682s feedback_info_pack=1.214s effective_fps=347.52 +2026-09-25 21:50:38.474 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=276325 infer_calls=276325 feedback_calls=276325 infer_wait=723.071s infer_obs_pack=4.500s env_step=44.940s feedback_total=22.152s feedback_obs_pack=5.899s feedback_info_pack=1.259s effective_fps=347.73 +2026-09-25 21:51:08.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=286632 infer_calls=286632 feedback_calls=286632 infer_wait=749.785s infer_obs_pack=4.671s env_step=46.667s feedback_total=22.976s feedback_obs_pack=6.125s feedback_info_pack=1.307s effective_fps=347.81 +2026-09-25 21:51:38.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=296975 infer_calls=296975 feedback_calls=296975 infer_wait=776.534s infer_obs_pack=4.845s env_step=48.358s feedback_total=23.820s feedback_obs_pack=6.350s feedback_info_pack=1.351s effective_fps=347.93 +2026-09-25 21:52:08.477 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=307242 infer_calls=307242 feedback_calls=307242 infer_wait=803.348s infer_obs_pack=5.014s env_step=50.026s feedback_total=24.635s feedback_obs_pack=6.562s feedback_info_pack=1.397s effective_fps=347.94 +2026-09-25 21:52:38.479 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=317515 infer_calls=317515 feedback_calls=317515 infer_wait=830.120s infer_obs_pack=5.189s env_step=51.725s feedback_total=25.443s feedback_obs_pack=6.784s feedback_info_pack=1.446s effective_fps=347.97 +2026-09-25 21:53:08.949 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=327682 infer_calls=327682 feedback_calls=327682 infer_wait=857.409s infer_obs_pack=5.358s env_step=53.404s feedback_total=26.230s feedback_obs_pack=6.998s feedback_info_pack=1.488s effective_fps=347.71 +2026-09-25 21:53:38.951 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=337934 infer_calls=337934 feedback_calls=337934 infer_wait=884.201s infer_obs_pack=5.521s env_step=55.096s feedback_total=27.040s feedback_obs_pack=7.218s feedback_info_pack=1.536s effective_fps=347.72 +2026-09-25 21:54:08.953 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=348288 infer_calls=348288 feedback_calls=348288 infer_wait=910.989s infer_obs_pack=5.691s env_step=56.754s feedback_total=27.886s feedback_obs_pack=7.434s feedback_info_pack=1.579s effective_fps=347.83 +2026-09-25 21:54:38.956 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=358652 infer_calls=358652 feedback_calls=358652 infer_wait=937.756s infer_obs_pack=5.857s env_step=58.457s feedback_total=28.707s feedback_obs_pack=7.655s feedback_info_pack=1.624s effective_fps=347.94 +2026-09-25 21:55:08.957 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=368913 infer_calls=368913 feedback_calls=368913 infer_wait=964.602s infer_obs_pack=6.020s env_step=60.107s feedback_total=29.517s feedback_obs_pack=7.869s feedback_info_pack=1.669s effective_fps=347.95 +2026-09-25 21:55:38.958 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=379251 infer_calls=379251 feedback_calls=379251 infer_wait=991.371s infer_obs_pack=6.182s env_step=61.789s feedback_total=30.354s feedback_obs_pack=8.095s feedback_info_pack=1.713s effective_fps=348.03 +2026-09-25 21:56:08.959 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=389380 infer_calls=389380 feedback_calls=389380 infer_wait=1018.199s infer_obs_pack=6.348s env_step=63.452s feedback_total=31.169s feedback_obs_pack=8.315s feedback_info_pack=1.760s effective_fps=347.92 +2026-09-25 21:56:38.960 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=399789 infer_calls=399789 feedback_calls=399789 infer_wait=1045.012s infer_obs_pack=6.517s env_step=65.103s feedback_total=32.001s feedback_obs_pack=8.536s feedback_info_pack=1.805s effective_fps=348.06 +2026-09-25 21:57:08.549 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1071.349s infer_obs_pack=6.659s env_step=66.542s feedback_total=32.712s feedback_obs_pack=8.723s feedback_info_pack=1.841s effective_fps=347.93 +2026-09-25 21:57:08.550 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 12717/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/wide/client-seed1.log b/experiments/e29-clamp-or-noise/results/wide/client-seed1.log new file mode 100644 index 0000000..7fd758c --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/client-seed1.log @@ -0,0 +1,49 @@ +2026-09-25 21:37:13.350 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:37:13.350 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:37:13.350 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:37:13.351 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:37:13.351 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:37:13.393 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213713-4e298fd8 output_dir=runs/mujoco-v1-nenv1-20260925-213713-4e298fd8 recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:37:13.431 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9621... +2026-09-25 21:37:13.433 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:43.436 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10220 infer_calls=10220 feedback_calls=10220 infer_wait=26.912s infer_obs_pack=0.159s env_step=1.589s feedback_total=0.792s feedback_obs_pack=0.206s feedback_info_pack=0.047s effective_fps=347.01 +2026-09-25 21:38:13.457 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20482 infer_calls=20482 feedback_calls=20482 infer_wait=53.712s infer_obs_pack=0.337s env_step=3.234s feedback_total=1.622s feedback_obs_pack=0.427s feedback_info_pack=0.093s effective_fps=347.71 +2026-09-25 21:38:43.459 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=31025 infer_calls=31025 feedback_calls=31025 infer_wait=80.466s infer_obs_pack=0.509s env_step=4.898s feedback_total=2.458s feedback_obs_pack=0.650s feedback_info_pack=0.142s effective_fps=351.24 +2026-09-25 21:39:13.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=41175 infer_calls=41175 feedback_calls=41175 infer_wait=107.287s infer_obs_pack=0.673s env_step=6.524s feedback_total=3.285s feedback_obs_pack=0.864s feedback_info_pack=0.191s effective_fps=349.62 +2026-09-25 21:39:43.461 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=51535 infer_calls=51535 feedback_calls=51535 infer_wait=134.002s infer_obs_pack=0.842s env_step=8.211s feedback_total=4.137s feedback_obs_pack=1.092s feedback_info_pack=0.239s effective_fps=350.12 +2026-09-25 21:40:13.464 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61864 infer_calls=61864 feedback_calls=61864 infer_wait=160.738s infer_obs_pack=1.017s env_step=9.886s feedback_total=4.974s feedback_obs_pack=1.315s feedback_info_pack=0.286s effective_fps=350.28 +2026-09-25 21:40:43.466 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=72335 infer_calls=72335 feedback_calls=72335 infer_wait=187.448s infer_obs_pack=1.191s env_step=11.554s feedback_total=5.839s feedback_obs_pack=1.540s feedback_info_pack=0.337s effective_fps=351.09 +2026-09-25 21:41:13.468 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=82425 infer_calls=82425 feedback_calls=82425 infer_wait=214.293s infer_obs_pack=1.362s env_step=13.177s feedback_total=6.642s feedback_obs_pack=1.759s feedback_info_pack=0.383s effective_fps=350.04 +2026-09-25 21:41:43.469 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=92841 infer_calls=92841 feedback_calls=92841 infer_wait=241.015s infer_obs_pack=1.535s env_step=14.862s feedback_total=7.489s feedback_obs_pack=1.985s feedback_info_pack=0.432s effective_fps=350.48 +2026-09-25 21:42:13.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=103043 infer_calls=103043 feedback_calls=103043 infer_wait=267.877s infer_obs_pack=1.695s env_step=16.484s feedback_total=8.291s feedback_obs_pack=2.199s feedback_info_pack=0.479s effective_fps=350.07 +2026-09-25 21:42:43.471 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=113621 infer_calls=113621 feedback_calls=113621 infer_wait=294.576s infer_obs_pack=1.866s env_step=18.181s feedback_total=9.151s feedback_obs_pack=2.425s feedback_info_pack=0.526s effective_fps=350.93 +2026-09-25 21:43:13.472 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=123854 infer_calls=123854 feedback_calls=123854 infer_wait=321.359s infer_obs_pack=2.033s env_step=19.845s feedback_total=9.972s feedback_obs_pack=2.644s feedback_info_pack=0.573s effective_fps=350.65 +2026-09-25 21:43:43.474 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=134227 infer_calls=134227 feedback_calls=134227 infer_wait=348.081s infer_obs_pack=2.207s env_step=21.527s feedback_total=10.824s feedback_obs_pack=2.872s feedback_info_pack=0.620s effective_fps=350.79 +2026-09-25 21:44:13.475 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=144516 infer_calls=144516 feedback_calls=144516 infer_wait=374.860s infer_obs_pack=2.376s env_step=23.187s feedback_total=11.654s feedback_obs_pack=3.091s feedback_info_pack=0.666s effective_fps=350.70 +2026-09-25 21:44:43.477 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=154936 infer_calls=154936 feedback_calls=154936 infer_wait=401.602s infer_obs_pack=2.544s env_step=24.857s feedback_total=12.500s feedback_obs_pack=3.314s feedback_info_pack=0.714s effective_fps=350.93 +2026-09-25 21:45:13.478 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=165262 infer_calls=165262 feedback_calls=165262 infer_wait=428.392s infer_obs_pack=2.714s env_step=26.502s feedback_total=13.335s feedback_obs_pack=3.547s feedback_info_pack=0.763s effective_fps=350.92 +2026-09-25 21:45:43.479 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=175709 infer_calls=175709 feedback_calls=175709 infer_wait=455.109s infer_obs_pack=2.889s env_step=28.201s feedback_total=14.180s feedback_obs_pack=3.771s feedback_info_pack=0.811s effective_fps=351.15 +2026-09-25 21:46:13.480 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=185823 infer_calls=185823 feedback_calls=185823 infer_wait=481.926s infer_obs_pack=3.059s env_step=29.852s feedback_total=14.984s feedback_obs_pack=3.989s feedback_info_pack=0.857s effective_fps=350.73 +2026-09-25 21:46:43.480 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=196317 infer_calls=196317 feedback_calls=196317 infer_wait=508.636s infer_obs_pack=3.235s env_step=31.535s feedback_total=15.839s feedback_obs_pack=4.217s feedback_info_pack=0.904s effective_fps=351.04 +2026-09-25 21:47:13.481 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=206548 infer_calls=206548 feedback_calls=206548 infer_wait=535.419s infer_obs_pack=3.406s env_step=33.184s feedback_total=16.675s feedback_obs_pack=4.439s feedback_info_pack=0.953s effective_fps=350.86 +2026-09-25 21:47:43.483 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=216866 infer_calls=216866 feedback_calls=216866 infer_wait=562.184s infer_obs_pack=3.585s env_step=34.863s feedback_total=17.500s feedback_obs_pack=4.669s feedback_info_pack=1.000s effective_fps=350.84 +2026-09-25 21:48:13.484 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=227025 infer_calls=227025 feedback_calls=227025 infer_wait=589.006s infer_obs_pack=3.753s env_step=36.497s feedback_total=18.321s feedback_obs_pack=4.888s feedback_info_pack=1.046s effective_fps=350.58 +2026-09-25 21:48:43.485 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=237526 infer_calls=237526 feedback_calls=237526 infer_wait=615.729s infer_obs_pack=3.927s env_step=38.182s feedback_total=19.180s feedback_obs_pack=5.111s feedback_info_pack=1.095s effective_fps=350.84 +2026-09-25 21:49:13.485 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=247759 infer_calls=247759 feedback_calls=247759 infer_wait=642.525s infer_obs_pack=4.096s env_step=39.843s feedback_total=20.001s feedback_obs_pack=5.333s feedback_info_pack=1.140s effective_fps=350.70 +2026-09-25 21:49:43.487 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=258071 infer_calls=258071 feedback_calls=258071 infer_wait=669.264s infer_obs_pack=4.270s env_step=41.532s feedback_total=20.840s feedback_obs_pack=5.560s feedback_info_pack=1.189s effective_fps=350.68 +2026-09-25 21:50:13.487 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=268492 infer_calls=268492 feedback_calls=268492 infer_wait=695.997s infer_obs_pack=4.444s env_step=43.228s feedback_total=21.686s feedback_obs_pack=5.782s feedback_info_pack=1.239s effective_fps=350.81 +2026-09-25 21:50:43.488 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=278768 infer_calls=278768 feedback_calls=278768 infer_wait=722.779s infer_obs_pack=4.614s env_step=44.895s feedback_total=22.516s feedback_obs_pack=6.007s feedback_info_pack=1.285s effective_fps=350.74 +2026-09-25 21:51:13.488 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=289241 infer_calls=289241 feedback_calls=289241 infer_wait=749.491s infer_obs_pack=4.786s env_step=46.612s feedback_total=23.359s feedback_obs_pack=6.233s feedback_info_pack=1.332s effective_fps=350.91 +2026-09-25 21:51:43.489 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=299619 infer_calls=299619 feedback_calls=299619 infer_wait=776.255s infer_obs_pack=4.956s env_step=48.299s feedback_total=24.187s feedback_obs_pack=6.457s feedback_info_pack=1.379s effective_fps=350.97 +2026-09-25 21:52:13.491 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=310068 infer_calls=310068 feedback_calls=310068 infer_wait=802.978s infer_obs_pack=5.129s env_step=50.013s feedback_total=25.029s feedback_obs_pack=6.689s feedback_info_pack=1.426s effective_fps=351.09 +2026-09-25 21:52:43.495 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=320355 infer_calls=320355 feedback_calls=320355 infer_wait=829.755s infer_obs_pack=5.297s env_step=51.692s feedback_total=25.860s feedback_obs_pack=6.911s feedback_info_pack=1.473s effective_fps=351.03 +2026-09-25 21:53:13.498 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=330738 infer_calls=330738 feedback_calls=330738 infer_wait=856.517s infer_obs_pack=5.464s env_step=53.381s feedback_total=26.697s feedback_obs_pack=7.131s feedback_info_pack=1.521s effective_fps=351.08 +2026-09-25 21:53:43.500 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=341086 infer_calls=341086 feedback_calls=341086 infer_wait=883.307s infer_obs_pack=5.634s env_step=55.028s feedback_total=27.552s feedback_obs_pack=7.355s feedback_info_pack=1.567s effective_fps=351.08 +2026-09-25 21:54:13.504 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=351532 infer_calls=351532 feedback_calls=351532 infer_wait=909.982s infer_obs_pack=5.813s env_step=56.749s feedback_total=28.418s feedback_obs_pack=7.583s feedback_info_pack=1.615s effective_fps=351.19 +2026-09-25 21:54:43.506 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=361826 infer_calls=361826 feedback_calls=361826 infer_wait=936.765s infer_obs_pack=5.982s env_step=58.417s feedback_total=29.255s feedback_obs_pack=7.807s feedback_info_pack=1.662s effective_fps=351.14 +2026-09-25 21:55:13.507 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=372193 infer_calls=372193 feedback_calls=372193 infer_wait=963.553s infer_obs_pack=6.151s env_step=60.090s feedback_total=30.086s feedback_obs_pack=8.026s feedback_info_pack=1.708s effective_fps=351.16 +2026-09-25 21:55:43.509 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=382537 infer_calls=382537 feedback_calls=382537 infer_wait=990.369s infer_obs_pack=6.319s env_step=61.746s feedback_total=30.925s feedback_obs_pack=8.240s feedback_info_pack=1.756s effective_fps=351.16 +2026-09-25 21:56:13.511 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=393019 infer_calls=393019 feedback_calls=393019 infer_wait=1017.106s infer_obs_pack=6.495s env_step=63.433s feedback_total=31.775s feedback_obs_pack=8.462s feedback_info_pack=1.802s effective_fps=351.28 +2026-09-25 21:56:43.512 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=403400 infer_calls=403400 feedback_calls=403400 infer_wait=1043.977s infer_obs_pack=6.661s env_step=65.036s feedback_total=32.611s feedback_obs_pack=8.675s feedback_info_pack=1.847s effective_fps=351.31 +2026-09-25 21:57:02.542 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1060.771s infer_obs_pack=6.753s env_step=65.942s feedback_total=33.054s feedback_obs_pack=8.794s feedback_info_pack=1.871s effective_fps=351.13 +2026-09-25 21:57:02.543 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 12503/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/wide/client-seed2.log b/experiments/e29-clamp-or-noise/results/wide/client-seed2.log new file mode 100644 index 0000000..a79b757 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/client-seed2.log @@ -0,0 +1,49 @@ +2026-09-25 21:37:18.353 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.classic.classic_env: pygame is not installed. Please install it with pip install "plugrl-env-client[classic]". +2026-09-25 21:37:18.353 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.atari.atari_env: Atari is not installed. Please install it with the 'atari' extra, e.g. 'pip install plugrl-env-client[atari]' +2026-09-25 21:37:18.353 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.d4rl.d4rl_env: d4rl is not installed. Please install it with pip install "plugrl-env-client[d4rl]". +2026-09-25 21:37:18.354 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.libero.libero_env: libero is not installed. Please install it with pip install "plugrl-env-client[libero]". +2026-09-25 21:37:18.354 | WARNING | plugrl_env_client.envs:_load_env_modules:27 - Skip loading env module plugrl_env_client.envs.robomimic.robomimic_env: Robomimic is not installed. Please install it with the 'robomimic' extra, e.g. 'pip install plugrl-env-client[robomimic]' +2026-09-25 21:37:18.396 | INFO | __main__:main:83 - Starting env client exp_name=mujoco-v1-nenv1-20260925-213718-93926e3a output_dir=runs/mujoco-v1-nenv1-20260925-213718-93926e3a recorder={'episode_freq': 0, 'thread0_only': True, 'record_video': False, 'video_fps': 30.0, 'record_full_rollout': False, 'record_obs_stats': True, 'record_episode_metrics': True, 'metric_window': 100} +2026-09-25 21:37:18.433 | INFO | plugrl_env_client.agent.websocket_env_client_agent:_wait_for_server:67 - Waiting for server at ws://127.0.0.1:9622... +2026-09-25 21:37:18.435 | INFO | plugrl_env_client.runner.run:report_server_metadata:56 - Server metadata: {'protocol_version': 1, 'server': 'plugrl-server', 'server_version': '0.1.0', 'algorithm': 'DPPOAlgorithm', 'policy': 'FPOPolicy', 'action_dim': 3, 'action_horizon': 1} +2026-09-25 21:37:48.438 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=10119 infer_calls=10119 feedback_calls=10119 infer_wait=26.894s infer_obs_pack=0.158s env_step=1.595s feedback_total=0.804s feedback_obs_pack=0.218s feedback_info_pack=0.046s effective_fps=343.57 +2026-09-25 21:38:18.769 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=20482 infer_calls=20482 feedback_calls=20482 infer_wait=54.009s infer_obs_pack=0.323s env_step=3.233s feedback_total=1.655s feedback_obs_pack=0.451s feedback_info_pack=0.094s effective_fps=345.87 +2026-09-25 21:38:48.771 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=30797 infer_calls=30797 feedback_calls=30797 infer_wait=80.811s infer_obs_pack=0.486s env_step=4.884s feedback_total=2.476s feedback_obs_pack=0.675s feedback_info_pack=0.140s effective_fps=347.37 +2026-09-25 21:39:19.153 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=40962 infer_calls=40962 feedback_calls=40962 infer_wait=108.037s infer_obs_pack=0.648s env_step=6.497s feedback_total=3.303s feedback_obs_pack=0.892s feedback_info_pack=0.187s effective_fps=345.71 +2026-09-25 21:39:49.154 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=51220 infer_calls=51220 feedback_calls=51220 infer_wait=134.815s infer_obs_pack=0.817s env_step=8.149s feedback_total=4.136s feedback_obs_pack=1.118s feedback_info_pack=0.233s effective_fps=346.28 +2026-09-25 21:40:19.189 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=61442 infer_calls=61442 feedback_calls=61442 infer_wait=161.719s infer_obs_pack=0.988s env_step=9.754s feedback_total=4.945s feedback_obs_pack=1.336s feedback_info_pack=0.281s effective_fps=346.34 +2026-09-25 21:40:49.191 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=71765 infer_calls=71765 feedback_calls=71765 infer_wait=188.526s infer_obs_pack=1.153s env_step=11.406s feedback_total=5.765s feedback_obs_pack=1.556s feedback_info_pack=0.328s effective_fps=346.94 +2026-09-25 21:41:19.484 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=81922 infer_calls=81922 feedback_calls=81922 infer_wait=215.683s infer_obs_pack=1.317s env_step=13.019s feedback_total=6.572s feedback_obs_pack=1.773s feedback_info_pack=0.374s effective_fps=346.26 +2026-09-25 21:41:49.484 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=92148 infer_calls=92148 feedback_calls=92148 infer_wait=242.481s infer_obs_pack=1.489s env_step=14.672s feedback_total=7.386s feedback_obs_pack=1.993s feedback_info_pack=0.419s effective_fps=346.39 +2026-09-25 21:42:19.486 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=102372 infer_calls=102372 feedback_calls=102372 infer_wait=269.306s infer_obs_pack=1.653s env_step=16.320s feedback_total=8.205s feedback_obs_pack=2.219s feedback_info_pack=0.469s effective_fps=346.46 +2026-09-25 21:42:49.489 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=112594 infer_calls=112594 feedback_calls=112594 infer_wait=296.088s infer_obs_pack=1.823s env_step=17.973s feedback_total=9.046s feedback_obs_pack=2.442s feedback_info_pack=0.515s effective_fps=346.52 +2026-09-25 21:43:19.491 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=122957 infer_calls=122957 feedback_calls=122957 infer_wait=322.922s infer_obs_pack=1.987s env_step=19.589s feedback_total=9.889s feedback_obs_pack=2.662s feedback_info_pack=0.561s effective_fps=346.96 +2026-09-25 21:43:49.491 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=133318 infer_calls=133318 feedback_calls=133318 infer_wait=349.692s infer_obs_pack=2.153s env_step=21.255s feedback_total=10.722s feedback_obs_pack=2.884s feedback_info_pack=0.608s effective_fps=347.34 +2026-09-25 21:44:19.493 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=143572 infer_calls=143572 feedback_calls=143572 infer_wait=376.500s infer_obs_pack=2.320s env_step=22.900s feedback_total=11.555s feedback_obs_pack=3.104s feedback_info_pack=0.656s effective_fps=347.40 +2026-09-25 21:44:49.494 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=153950 infer_calls=153950 feedback_calls=153950 infer_wait=403.236s infer_obs_pack=2.496s env_step=24.589s feedback_total=12.394s feedback_obs_pack=3.328s feedback_info_pack=0.705s effective_fps=347.74 +2026-09-25 21:45:19.497 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=164029 infer_calls=164029 feedback_calls=164029 infer_wait=430.075s infer_obs_pack=2.658s env_step=26.234s feedback_total=13.193s feedback_obs_pack=3.542s feedback_info_pack=0.752s effective_fps=347.40 +2026-09-25 21:45:49.498 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=174381 infer_calls=174381 feedback_calls=174381 infer_wait=456.829s infer_obs_pack=2.830s env_step=27.902s feedback_total=14.035s feedback_obs_pack=3.760s feedback_info_pack=0.799s effective_fps=347.65 +2026-09-25 21:46:19.500 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=184563 infer_calls=184563 feedback_calls=184563 infer_wait=483.663s infer_obs_pack=2.997s env_step=29.537s feedback_total=14.854s feedback_obs_pack=3.974s feedback_info_pack=0.846s effective_fps=347.54 +2026-09-25 21:46:49.503 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=194960 infer_calls=194960 feedback_calls=194960 infer_wait=510.414s infer_obs_pack=3.163s env_step=31.224s feedback_total=15.691s feedback_obs_pack=4.197s feedback_info_pack=0.893s effective_fps=347.84 +2026-09-25 21:47:19.504 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=205193 infer_calls=205193 feedback_calls=205193 infer_wait=537.279s infer_obs_pack=3.325s env_step=32.834s feedback_total=16.509s feedback_obs_pack=4.418s feedback_info_pack=0.937s effective_fps=347.82 +2026-09-25 21:47:49.504 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=215512 infer_calls=215512 feedback_calls=215512 infer_wait=564.067s infer_obs_pack=3.496s env_step=34.496s feedback_total=17.338s feedback_obs_pack=4.640s feedback_info_pack=0.987s effective_fps=347.94 +2026-09-25 21:48:19.504 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=225620 infer_calls=225620 feedback_calls=225620 infer_wait=590.897s infer_obs_pack=3.659s env_step=36.145s feedback_total=18.155s feedback_obs_pack=4.859s feedback_info_pack=1.034s effective_fps=347.72 +2026-09-25 21:48:49.506 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=235943 infer_calls=235943 feedback_calls=235943 infer_wait=617.644s infer_obs_pack=3.832s env_step=37.851s feedback_total=18.976s feedback_obs_pack=5.087s feedback_info_pack=1.080s effective_fps=347.84 +2026-09-25 21:49:19.509 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=246136 infer_calls=246136 feedback_calls=246136 infer_wait=644.444s infer_obs_pack=4.002s env_step=39.520s feedback_total=19.794s feedback_obs_pack=5.309s feedback_info_pack=1.125s effective_fps=347.77 +2026-09-25 21:49:49.509 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=256511 infer_calls=256511 feedback_calls=256511 infer_wait=671.170s infer_obs_pack=4.175s env_step=41.226s feedback_total=20.639s feedback_obs_pack=5.540s feedback_info_pack=1.174s effective_fps=347.95 +2026-09-25 21:50:19.510 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=266710 infer_calls=266710 feedback_calls=266710 infer_wait=698.004s infer_obs_pack=4.344s env_step=42.873s feedback_total=21.458s feedback_obs_pack=5.755s feedback_info_pack=1.218s effective_fps=347.88 +2026-09-25 21:50:49.513 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=277195 infer_calls=277195 feedback_calls=277195 infer_wait=724.727s infer_obs_pack=4.516s env_step=44.577s feedback_total=22.315s feedback_obs_pack=5.979s feedback_info_pack=1.266s effective_fps=348.18 +2026-09-25 21:51:19.513 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=287413 infer_calls=287413 feedback_calls=287413 infer_wait=751.569s infer_obs_pack=4.681s env_step=46.231s feedback_total=23.122s feedback_obs_pack=6.199s feedback_info_pack=1.310s effective_fps=348.13 +2026-09-25 21:51:49.516 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=297774 infer_calls=297774 feedback_calls=297774 infer_wait=778.313s infer_obs_pack=4.854s env_step=47.927s feedback_total=23.972s feedback_obs_pack=6.421s feedback_info_pack=1.356s effective_fps=348.25 +2026-09-25 21:52:19.517 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=307960 infer_calls=307960 feedback_calls=307960 infer_wait=805.126s infer_obs_pack=5.018s env_step=49.602s feedback_total=24.790s feedback_obs_pack=6.641s feedback_info_pack=1.404s effective_fps=348.16 +2026-09-25 21:52:49.519 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=318209 infer_calls=318209 feedback_calls=318209 infer_wait=831.920s infer_obs_pack=5.186s env_step=51.278s feedback_total=25.623s feedback_obs_pack=6.864s feedback_info_pack=1.451s effective_fps=348.15 +2026-09-25 21:53:19.520 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=328324 infer_calls=328324 feedback_calls=328324 infer_wait=858.766s infer_obs_pack=5.356s env_step=52.922s feedback_total=26.438s feedback_obs_pack=7.083s feedback_info_pack=1.496s effective_fps=347.99 +2026-09-25 21:53:49.521 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=338510 infer_calls=338510 feedback_calls=338510 infer_wait=885.552s infer_obs_pack=5.523s env_step=54.599s feedback_total=27.272s feedback_obs_pack=7.309s feedback_info_pack=1.541s effective_fps=347.92 +2026-09-25 21:54:19.523 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=348717 infer_calls=348717 feedback_calls=348717 infer_wait=912.391s infer_obs_pack=5.692s env_step=56.245s feedback_total=28.097s feedback_obs_pack=7.529s feedback_info_pack=1.589s effective_fps=347.87 +2026-09-25 21:54:49.525 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=358997 infer_calls=358997 feedback_calls=358997 infer_wait=939.219s infer_obs_pack=5.864s env_step=57.892s feedback_total=28.917s feedback_obs_pack=7.749s feedback_info_pack=1.639s effective_fps=347.90 +2026-09-25 21:55:19.526 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=369069 infer_calls=369069 feedback_calls=369069 infer_wait=966.058s infer_obs_pack=6.039s env_step=59.550s feedback_total=29.722s feedback_obs_pack=7.971s feedback_info_pack=1.685s effective_fps=347.73 +2026-09-25 21:55:49.527 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=379394 infer_calls=379394 feedback_calls=379394 infer_wait=992.825s infer_obs_pack=6.211s env_step=61.245s feedback_total=30.555s feedback_obs_pack=8.190s feedback_info_pack=1.734s effective_fps=347.80 +2026-09-25 21:56:19.529 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=389629 infer_calls=389629 feedback_calls=389629 infer_wait=1019.659s infer_obs_pack=6.380s env_step=62.901s feedback_total=31.381s feedback_obs_pack=8.413s feedback_info_pack=1.781s effective_fps=347.78 +2026-09-25 21:56:49.531 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Intermediate rollout timing summary: env_steps=399911 infer_calls=399911 feedback_calls=399911 infer_wait=1046.556s infer_obs_pack=6.544s env_step=64.521s feedback_total=32.166s feedback_obs_pack=8.630s feedback_info_pack=1.828s effective_fps=347.81 +2026-09-25 21:57:18.645 | INFO | plugrl_env_client.runner.rollout:log_timing_summary:98 - Final rollout timing summary: env_steps=409601 infer_calls=409601 feedback_calls=409601 infer_wait=1072.509s infer_obs_pack=6.684s env_step=65.901s feedback_total=32.877s feedback_obs_pack=8.811s feedback_info_pack=1.863s effective_fps=347.72 +2026-09-25 21:57:18.646 | INFO | plugrl_env_client.runner.run:run:156 - Server signalled the end of the run; collection stopped after 12031/1638400 episodes. diff --git a/experiments/e29-clamp-or-noise/results/wide/server-seed0.log b/experiments/e29-clamp-or-noise/results/wide/server-seed0.log new file mode 100644 index 0000000..4d07bf7 --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/server-seed0.log @@ -0,0 +1,25 @@ +21:37:08|INFO|plugrl_server version: 0.1.0 +21:37:08|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=1.0, sampling_noise_level=1.0, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:37:08|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:37:08|INFO|Seeded python, numpy and torch with 0. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:37:08|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0 +21:37:08|INFO|Policy created... +21:37:08|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:37:08|INFO|Algorithm created: + +21:37:08|INFO|Agent Server is listening on 0.0.0.0:9620 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:41:09|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0/81921 +21:45:09|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0/163841 +21:49:10|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0/245761 +21:53:08|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0/327681 +21:57:08|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed0/409601 +21:57:08|INFO|Stopping server as the algorithm signaled to stop. +21:57:08|INFO|Shutdown started: aborting pending infer requests. +21:57:08|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:57:08|INFO|Shutdown closing 1 websocket connection(s). +21:57:08|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 48606). +21:57:08|INFO|WebSocket server closed. +21:57:08|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/wide/server-seed1.log b/experiments/e29-clamp-or-noise/results/wide/server-seed1.log new file mode 100644 index 0000000..5a892eb --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/server-seed1.log @@ -0,0 +1,25 @@ +21:37:13|INFO|plugrl_server version: 0.1.0 +21:37:13|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=1.0, sampling_noise_level=1.0, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:37:13|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:37:13|INFO|Seeded python, numpy and torch with 1. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:37:13|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1 +21:37:13|INFO|Policy created... +21:37:13|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:37:13|INFO|Algorithm created: + +21:37:13|INFO|Agent Server is listening on 0.0.0.0:9621 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:41:12|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1/81921 +21:45:09|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1/163841 +21:49:08|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1/245761 +21:53:04|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1/327681 +21:57:02|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed1/409601 +21:57:02|INFO|Stopping server as the algorithm signaled to stop. +21:57:02|INFO|Shutdown started: aborting pending infer requests. +21:57:02|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:57:02|INFO|Shutdown closing 1 websocket connection(s). +21:57:02|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 33064). +21:57:02|INFO|WebSocket server closed. +21:57:02|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/results/wide/server-seed2.log b/experiments/e29-clamp-or-noise/results/wide/server-seed2.log new file mode 100644 index 0000000..5356b9d --- /dev/null +++ b/experiments/e29-clamp-or-noise/results/wide/server-seed2.log @@ -0,0 +1,25 @@ +21:37:18|INFO|plugrl_server version: 0.1.0 +21:37:18|INFO|Algorithm: dppo, Config: DPPOAlgoConfigHopper(global_steps=409600, gamma=0.99, gamma_denoising=0.99, actor_lr=0.0001, critic_lr=0.001, actor_weight_decay=0.0, critic_weight_decay=0.0, actor_lr_scheduler=SchedulerConfig(min_lr=0.0001, warmup_steps=10), critic_lr_scheduler=SchedulerConfig(min_lr=0.001, warmup_steps=10), buffer_size=4096, gae_lambda=0.95, update_epochs=5, norm_adv=True, clip_ploss_coef=0.01, clip_ploss_coef_base=0.01, clip_ploss_coef_rate=3, clip_vloss_coef=inf, ent_coef=0.0, vf_coef=0.5, max_grad_norm=inf, target_kl=inf, logprob_noise_level=1.0, sampling_noise_level=1.0, logprob_clamp_min=-5.0, logprob_clamp_max=2.0, clip_advantage_lower_quantile=0, clip_advantage_upper_quantile=1, n_critic_warmup_itrs=0, use_normalized_rewards=True, batch_size=2048, train_itrs=100, save_interval=20, grad_accum_steps=8, policy_checkpoint_path=None, restore='all') +21:37:18|INFO|Policy: fpo-policy, Config: FPOPolicyConfig(algo='dppo', device=device(type='cpu'), obs_dim=11, action_dim=3, flow_steps=10, timestep_embed_dim=8, action_horizon=1, feather_std=0.0, policy_mlp_output_scale=0.25, normalize_observations=True, hidden_dims=(32, 32, 32, 32), value_hidden_dims=(256, 256, 256, 256, 256)) +21:37:18|INFO|Seeded python, numpy and torch with 2. One env client is then reproducible; several are not, because batch composition depends on when their requests arrive. +21:37:18|INFO|Checkpoint Manager created: + at /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2 +21:37:18|INFO|Policy created... +21:37:18|INFO|Initialized RolloutBuffer buffer_size=4096 action_shape=(4096, 10, 1, 3) value_shape=(4096,) +21:37:18|INFO|Algorithm created: + +21:37:18|INFO|Agent Server is listening on 0.0.0.0:9622 +/home/guangzhao/zuogou/plugrl/plugrl-server/.venv/lib/python3.11/site-packages/torch/utils/data/dataloader.py:665: UserWarning: 'pin_memory' argument is set as true but no accelerator is found, then device pinned memory won't be used. + warnings.warn(warn_msg) +21:41:19|INFO|Checkpoint saved at step 81921 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2/81921 +21:45:18|INFO|Checkpoint saved at step 163841 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2/163841 +21:49:18|INFO|Checkpoint saved at step 245761 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2/245761 +21:53:17|INFO|Checkpoint saved at step 327681 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2/327681 +21:57:18|INFO|Checkpoint saved at step 409601 to /home/guangzhao/zuogou/plugrl/e29/experiments/e29-clamp-or-noise/results/wide/dppo/fpo-policy/wide-seed2/409601 +21:57:18|INFO|Stopping server as the algorithm signaled to stop. +21:57:18|INFO|Shutdown started: aborting pending infer requests. +21:57:18|INFO|Shutdown cleanup finished: pending_futures=1, drained_requests=1 +21:57:18|INFO|Shutdown closing 1 websocket connection(s). +21:57:18|INFO|Shutdown interrupted an in-flight inference request from ('127.0.0.1', 52020). +21:57:18|INFO|WebSocket server closed. +21:57:18|INFO|Scheduler task cancelled and cleaned up. diff --git a/experiments/e29-clamp-or-noise/summary.tsv b/experiments/e29-clamp-or-noise/summary.tsv new file mode 100644 index 0000000..88fd1d0 --- /dev/null +++ b/experiments/e29-clamp-or-noise/summary.tsv @@ -0,0 +1,10 @@ +arm seed iterations runs first_return last10_return gain first_length last10_length +control 0 100 true 14.9 25.5 10.6 20.2 27.4 +control 1 100 true 19.7 24.2 4.5 22.8 25.3 +control 2 100 true 17.0 26.7 9.7 21.9 27.4 +noclamp 0 100 true 14.9 21.9 7.0 20.2 25.3 +noclamp 1 100 true 19.7 22.7 3.0 22.8 24.2 +noclamp 2 100 true 17.0 24.8 7.8 21.9 26.3 +wide 0 100 true 16.7 90.3 73.6 22.7 58.4 +wide 1 100 true 21.2 105.7 84.6 23.6 61.5 +wide 2 100 true 19.6 125.8 106.2 23.8 75.9