From ca184c899afc0e8b7fae9026077655e4fc322f5a Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Sat, 3 Oct 2026 04:58:34 -0400 Subject: [PATCH 1/2] README: crossing costs once the observation's bytes (E46), other trainers (E48, E49), the boundary is transparent (E50, E52, E53); E49's figure - README: the cost row now cites E46 (11.6 ms for 184 KiB at 23.7 MB/s, each observation once with reuse-feedback-obs) instead of E43's twice-the-bytes model; the E10 ratio is 12% on that link, not 21%. New rows for E49/E48 and E50/E52/E53. The experiment count no longer names a number that goes stale with every experiment. - experiments/README: plugrl-bridges is private, so its link 404'd; it is named as not public yet. The E10 row adds E46's 12%. - experiments/e49-multi-trainer/figure.py: every seed's curve from the bridge's episode logs, for the project page (docs/media/other-trainers.png in plugrl.github.io). Reruns are byte-identical. --- README.md | 10 ++- experiments/README.md | 8 +- experiments/e49-multi-trainer/figure.py | 98 +++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 experiments/e49-multi-trainer/figure.py diff --git a/README.md b/README.md index ace2a07..b7280db 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,17 @@ has been measured: | [E45](experiments/e45-env-side-footprint/) | And the other systems that train through their channel? | **Torch on the env side** - RLlib's external-env client and LeRobot's HIL-SERL actor install 6.1 GB each, 27x plugrl-env-client. dm_env_rpc and openpi-client are thin too, but do not train | | [E12](experiments/e12-cuda-free-rollout/) | Does a rollout machine need CUDA, or a GPU? | **No** - LIBERO's env client goes from 7.8G to 3.4G with no nvidia wheels, and renders on the CPU at 1.91x the wall clock ([E13](experiments/e13-gpu-free-rendering/)) | | [E43](experiments/e43-cross-machine-training/) | Does training still work with the env clients on another physical machine? | **Yes** - the quickstart pair learns with its env clients on a Windows laptop over campus Wi-Fi | -| [E43](experiments/e43-cross-machine-training/) | What does crossing cost? | About **3 ms plus twice the observation's bytes over the link** per exchange: 21 ms for 184 KiB at 18 MB/s | -| [E10](experiments/e10-vla-forward-cost/) | Is that cheap beside a VLA forward pass? | **On a fast link.** Over that Wi-Fi a 184 KiB observation is 21% of pi0.5's 100 ms forward, not E10's 1.3-3.6% | +| [E46](experiments/e46-reuse-feedback-obs/) | What does crossing cost? | About **3 ms plus the observation's bytes over the link** per exchange: 11.6 ms for 184 KiB at 23.7 MB/s. Each observation crosses once with `reuse-feedback-obs`, which the server offers and the env client uses by default; [E43](experiments/e43-cross-machine-training/) measured twice the bytes before it, and the weights are byte-identical either way | +| [E10](experiments/e10-vla-forward-cost/) | Is that cheap beside a VLA forward pass? | **On a fast link.** On that link a 184 KiB observation is 12% of pi0.5's 100 ms forward, not E10's 1.3-3.6% | +| [E49](experiments/e49-multi-trainer/) | Do other trainers train through it? | **Yes** - RLinf, Stable-Baselines3 and CleanRL train PlugRL env clients through `plugrl-bridges` (not public yet), 3 of 3 seeds in every pairing; RLinf also with the env clients on a laptop with no torch, Ray or RLinf ([E48](experiments/e48-rlinf-bridge/)) | +| [E50](experiments/e50-boundary-transparency/) | Does training see the boundary? | **No** - SB3 with its 16 environments behind the protocol ends on byte-identical weights to SB3 with them in its own process, also on Atari frames ([E53](experiments/e53-image-observations/)) and with 25 ms each way between the two ([E52](experiments/e52-latency-sweep/)). It costs 0.4-0.6 ms per vector step on one machine | | [E11](experiments/e11-vla-rl-libero/) | Does a full-size VLA go through it? | **Yes** - pi0.5 on LIBERO scores what openpi publishes, and the server's episode and step counts match the clients' exactly | Fine-tuning pi0.5 with reinforcement learning through it has not made the policy better yet; that record is on [its own page](https://plugrl.github.io/vla/). -[`experiments/`](experiments/) holds forty-four experiments, forty-three of them -run. Each carries its data and a `FINDINGS.md` stating what the result does +[`experiments/`](experiments/) holds over fifty experiments. Each that has +run carries its data and a `FINDINGS.md` stating what the result does **not** support. The documentation, including a quickstart that trains FPO on HalfCheetah with no GPU and nothing to download, is at [plugrl.github.io](https://plugrl.github.io). diff --git a/experiments/README.md b/experiments/README.md index 518bce3..ad6cd43 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -34,15 +34,15 @@ protocol is dated in an `AMENDMENT.md`, not folded into it. | [`e7-cross-machine`](e7-cross-machine/) | And once packets leave the machine? | **+0.52 ms** on a 184 KiB observation, from a virtual machine to its host. The cost is in leaving the machine, not in the network stack | | [`e43-cross-machine-training`](e43-cross-machine-training/) | And between two physical machines? | **About 3 ms plus twice the observation's bytes over the link**, per exchange: 21 ms for 184 KiB at 18 MB/s over Wi-Fi | | [`e46-reuse-feedback-obs`](e46-reuse-feedback-obs/) | Can the second copy of the observation go? | **Yes.** With `reuse-feedback-obs` the size term halves, 18.5 to 11.6 ms at 184 KiB, and the same training run gives bit-identical weights. On the way: the server's scheduler sleep costs up to 1 ms per infer | -| [`e10-vla-forward-cost`](e10-vla-forward-cost/) | Is that cheap beside a VLA forward pass? | **On a fast link.** Through openpi's compiled path a forward takes 34.9 ms for the policy PlugRL ships and 100 ms for full-size pi0.5. A 1.3 ms crossing is 1.3-3.6% of that; E43's over Wi-Fi is 21% of the 100 ms | +| [`e10-vla-forward-cost`](e10-vla-forward-cost/) | Is that cheap beside a VLA forward pass? | **On a fast link.** Through openpi's compiled path a forward takes 34.9 ms for the policy PlugRL ships and 100 ms for full-size pi0.5. A 1.3 ms crossing is 1.3-3.6% of that; E43's over Wi-Fi is 21% of the 100 ms, and E46's, with each observation sent once, 12% | | [`e9-many-clients`](e9-many-clients/) | Does one server stay correct with eight clients feeding it? | **Yes.** Twelve runs at 1, 2, 4 and 8 clients, every one exact to the unit. Throughput was still rising at eight, which falsified its prediction | | [`e8-keepalive-hypothesis`](e8-keepalive-hypothesis/) | Does a long learn step kill the connection? | **No.** Learn steps of 190 s close nothing. The drop that prompted it was the machine suspending, and a real reconnect bug was found and fixed anyway | ## Trainers that were not written for it -Through [plugrl-bridges](https://github.com/PlugRL/plugrl-bridges): one -lockstep server, with adapters for RLinf, gymnasium's `VectorEnv` and -Stable-Baselines3's `VecEnv`. +Through `plugrl-bridges` (not public yet): one lockstep server, with +adapters for RLinf, gymnasium's `VectorEnv` and Stable-Baselines3's +`VecEnv`. | | Question | Answer | |---|---|---| diff --git a/experiments/e49-multi-trainer/figure.py b/experiments/e49-multi-trainer/figure.py new file mode 100644 index 0000000..558be77 --- /dev/null +++ b/experiments/e49-multi-trainer/figure.py @@ -0,0 +1,98 @@ +"""E49's figure: three trainers that were not written for PlugRL, training PlugRL env clients. + + python figure.py OUT.png # anywhere with matplotlib + +Reads each registered run's episode log, results/registered/-seed/episodes.csv, which +the bridge writes: one row per finished episode (step, slot, episode, return, length), where step +counts vector steps of the 16 env clients. Plotted: every seed's mean return over the 16 clients' +k-th episodes, against environment steps (vector steps x 16). No mean line and no smoothing. +""" + +import csv +import pathlib +import sys +from collections import defaultdict + +HERE = pathlib.Path(__file__).resolve().parent +RUNS = HERE / "results" / "registered" +CLIENTS = 16 +# (panel title, [(cell, trainer label, colour)]); Okabe-Ito colours, one per trainer. +PANELS = [ + ( + "Pendulum, C++ env client", + [ + ("rlinf-pendulum", "RLinf", "#D55E00"), + ("sb3-pendulum", "Stable-Baselines3", "#0072B2"), + ], + ), + ( + "HalfCheetah-v5, Python env client", + [ + ("sb3-halfcheetah", "Stable-Baselines3", "#0072B2"), + ("cleanrl-halfcheetah", "CleanRL", "#009E73"), + ], + ), +] + + +def curve(path: pathlib.Path) -> tuple[list[float], list[float]]: + """(environment steps in thousands, mean return) per round of the clients' episodes.""" + rounds = defaultdict(list) + with open(path, encoding="utf-8", newline="") as f: + for r in csv.DictReader(f): + rounds[int(r["episode"])].append((int(r["step"]), float(r["return"]))) + xs, ys = [], [] + for k in sorted(rounds): + if len(rounds[k]) < CLIENTS: # a round some client had not finished + continue + xs.append(max(s for s, _ in rounds[k]) * CLIENTS / 1000) + ys.append(sum(v for _, v in rounds[k]) / CLIENTS) + return xs, ys + + +def plot(out: pathlib.Path) -> None: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.lines import Line2D + + matplotlib.rcParams["svg.hashsalt"] = "e49" + fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.3), dpi=200) + trainers = {} + for ax, (title, cells) in zip(axes, PANELS): + for cell, label, colour in cells: + for seed in range(3): + xs, ys = curve(RUNS / f"{cell}-seed{seed}" / "episodes.csv") + ax.plot(xs, ys, color=colour, linewidth=0.9, alpha=0.8) + print( + f"{cell} seed {seed}: {ys[0]:.0f} -> {ys[-1]:.0f} over {xs[-1]:.1f}k steps" + ) + trainers[label] = colour + ax.set_title(title, fontsize=10) + ax.set_xlabel("environment steps (thousands)") + ax.set_xlim(0, None) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + ax.grid(axis="y", color="#e3e6f0", linewidth=0.6) + axes[0].set_ylabel("episode return") + # One legend for both panels: a trainer keeps its colour in each. + handles = [ + Line2D([], [], color=c, linewidth=2, label=f"{t} PPO") + for t, c in trainers.items() + ] + fig.legend( + handles=handles, + frameon=False, + loc="upper center", + ncol=3, + fontsize=8.5, + bbox_to_anchor=(0.5, 1.0), + ) + fig.tight_layout(rect=(0, 0, 1, 0.92)) + fig.savefig(out, facecolor="white", metadata={"Software": None}) + print("wrote", out) + + +if __name__ == "__main__": + plot(pathlib.Path(sys.argv[1])) From e7344b40a06f9cfcf4c7088780a4cee4607064e4 Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Sat, 3 Oct 2026 05:02:38 -0400 Subject: [PATCH 2/2] E49 figure script: LF line endings It was written with CRLF, which pre-commit's mixed-line-ending hook rejects. --- experiments/e49-multi-trainer/figure.py | 196 ++++++++++++------------ 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/experiments/e49-multi-trainer/figure.py b/experiments/e49-multi-trainer/figure.py index 558be77..9299e09 100644 --- a/experiments/e49-multi-trainer/figure.py +++ b/experiments/e49-multi-trainer/figure.py @@ -1,98 +1,98 @@ -"""E49's figure: three trainers that were not written for PlugRL, training PlugRL env clients. - - python figure.py OUT.png # anywhere with matplotlib - -Reads each registered run's episode log, results/registered/-seed/episodes.csv, which -the bridge writes: one row per finished episode (step, slot, episode, return, length), where step -counts vector steps of the 16 env clients. Plotted: every seed's mean return over the 16 clients' -k-th episodes, against environment steps (vector steps x 16). No mean line and no smoothing. -""" - -import csv -import pathlib -import sys -from collections import defaultdict - -HERE = pathlib.Path(__file__).resolve().parent -RUNS = HERE / "results" / "registered" -CLIENTS = 16 -# (panel title, [(cell, trainer label, colour)]); Okabe-Ito colours, one per trainer. -PANELS = [ - ( - "Pendulum, C++ env client", - [ - ("rlinf-pendulum", "RLinf", "#D55E00"), - ("sb3-pendulum", "Stable-Baselines3", "#0072B2"), - ], - ), - ( - "HalfCheetah-v5, Python env client", - [ - ("sb3-halfcheetah", "Stable-Baselines3", "#0072B2"), - ("cleanrl-halfcheetah", "CleanRL", "#009E73"), - ], - ), -] - - -def curve(path: pathlib.Path) -> tuple[list[float], list[float]]: - """(environment steps in thousands, mean return) per round of the clients' episodes.""" - rounds = defaultdict(list) - with open(path, encoding="utf-8", newline="") as f: - for r in csv.DictReader(f): - rounds[int(r["episode"])].append((int(r["step"]), float(r["return"]))) - xs, ys = [], [] - for k in sorted(rounds): - if len(rounds[k]) < CLIENTS: # a round some client had not finished - continue - xs.append(max(s for s, _ in rounds[k]) * CLIENTS / 1000) - ys.append(sum(v for _, v in rounds[k]) / CLIENTS) - return xs, ys - - -def plot(out: pathlib.Path) -> None: - import matplotlib - - matplotlib.use("Agg") - import matplotlib.pyplot as plt - from matplotlib.lines import Line2D - - matplotlib.rcParams["svg.hashsalt"] = "e49" - fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.3), dpi=200) - trainers = {} - for ax, (title, cells) in zip(axes, PANELS): - for cell, label, colour in cells: - for seed in range(3): - xs, ys = curve(RUNS / f"{cell}-seed{seed}" / "episodes.csv") - ax.plot(xs, ys, color=colour, linewidth=0.9, alpha=0.8) - print( - f"{cell} seed {seed}: {ys[0]:.0f} -> {ys[-1]:.0f} over {xs[-1]:.1f}k steps" - ) - trainers[label] = colour - ax.set_title(title, fontsize=10) - ax.set_xlabel("environment steps (thousands)") - ax.set_xlim(0, None) - for side in ("top", "right"): - ax.spines[side].set_visible(False) - ax.grid(axis="y", color="#e3e6f0", linewidth=0.6) - axes[0].set_ylabel("episode return") - # One legend for both panels: a trainer keeps its colour in each. - handles = [ - Line2D([], [], color=c, linewidth=2, label=f"{t} PPO") - for t, c in trainers.items() - ] - fig.legend( - handles=handles, - frameon=False, - loc="upper center", - ncol=3, - fontsize=8.5, - bbox_to_anchor=(0.5, 1.0), - ) - fig.tight_layout(rect=(0, 0, 1, 0.92)) - fig.savefig(out, facecolor="white", metadata={"Software": None}) - print("wrote", out) - - -if __name__ == "__main__": - plot(pathlib.Path(sys.argv[1])) +"""E49's figure: three trainers that were not written for PlugRL, training PlugRL env clients. + + python figure.py OUT.png # anywhere with matplotlib + +Reads each registered run's episode log, results/registered/-seed/episodes.csv, which +the bridge writes: one row per finished episode (step, slot, episode, return, length), where step +counts vector steps of the 16 env clients. Plotted: every seed's mean return over the 16 clients' +k-th episodes, against environment steps (vector steps x 16). No mean line and no smoothing. +""" + +import csv +import pathlib +import sys +from collections import defaultdict + +HERE = pathlib.Path(__file__).resolve().parent +RUNS = HERE / "results" / "registered" +CLIENTS = 16 +# (panel title, [(cell, trainer label, colour)]); Okabe-Ito colours, one per trainer. +PANELS = [ + ( + "Pendulum, C++ env client", + [ + ("rlinf-pendulum", "RLinf", "#D55E00"), + ("sb3-pendulum", "Stable-Baselines3", "#0072B2"), + ], + ), + ( + "HalfCheetah-v5, Python env client", + [ + ("sb3-halfcheetah", "Stable-Baselines3", "#0072B2"), + ("cleanrl-halfcheetah", "CleanRL", "#009E73"), + ], + ), +] + + +def curve(path: pathlib.Path) -> tuple[list[float], list[float]]: + """(environment steps in thousands, mean return) per round of the clients' episodes.""" + rounds = defaultdict(list) + with open(path, encoding="utf-8", newline="") as f: + for r in csv.DictReader(f): + rounds[int(r["episode"])].append((int(r["step"]), float(r["return"]))) + xs, ys = [], [] + for k in sorted(rounds): + if len(rounds[k]) < CLIENTS: # a round some client had not finished + continue + xs.append(max(s for s, _ in rounds[k]) * CLIENTS / 1000) + ys.append(sum(v for _, v in rounds[k]) / CLIENTS) + return xs, ys + + +def plot(out: pathlib.Path) -> None: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.lines import Line2D + + matplotlib.rcParams["svg.hashsalt"] = "e49" + fig, axes = plt.subplots(1, 2, figsize=(7.2, 3.3), dpi=200) + trainers = {} + for ax, (title, cells) in zip(axes, PANELS): + for cell, label, colour in cells: + for seed in range(3): + xs, ys = curve(RUNS / f"{cell}-seed{seed}" / "episodes.csv") + ax.plot(xs, ys, color=colour, linewidth=0.9, alpha=0.8) + print( + f"{cell} seed {seed}: {ys[0]:.0f} -> {ys[-1]:.0f} over {xs[-1]:.1f}k steps" + ) + trainers[label] = colour + ax.set_title(title, fontsize=10) + ax.set_xlabel("environment steps (thousands)") + ax.set_xlim(0, None) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + ax.grid(axis="y", color="#e3e6f0", linewidth=0.6) + axes[0].set_ylabel("episode return") + # One legend for both panels: a trainer keeps its colour in each. + handles = [ + Line2D([], [], color=c, linewidth=2, label=f"{t} PPO") + for t, c in trainers.items() + ] + fig.legend( + handles=handles, + frameon=False, + loc="upper center", + ncol=3, + fontsize=8.5, + bbox_to_anchor=(0.5, 1.0), + ) + fig.tight_layout(rect=(0, 0, 1, 0.92)) + fig.savefig(out, facecolor="white", metadata={"Software": None}) + print("wrote", out) + + +if __name__ == "__main__": + plot(pathlib.Path(sys.argv[1]))