diff --git a/figures/coverage/cells.json b/figures/coverage/cells.json index af8bf16..f531ed6 100644 --- a/figures/coverage/cells.json +++ b/figures/coverage/cells.json @@ -1,5 +1,5 @@ { - "_about": "The coverage figure's cells. `runs` is relative to the machine's PlugRL home and takes {seed}; `seed` is the one whose mean return over its last ten iterations is the median of the three (ties: the lowest seed). `status` is what the experiments established, not what the video shows.", + "_about": "The coverage figure's cells. `runs` is relative to the machine's PlugRL home and takes {seed}; `seed` is the one whose mean return over its last ten iterations is the median of the three (ties: the lowest seed). `seeds`, when present, are the three the experiment ran; otherwise 0-2. `status` is what the experiments established, not what the video shows.", "rows": [ {"id": "fpo-fpo", "label": "fpo-policy · FPO"}, {"id": "fpo-dppo", "label": "fpo-policy · DPPO"}, @@ -51,30 +51,33 @@ }, { "id": "fpo-dppo-cheetah", "row": "fpo-dppo", "column": "cheetah", - "status": "flat", "experiments": ["E17", "E18"], - "note": "no learning in 409,600 steps", "note_zh": "409,600 步内没学会", - "runs": "coverage/stage/e18/halfcheetah-seed{seed}", "seed": 1, - "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "17", "--policy.action-dim", "6"], + "status": "learns", "experiments": ["E17", "E18", "E30", "E33"], + "note": "+1,294 to +1,411 with dppo-policy's structure", "note_zh": "换成 dppo-policy 的结构后 +1294 到 +1411", + "runs": "e33/experiments/e33-fpo-dppo-structure/results/fpo-cheetah/dppo/fpo-policy/fpo-cheetah-seed{seed}", "seed": 0, + "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "17", "--policy.action-dim", "6", + "--policy.action-horizon", "4", "--policy.flow-steps", "20", "--policy.hidden-dims", "512", "512", "512"], "client": ["mujoco-v1", "--env.name", "HalfCheetah-v5", "--env.render"], - "replan": 1, "native_fps": 20 + "replan": 4, "native_fps": 20 }, { "id": "fpo-dppo-hopper", "row": "fpo-dppo", "column": "hopper", - "status": "flat", "experiments": ["E28", "E29"], - "note": "no learning; with 10× the noise, +74 to +106", "note_zh": "没学会;噪声放大 10 倍后 +74 到 +106", - "runs": "e28/experiments/e28-dppo-learns/results/fpodppo-hopper/dppo/fpo-policy/fpodppo-hopper-seed{seed}", "seed": 0, - "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "11", "--policy.action-dim", "3"], + "status": "learns", "experiments": ["E28", "E29", "E30", "E33"], + "note": "998–1,037 with dppo-policy's structure, mostly by staying up", "note_zh": "换成 dppo-policy 的结构后 998–1037,主要靠站住不倒", + "runs": "e33/experiments/e33-fpo-dppo-structure/results/fpo-hopper/dppo/fpo-policy/fpo-hopper-seed{seed}", "seed": 3, "seeds": [3, 4, 5], + "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "11", "--policy.action-dim", "3", + "--policy.action-horizon", "4", "--policy.flow-steps", "20", "--policy.hidden-dims", "512", "512", "512"], "client": ["mujoco-v1", "--env.name", "Hopper-v5", "--env.render"], - "replan": 1, "native_fps": 125 + "replan": 4, "native_fps": 125 }, { "id": "fpo-dppo-walker", "row": "fpo-dppo", "column": "walker", - "status": "flat", "experiments": ["E28"], - "note": "no learning in 409,600 steps", "note_zh": "409,600 步内没学会", - "runs": "e28/experiments/e28-dppo-learns/results/fpodppo-walker/dppo/fpo-policy/fpodppo-walker-seed{seed}", "seed": 2, - "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "17", "--policy.action-dim", "6"], + "status": "learns", "experiments": ["E28", "E33"], + "note": "283–569 with dppo-policy's structure; 2 of 3 seeds past the bar", "note_zh": "换成 dppo-policy 的结构后 283–569,3 个种子里 2 个过线", + "runs": "e33/experiments/e33-fpo-dppo-structure/results/fpo-walker/dppo/fpo-policy/fpo-walker-seed{seed}", "seed": 0, + "server": ["fpo-policy", "default", "eval", "default", "--policy.obs-dim", "17", "--policy.action-dim", "6", + "--policy.action-horizon", "4", "--policy.flow-steps", "20", "--policy.hidden-dims", "512", "512", "512"], "client": ["mujoco-v1", "--env.name", "Walker2d-v5", "--env.render"], - "replan": 1, "native_fps": 125 + "replan": 4, "native_fps": 125 }, { "id": "fpo-dppo-square", "row": "fpo-dppo", "column": "square", @@ -97,28 +100,28 @@ }, { "id": "dppo-dppo-hopper", "row": "dppo-dppo", "column": "hopper", - "status": "rising", "experiments": ["E28"], - "note": "11–16 → 276–430, still rising; the bar is 500", "note_zh": "从 11–16 升到 276–430,还在涨;线是 500", - "runs": "e28/experiments/e28-dppo-learns/results/dppo-hopper/dppo/dppo-policy/dppo-hopper-seed{seed}", "seed": 0, + "status": "learns", "experiments": ["E28", "E31"], + "note": "812–961 at 300 iterations; 3 of 3 seeds past the bar", "note_zh": "300 轮后 812–961,3 个种子全部过线", + "runs": "e31/experiments/e31-dppo-longer/results/dppo-hopper/dppo/dppo-policy/dppo-hopper-seed{seed}", "seed": 0, "server": ["dppo-policy", "hopper", "eval", "default"], "client": ["mujoco-v1", "--env.name", "Hopper-v5", "--env.render"], "replan": 4, "native_fps": 125 }, { "id": "dppo-dppo-walker", "row": "dppo-dppo", "column": "walker", - "status": "rising", "experiments": ["E28"], - "note": "about −2 → 260–292, then flat; the bar is 500", "note_zh": "从约 −2 升到 260–292 后走平;线是 500", - "runs": "e28/experiments/e28-dppo-learns/results/dppo-walker/dppo/dppo-policy/dppo-walker-seed{seed}", "seed": 2, + "status": "learns", "experiments": ["E28", "E31"], + "note": "528–622 at 300 iterations; 3 of 3 seeds past the bar", "note_zh": "300 轮后 528–622,3 个种子全部过线", + "runs": "e31/experiments/e31-dppo-longer/results/dppo-walker/dppo/dppo-policy/dppo-walker-seed{seed}", "seed": 2, "server": ["dppo-policy", "walker", "eval", "default"], "client": ["mujoco-v1", "--env.name", "Walker2d-v5", "--env.render"], "replan": 4, "native_fps": 125 }, { "id": "dppo-dppo-square", "row": "dppo-dppo", "column": "square", - "status": "runs", "experiments": ["E27"], - "note": "no success yet: sparse reward, random start", "note_zh": "还没成功过:稀疏奖励、随机初始化", - "runs": "e27/experiments/e27-robomimic/results/dppo-rm/dppo/dppo-policy/dppo-rm-seed{seed}", "seed": 0, - "server": ["dppo-policy", "default", "eval", "default", "--policy.env-type", "robomimic", "--policy.env-name", "square"], + "status": "learns", "experiments": ["E27", "E34"], + "note": "from DPPO's released policy, success 0.33–0.36 → 0.65–0.70", "note_zh": "从 DPPO 官方预训练策略起步,成功率 0.33–0.36 → 0.65–0.70", + "runs": "e34/experiments/e34-square-dppo/results/dppo-square/dppo/dppo-policy/dppo-square-seed{seed}", "seed": 2, + "server": ["dppo-policy", "square", "eval", "default"], "client": ["robomimic-v1", "--env.name", "square-img", "--env.agentview-image-size", "256", "256"], "replan": 4, "native_fps": 20 } @@ -136,9 +139,9 @@ }, { "id": "pi0-fpo", "label": "pi0-policy · FPO", "label_zh": "pi0-policy · FPO", - "status": "collapses", "experiments": ["E14", "E26"], "score": [0, 50], - "note": "one iteration takes it to 0; our defect, under investigation", - "note_zh": "一轮后掉到 0;是我们平台的问题,正在排查", + "status": "holds", "experiments": ["E14", "E26", "E32"], "score": [33, 50], + "note": "one update with FPO++'s chunk loss leaves it intact; FPO's own took it to 0", + "note_zh": "换成 FPO++ 的块损失后一次更新没毁掉它;原版 FPO 会让它归零", "video_key": "base", "native_fps": 20, "whole": true }, { diff --git a/figures/coverage/commands.json b/figures/coverage/commands.json index 4cd24dd..65b52ad 100644 --- a/figures/coverage/commands.json +++ b/figures/coverage/commands.json @@ -28,19 +28,19 @@ "env": "robomimic", "source": "experiments/e27-robomimic/run_cell.sh" }, "fpo-dppo-cheetah": { - "server": ["fpo-policy default dppo cheetah", "--port 8000 --seed 0 --policy.device cpu", "--algo.buffer-size 4096 --algo.train-itrs 100"], - "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name HalfCheetah-v5 --runner.replan-steps 1 --runner.seed 0"], - "env": "mujoco", "source": "experiments/e18-dppo-normalised/run.sh" + "server": ["fpo-policy default dppo cheetah", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 17 --policy.action-dim 6", "--policy.action-horizon 4 --policy.flow-steps 20 --policy.hidden-dims 512 512 512", "--algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 300"], + "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name HalfCheetah-v5 --runner.replan-steps 4 --runner.seed 0"], + "env": "mujoco", "source": "experiments/e33-fpo-dppo-structure/run.sh" }, "fpo-dppo-hopper": { - "server": ["fpo-policy default dppo hopper", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 11 --policy.action-dim 3", "--algo.buffer-size 4096 --algo.train-itrs 100"], - "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Hopper-v5 --runner.replan-steps 1 --runner.seed 0"], - "env": "mujoco", "source": "experiments/e28-dppo-learns/run.sh" + "server": ["fpo-policy default dppo hopper", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 11 --policy.action-dim 3", "--policy.action-horizon 4 --policy.flow-steps 20 --policy.hidden-dims 512 512 512", "--algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 300"], + "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Hopper-v5 --runner.replan-steps 4 --runner.seed 0"], + "env": "mujoco", "source": "experiments/e33-fpo-dppo-structure/run.sh" }, "fpo-dppo-walker": { - "server": ["fpo-policy default dppo walker", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 17 --policy.action-dim 6", "--algo.buffer-size 4096 --algo.train-itrs 100"], - "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Walker2d-v5 --runner.replan-steps 1 --runner.seed 0"], - "env": "mujoco", "source": "experiments/e28-dppo-learns/run.sh" + "server": ["fpo-policy default dppo walker", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 17 --policy.action-dim 6", "--policy.action-horizon 4 --policy.flow-steps 20 --policy.hidden-dims 512 512 512", "--algo.sampling-noise-level 1.0 --algo.logprob-noise-level 1.0", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 300"], + "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Walker2d-v5 --runner.replan-steps 4 --runner.seed 0"], + "env": "mujoco", "source": "experiments/e33-fpo-dppo-structure/run.sh" }, "fpo-dppo-square": { "server": ["fpo-policy default dppo default", "--port 8000 --seed 0 --policy.device cpu", "--policy.obs-dim 23 --policy.action-dim 7", "--policy.state-keys robot0_eef_pos robot0_eef_quat robot0_gripper_qpos object", "--algo.buffer-size 4096 --algo.batch-size 256 --algo.train-itrs 20"], @@ -53,19 +53,19 @@ "env": "mujoco", "source": "experiments/e28-dppo-learns/run.sh" }, "dppo-dppo-hopper": { - "server": ["dppo-policy hopper dppo hopper", "--port 8000 --seed 0 --policy.device cpu", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 100"], + "server": ["dppo-policy hopper dppo hopper", "--port 8000 --seed 0 --policy.device cpu", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 300"], "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Hopper-v5 --runner.replan-steps 4 --runner.seed 0"], - "env": "mujoco", "source": "experiments/e28-dppo-learns/run.sh" + "env": "mujoco", "source": "experiments/e31-dppo-longer/run.sh" }, "dppo-dppo-walker": { - "server": ["dppo-policy walker dppo walker", "--port 8000 --seed 0 --policy.device cpu", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 100"], + "server": ["dppo-policy walker dppo walker", "--port 8000 --seed 0 --policy.device cpu", "--algo.buffer-size 1024 --algo.batch-size 512 --algo.train-itrs 300"], "client": ["mujoco-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name Walker2d-v5 --runner.replan-steps 4 --runner.seed 0"], - "env": "mujoco", "source": "experiments/e28-dppo-learns/run.sh" + "env": "mujoco", "source": "experiments/e31-dppo-longer/run.sh" }, "dppo-dppo-square": { - "server": ["dppo-policy default dppo default", "--port 8000 --seed 0 --policy.device cpu", "--policy.env-type robomimic --policy.env-name square", "--algo.buffer-size 1024 --algo.batch-size 256 --algo.train-itrs 20"], - "client": ["robomimic-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-episodes 100000", "--env.name square-img --env.agentview-image-size 84 84", "--runner.replan-steps 4"], - "env": "robomimic", "source": "experiments/e27-robomimic/run_cell.sh" + "server": ["dppo-policy square dppo square", "--port 8000 --seed 0 --policy.device cpu", "--policy.checkpoint-path DPPO_SQUARE_CHECKPOINT", "--algo.train-itrs 40"], + "client": ["robomimic-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-procs 6 --num-episodes 100000", "--env.name square-img --env.agentview-image-size 84 84", "--env.no-terminate-on-success --runner.replan-steps 4"], + "env": "robomimic", "source": "experiments/e34-square-dppo/run_cell.sh" }, "pi0-base": { "server": ["pi0-policy default eval default", "--port 8000 --policy.device cuda", "--policy.name pi05_libero --policy.checkpoint-path PI05_LIBERO_CHECKPOINT"], @@ -73,9 +73,9 @@ "env": "libero", "source": "experiments/e26-frozen-mlp/eval.sh" }, "pi0-fpo": { - "server": ["pi0-policy default fpo default", "--port 8000 --seed 7 --policy.device cuda:0", "--policy.name pi05_libero --policy.checkpoint-path PI05_LIBERO_CHECKPOINT", "--algo.learning-rate 1e-5 --algo.clipping-epsilon 0.05", "--algo.num-updates-per-batch 4 --algo.n-samples-per-action 4", "--algo.buffer-size 4096 --algo.batch-size 8 --algo.global-steps 4096", "--algo.master-weights-device cuda:1"], + "server": ["pi0-policy default fpo default", "--port 8000 --seed 7 --policy.device cuda:0", "--policy.name pi05_libero --policy.checkpoint-path PI05_LIBERO_CHECKPOINT", "--algo.learning-rate 1e-5 --algo.clipping-epsilon 0.05", "--algo.num-updates-per-batch 4 --algo.n-samples-per-action 4", "--algo.buffer-size 4096 --algo.batch-size 8 --algo.global-steps 8192", "--algo.master-weights-device cuda:1", "--algo.n-critic-warmup-itrs 1", "--algo.output-mode u --algo.no-discretize-t-for-training", "--algo.cfm-loss-steps 5 --algo.cfm-loss-dims 7 --algo.cfm-loss-sum-over-steps", "--algo.ratio-per-sample"], "client": ["libero-v1 --server-host 127.0.0.1 --server-port 8000", "--num-envs 1 --num-procs 10 --num-episodes 100000", "--env.task-suite-name libero_10 --env.task-id 8", "--env.randomize-initial-state --runner.replan-steps 5 --runner.seed 7"], - "env": "libero", "source": "experiments/e26-frozen-mlp/train.sh" + "env": "libero", "source": "experiments/e32-pi0-fpo-plus-plus/train.sh" }, "pi0-dppo": { "server": ["pi0-policy default dppo libero", "--port 8000 --seed 7 --policy.device cuda:0", "--policy.name pi05_libero --policy.checkpoint-path PI05_LIBERO_CHECKPOINT", "--algo.buffer-size 4096 --algo.batch-size 8 --algo.train-itrs 2"], diff --git a/figures/coverage/data.py b/figures/coverage/data.py index 91d2dbd..e94ee6a 100644 --- a/figures/coverage/data.py +++ b/figures/coverage/data.py @@ -48,7 +48,9 @@ def main() -> int: cells = [] for cell in spec["cells"]: - curves = [returns(HOME / cell["runs"].format(seed=s)) for s in range(3)] + # Most cells ran seeds 0-2; a cell that ran others names them. + seeds = cell.get("seeds", [0, 1, 2]) + curves = [returns(HOME / cell["runs"].format(seed=s)) for s in seeds] cells.append( { "id": cell["id"],