Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ Advanced/SO-100/internal commands stay callable directly (`config`, `calibrate`,

### Install notes

- `[monolithic]` extra is required for the cos=+1.000000 verified export path (pins transformers==5.3.0)
- `[monolithic]` extra is required for the cos=+1.000000 verified export path (pins transformers==5.5.4, the release LeRobot 0.6.1 supports)
- CPU-only: `pip install 'fastcrest-tether[serve,onnx,monolithic]'`
- GPU install needs the FULL cuDNN 9 system library (not just the pip wheel). Easiest path: NVIDIA's container `docker run --gpus all -it nvcr.io/nvidia/tensorrt:24.10-py3`, then `apt-get install -y clang` (for lerobot→evdev), then the pip install
- `tether serve` errors loudly if cuDNN can't load — no silent CPU fallback
Expand Down
260 changes: 260 additions & 0 deletions docs/research/verification/lerobot-06-export-port-20261012.md

Large diffs are not rendered by default.

61 changes: 39 additions & 22 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ dependencies = [
# compiled against numpy 1.x ABI; numpy 2.x's ABI break corrupts ORT +
# JAX wheels on aarch64. On x86_64 (Modal A100, customer desktops) +
# arm64 (M-series Mac dev hosts) numpy 2.x is fine, AND the
# [monolithic] extra's lerobot==0.5.1 dep REQUIRES numpy>=2.0 — pinning
# [monolithic] extra's lerobot==0.6.1 dep REQUIRES numpy>=2.0 — pinning
# numpy<2 globally makes lerobot uninstallable on Modal (caught by the
# 2026-05-20 lift #4 smoke fire). Split via PEP-508 platform marker.
"numpy>=1.24.0,<2.0; platform_machine == 'aarch64'",
Expand All @@ -39,12 +39,11 @@ dependencies = [
# loading for the auto-detect path. Apr-14 install-path verification caught
# that omitting these breaks `tether export` even with [serve,gpu].
"huggingface_hub>=0.20.0",
# Base allows 4.40 → 5.3.x. [monolithic] extra pins ==5.3.0 exactly (5.4+ has
# a q_length regression that breaks onnx-diagnostic patches). Without the
# upper bound, pip would pull 5.4/5.5 here and conflict with [monolithic].
# Without raising the upper bound past 5.0, [monolithic] can't install
# alongside the base (the v0.2.1 customer-dogfood finding).
"transformers>=4.40,<5.4",
# Base allows 4.40 → 5.5.x. [monolithic] pins ==5.5.4 exactly, inside
# LeRobot 0.6.1's declared >=5.4,<5.6 (on 5.3 its pi0/pi0.5 embeddings are
# mis-scaled; Studio ADR 0003 correction 2026-10-12). The upper bound keeps
# pip from pulling an unqualified 5.6+ here.
"transformers>=4.40,<5.6",
"onnx>=1.15.0",
"onnxscript>=0.1.0",
# TetherClient (Phase 0.5 customer-sdk) — sync + async HTTP client.
Expand Down Expand Up @@ -141,32 +140,42 @@ comply = []
# gate in GOALS.yaml. Heavy (pulls torch, torchvision, accelerate, etc.) —
# that's why it's a separate extra, not a base dep.
#
# lerobot 0.5.x requires Python >=3.12. Jetson (JetPack 6) ships 3.10,
# lerobot 0.6.x requires Python >=3.12. Jetson (JetPack 6) ships 3.10,
# so we gate lerobot behind a version marker — pip skips it on 3.10/3.11
# instead of hard-failing with ResolutionImpossible. Native parity
# verification should be run on a Python 3.12+ host.
native = [
"lerobot==0.5.1; python_version >= '3.12'",
"transformers==5.5.4",
"lerobot==0.6.1; python_version >= '3.12'",
"draccus==0.11.6; python_version >= '3.12'",
"num2words",
"accelerate>=1.14,<2",
"diffusers==0.39.0",
]
# Monolithic ONNX export — cos=1.0 verified production path. Runs locally
# via `tether export --monolithic` (the DEFAULT as of v0.2.1). Requires
# transformers==5.3.0 EXACTLY (5.4+ has a q_length regression in
# masking_utils.sdpa_mask that breaks onnx-diagnostic patches). Base pin
# `transformers>=4.40,<5.4` permits this extra to pin-to-exact.
# transformers==5.5.4 EXACTLY: LeRobot 0.6.1 needs >=5.4,<5.6, and the
# exporter is qualified on this release (tests/_lerobot_pi_golden.py runs the
# full onnx-diagnostic export against the pre-migration golden).
#
# lerobot 0.5.x requires Python >=3.12. On Jetson (Python 3.10) the
# lerobot 0.6.x requires Python >=3.12. On Jetson (Python 3.10) the
# monolithic export can't run locally — export on a cloud/desktop host
# (Python 3.12+) and `tether serve` the resulting ONNX on the Jetson
# with `pip install 'fastcrest-tether[serve]'` only.
monolithic = [
"lerobot==0.5.1; python_version >= '3.12'",
"transformers==5.3.0",
"lerobot==0.6.1; python_version >= '3.12'",
"draccus==0.11.6; python_version >= '3.12'",
"transformers==5.5.4",
# onnx-diagnostic's export patches reference a torch 2.8+ symbol;
# _require_monolithic_deps refuses older torch with the same floor.
"torch>=2.8",
"onnx-diagnostic>=0.9",
"onnxscript>=0.1",
"optree",
"scipy",
"num2words",
"accelerate>=1.14,<2",
"diffusers==0.39.0",
]
# Task-success evaluation (heavy sim deps). Gated behind this extra so the
# base install stays light. Use with `tether bench --benchmark libero_10`.
Expand All @@ -177,8 +186,14 @@ eval = [
"gymnasium",
"h5py",
]
# Spell out the SmolVLA dependencies (LeRobot's transformers-dep extra,
# >=5.4,<5.6) at the exact qualified transformers.
eval-local = [
"lerobot[smolvla]==0.5.1; python_version >= '3.12' and sys_platform == 'linux'",
"transformers==5.5.4; python_version >= '3.12' and sys_platform == 'linux'",
"num2words>=0.5.14,<0.6; python_version >= '3.12' and sys_platform == 'linux'",
"accelerate>=1.14,<2; python_version >= '3.12' and sys_platform == 'linux'",
"lerobot==0.6.1; python_version >= '3.12' and sys_platform == 'linux'",
"draccus==0.11.6; python_version >= '3.12' and sys_platform == 'linux'",
"peft; python_version >= '3.12' and sys_platform == 'linux'",
"mujoco==3.3.2; sys_platform == 'linux'",
"robosuite==1.4.1; sys_platform == 'linux'",
Expand Down Expand Up @@ -212,13 +227,14 @@ tracing = [
# pip install 'fastcrest-tether[lerobot]'
# tether serve bundle/ --embodiment so_arm100 --port /dev/ttyUSB0
#
# lerobot 0.5.x requires Python >=3.12 (we skip on 3.10/3.11 to avoid pip's
# lerobot 0.6.x requires Python >=3.12 (we skip on 3.10/3.11 to avoid pip's
# ResolutionImpossible; users on JetPack 6 Python 3.10 can still construct an
# adapter + run the conversion math, they just can't open a serial bus until
# they upgrade to Python 3.12 or install scservo_sdk directly via the [so100]
# extra).
lerobot = [
"lerobot>=0.5.1; python_version >= '3.12'",
"lerobot==0.6.1; python_version >= '3.12'",
"draccus==0.11.6; python_version >= '3.12'",
]
# SO-100 hardware extra — installs the scservo_sdk Python package on the
# Raspberry Pi / host wired to the arm. Separate from [lerobot] because the
Expand All @@ -234,9 +250,10 @@ so100 = [
# RTC-enabled deployments. Skeleton + config validation work without this
# extra; only the actual processor construction needs it.
#
# lerobot 0.5.x requires Python >=3.12 (skipped on 3.10/3.11).
# lerobot 0.6.x requires Python >=3.12 (skipped on 3.10/3.11).
rtc = [
"lerobot==0.5.1; python_version >= '3.12'",
"lerobot==0.6.1; python_version >= '3.12'",
"draccus==0.11.6; python_version >= '3.12'",
]
# Curate format-converter extras. Phase 1 ships LeRobot v3 + HDF5 with the
# core install (parquet via pyarrow which huggingface_hub already pulls; h5py
Expand Down Expand Up @@ -270,7 +287,7 @@ curate-rlds = [

# Jetson / ARM64 edge stack.
# - Skips [gpu] x86_64 wheels (onnxruntime-gpu PyPI has no aarch64 builds)
# - Skips lerobot (0.5.1 requires Python 3.12; JetPack 6 ships 3.10)
# - Skips lerobot (0.6.1 requires Python 3.12; JetPack 6 ships 3.10)
# - User must pre-install Jetson Zoo onnxruntime-gpu from pypi.jetson-ai-lab.io
# and mount host CUDA via --runtime=nvidia.
jetson = [
Expand All @@ -281,7 +298,7 @@ jetson = [
"jsonschema>=4.0",
"prometheus-client>=0.19.0",
# monolithic export stack minus lerobot
"transformers==5.3.0",
"transformers==5.5.4",
"onnx-diagnostic>=0.9",
"onnxscript>=0.1",
"optree",
Expand Down
33 changes: 2 additions & 31 deletions scripts/local_pi05_monolithic_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,36 +34,6 @@
sys.modules[_mod] = _stub


def _patch_pi05_for_transformers() -> None:
from lerobot.policies.pi05 import modeling_pi05
from lerobot.policies import pi_gemma
from transformers import masking_utils

def patched_embed_image(self, image):
out_dtype = image.dtype
if image.dtype != torch.float32:
image = image.to(torch.float32)
output = self.paligemma.model.get_image_features(image)
features = output.pooler_output if hasattr(output, "pooler_output") else output
features = features * self.paligemma.config.text_config.hidden_size**0.5
if features.dtype != out_dtype:
features = features.to(out_dtype)
return features

modeling_pi05.PaliGemmaWithExpertModel.embed_image = patched_embed_image

original = masking_utils.create_causal_mask

def causal_mask_shim(*args, **kwargs):
if "inputs_embeds" in kwargs and "input_embeds" not in kwargs:
kwargs["input_embeds"] = kwargs.pop("inputs_embeds")
return original(*args, **kwargs)

masking_utils.create_causal_mask = causal_mask_shim
if hasattr(pi_gemma, "create_causal_mask"):
pi_gemma.create_causal_mask = causal_mask_shim


def _repo_commit() -> str:
root = Path(__file__).resolve().parents[1]
try:
Expand Down Expand Up @@ -160,7 +130,8 @@ def main(argv: list[str] | None = None) -> int:
if args.num_steps <= 0:
raise ValueError("--num-steps must be positive.")

_patch_pi05_for_transformers()
# Reference is native LeRobot 0.6.1 under the qualified transformers
# (5.5.4): no embedding or mask shims (Studio ADR 0003 correction 2026-10-12).

import onnxruntime as ort
from huggingface_hub import snapshot_download
Expand Down
42 changes: 2 additions & 40 deletions scripts/local_pi0_monolithic_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,44 +35,6 @@
sys.modules[_mod] = _stub


def _patch_pi0_for_transformers_457() -> None:
from lerobot.policies.pi0 import modeling_pi0

def patched_embed_image(self, image):
out_dtype = image.dtype
if image.dtype != torch.float32:
image = image.to(torch.float32)
image_outputs = self.paligemma.model.get_image_features(image)
features = (
image_outputs.pooler_output
if hasattr(image_outputs, "pooler_output")
else image_outputs
)
features = features * self.paligemma.config.text_config.hidden_size**0.5
if features.dtype != out_dtype:
features = features.to(out_dtype)
return features

modeling_pi0.PaliGemmaWithExpertModel.embed_image = patched_embed_image


def _patch_create_causal_mask_kwarg() -> None:
from transformers import masking_utils

original = masking_utils.create_causal_mask

def shim(*args, **kwargs):
if "inputs_embeds" in kwargs and "input_embeds" not in kwargs:
kwargs["input_embeds"] = kwargs.pop("inputs_embeds")
return original(*args, **kwargs)

masking_utils.create_causal_mask = shim
from lerobot.policies import pi_gemma

if hasattr(pi_gemma, "create_causal_mask"):
pi_gemma.create_causal_mask = shim


def _repo_commit() -> str:
root = Path(__file__).resolve().parents[1]
try:
Expand Down Expand Up @@ -167,8 +129,8 @@ def main(argv: list[str] | None = None) -> int:
if args.num_steps <= 0:
raise ValueError("--num-steps must be positive.")

_patch_pi0_for_transformers_457()
_patch_create_causal_mask_kwarg()
# Reference is native LeRobot 0.6.1 under the qualified transformers
# (5.5.4): no embedding or mask shims (Studio ADR 0003 correction 2026-10-12).

import onnxruntime as ort
from huggingface_hub import snapshot_download
Expand Down
10 changes: 5 additions & 5 deletions scripts/modal_gr00t_lerobot_probe.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Quick probe — what does lerobot 0.5.1 actually expose for GR00T?
"""Quick probe — what does lerobot 0.6.1 actually expose for GR00T?

lerobot-0.5.1 on Modal may or may not have GR00T support. The cloned
lerobot-0.6.1 on Modal may or may not have GR00T support. The cloned
local lerobot is `main` (newer). Need to reconcile.

Prints the actual module contents of lerobot's groot subpackage.
Expand All @@ -23,7 +23,7 @@
"numpy",
"safetensors>=0.4.0",
"huggingface_hub",
"lerobot==0.5.1",
"lerobot==0.6.1",
"num2words",
)
)
Expand Down Expand Up @@ -52,7 +52,7 @@ def probe():

# Attempt to import groot_n1 + list symbols
try:
from lerobot.policies.groot import groot_n1
from lerobot.policies.groot import groot_n1_7 as groot_n1
print(f"\ngroot_n1 symbols (top-level, non-dunder):")
for name in sorted(dir(groot_n1)):
if not name.startswith("_"):
Expand All @@ -64,7 +64,7 @@ def probe():

# Attempt action_head
try:
from lerobot.policies.groot.action_head import flow_matching_action_head as fma
from lerobot.policies.groot.action_head import cross_attention_dit as fma
print(f"\nflow_matching_action_head symbols:")
for name in sorted(dir(fma)):
if not name.startswith("_"):
Expand Down
29 changes: 18 additions & 11 deletions scripts/modal_groot_export_parity_pinned.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
)

HF_CACHE_PATH = "/root/.cache/huggingface"
PARITY_OUT_PATH = "/parity_out"
PARITY_OUT_PATH = "/parity_out/lerobot-061-20261011"


def _hf_secret():
Expand All @@ -63,24 +63,26 @@ def _hf_secret():
modal.Image.debian_slim(python_version="3.12")
.apt_install("git")
.pip_install(
"torch",
# torch>=2.8: onnx-diagnostic 0.9.x torch_export_patches needs it (see pi0 vehicle).
"torch>=2.8,<2.12",
"safetensors>=0.4.0",
"huggingface_hub",
# transformers==5.3.0 EXACTLY: the monolithic exporter refuses
# transformers==5.5.4 EXACTLY: the monolithic exporter refuses
# anything else.
"transformers==5.3.0",
"transformers==5.5.4",
"onnx>=1.16",
"onnxruntime>=1.20",
"onnxscript>=0.1",
# monolithic dep group (export_gr00t_monolithic gates on these):
"lerobot==0.5.1",
"lerobot==0.6.1",
"num2words",
"diffusers==0.39.0",
"onnx-diagnostic>=0.9",
"optree",
"scipy",
"accelerate",
"draccus",
"numpy",
"draccus==0.11.6",
"numpy==2.2.6",
"Pillow",
"typer",
"rich",
Expand Down Expand Up @@ -122,15 +124,15 @@ def _hf_secret():
timeout=3600,
volumes={
HF_CACHE_PATH: _hf_cache_volume,
PARITY_OUT_PATH: _parity_out_volume,
"/parity_out": _parity_out_volume,
},
secrets=[_hf_secret()],
)
def run_export_parity(
model_revision: str = GROOT_REVISION,
tether_revision: str = TETHER_REVISION,
embodiment_id: int = 0,
reuse_export: bool = True,
reuse_export: bool = False,
watchdog_seconds: float = 3300,
) -> dict:
import subprocess
Expand Down Expand Up @@ -322,8 +324,13 @@ def _progress(stage: str) -> None:


@app.local_entrypoint()
def main() -> None:
result = run_export_parity.remote()
def main(
tether_revision: str = TETHER_REVISION,
watchdog_seconds: float = 1800,
) -> None:
result = run_export_parity.remote(
tether_revision=tether_revision, watchdog_seconds=watchdog_seconds
)
print("\n=== RESULT ===")
for key, value in result.items():
print(f" {key}: {value}")
7 changes: 4 additions & 3 deletions scripts/modal_layer_pruning_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ def _build_bust() -> str:
modal.Image.debian_slim(python_version="3.12")
.apt_install("git")
.pip_install(
"lerobot==0.5.1",
"transformers==5.3.0",
"lerobot==0.6.1",
"transformers==5.5.4",
"num2words",
"safetensors>=0.4.0",
"scipy",
Expand Down Expand Up @@ -164,6 +164,7 @@ def hook(_module, _inp, output):
images = [img, img, img]
img_masks = [mask, mask, mask]
from lerobot.policies.pi05.modeling_pi05 import make_att_2d_masks
from lerobot.policies.common.vla_utils import prepare_attention_masks_4d
prefix_embs, prefix_pad_masks, prefix_att_masks = (
policy.model.embed_prefix(
images, img_masks, lang_tokens, lang_masks,
Expand All @@ -172,7 +173,7 @@ def hook(_module, _inp, output):
prefix_att_2d = make_att_2d_masks(prefix_pad_masks, prefix_att_masks)
prefix_position_ids = torch.cumsum(prefix_pad_masks, dim=1) - 1
prefix_att_2d_4d = (
policy.model._prepare_attention_masks_4d(prefix_att_2d)
prepare_attention_masks_4d(prefix_att_2d)
)
language_model.config._attn_implementation = "eager"
_ = policy.model.paligemma_with_expert.forward(
Expand Down
Loading
Loading