feat(eval): add live-agent VLN-CE benchmark

[TomCC7]

Superseded by #3471. During the stack reorder, GitHub marked this draft merged into its former feature-branch base; it did not merge to main. Review and continue the VLN-CE change in #3471.

Contribution path

• Original stack position: 4/4
• Replacement PR: feat(eval): add live-agent VLN-CE benchmark #3471

This change adds the live-agent runtime and containerized VLN-CE R2R benchmark. The replacement PR carries the same rewritten commits on the corrected transcript base.

[codecov]

❌ 3 Tests Failed:

Tests completed	Failed	Passed	Skipped
3831	3	3828	175

View the top 3 failed test(s) by shortest run time

dimos.codebase_checks.test_no_init_files::test_no_init_files

Stack Traces | 0.022s run time

def test_no_init_files():
        dimos_dir = DIMOS_PROJECT_ROOT / "dimos"
        init_files = sorted(dimos_dir.rglob("__init__.py"))
        # The root dimos/__init__.py is allowed for the porcelain lazy import.
        init_files = [f for f in init_files if f != dimos_dir / "__init__.py"]
        if init_files:
            listing = "\n".join(f"  - {f.relative_to(dimos_dir)}" for f in init_files)
>           raise AssertionError(
                f"Found __init__.py files in dimos/:\n{listing}\n\n"
                "__init__.py files are not allowed because they lead to unnecessary "
                "extraneous imports. Everything should be imported straight from the "
                "source module."
            )
E           AssertionError: Found __init__.py files in dimos/:
E             - benchmark/libero_pro/__init__.py
E             - .../libero_pro/proto/__init__.py
E           
E           __init__.py files are not allowed because they lead to unnecessary extraneous imports. Everything should be imported straight from the source module.

dimos_dir  = PosixPath('.../dimos/dimos/dimos')
init_files = [PosixPath('.../dimos/dimos/dimos/benchmark/libero_pro/__init__.py'), PosixPath('.../dimos/dimos/dimos/.../libero_pro/proto/__init__.py')]
listing    = '  - benchmark/libero_pro/__init__.py\n  - .../libero_pro/proto/__init__.py'

dimos/codebase_checks/test_no_init_files.py:25: AssertionError

dimos.codebase_checks.test_inline_heavy_imports::test_heavy_imports_are_inline

Stack Traces | 1.84s run time

def test_heavy_imports_are_inline() -> None:
        """Fail if any file imports cv2/open3d/rerun at module level."""
        hits = find_eager_heavy_imports()
        if hits:
            listing = "\n".join(
                f"  - dimos/{f}:{line}: `{module}`"
                for f, lines in sorted(hits.items())
                for line, module in lines
            )
>           raise AssertionError(
                f"Found module-level import(s) of {'/'.join(HEAVY_MODULES)}:\n{listing}\n\n"
                "These libraries load large native extensions into every process that "
                "transitively imports the module. Import them inside the function or "
                "method that uses them; imports needed only for type annotations go "
                "under `if TYPE_CHECKING:`."
            )
E           AssertionError: Found module-level import(s) of cv2/open3d/rerun:
E             - .../benchmark/libero_pro/video.py:9: `cv2`
E           
E           These libraries load large native extensions into every process that transitively imports the module. Import them inside the function or method that uses them; imports needed only for type annotations go under `if TYPE_CHECKING:`.

hits       = {'benchmark/libero_pro/video.py': [(9, 'cv2')]}
listing    = '  - .../benchmark/libero_pro/video.py:9: `cv2`'

dimos/codebase_checks/test_inline_heavy_imports.py:92: AssertionError

dimos.benchmark.evaluation.test_policy_runtime::test_policy_result_larger_than_pipe_buffer_does_not_deadlock

Stack Traces | 2.01s run time

def test_policy_result_larger_than_pipe_buffer_does_not_deadlock() -> None:
        context = multiprocessing.get_context("spawn")
        messages, worker_messages = context.Pipe(duplex=False)
        start_event = context.Event()
        result_sending = context.Event()
        process = context.Process(
            target=_send_large_policy_error,
            args=(worker_messages, start_event, result_sending),
            daemon=True,
        )
        process.start()
        worker_messages.close()
        execution = _PolicyExecutionProcess(process, messages, start_event)
        execution.start()
>       assert result_sending.wait(timeout=2)
E       assert False
E        +  where False = wait(timeout=2)
E        +    where wait = <Event at 0xff2de407f4d0 unset>.wait

context    = <multiprocessing.context.SpawnContext object at 0xff2edd8dcda0>
execution  = <dimos.benchmark.evaluation.runtime._PolicyExecutionProcess object at 0xff2de46923c0>
messages   = <multiprocessing.connection.Connection object at 0xff2de410cb90>
process    = <SpawnProcess name='SpawnProcess-57' pid=15908 parent=3063 started daemon>
result_sending = <Event at 0xff2de407f4d0 unset>
start_event = <Event at 0xff2de410f7d0 set>
worker_messages = <multiprocessing.connection.Connection object at 0xff2de410dc40>

.../benchmark/evaluation/test_policy_runtime.py:193: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}