Shell API and CLI: Add option for staging additional runtime targets

src/buildstream/_frontend/cli.py

@@ -20,6 +20,7 @@
 import shutil
 import click
 from .. import _yaml
+from .._frontend.app import App
 from .._exceptions import BstError, LoadError, AppError, RemoteError
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
@@ -377,8 +378,6 @@ def cli(context, **kwargs):
     user preferences configuration file.
     """
 
-    from .app import App
-
     # Create the App, giving it the main arguments
     context.obj = App.create(dict(kwargs))
     context.call_on_close(context.obj.cleanup)
@@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_):
     metavar="HOSTPATH PATH",
     help="Mount a file or directory into the sandbox",
 )
+@click.option(
+    "--with",
+    "other_targets",
+    type=click.Path(readable=False),
+    multiple=True,
+    help="A additional target to stage into an element's sandbox environment",
+)
 @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
 @click.option(
     "--use-buildtree",
@@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_):
 @click.argument("command", type=click.STRING, nargs=-1)
 @click.pass_obj
 def shell(
-    app,
+    app: App,
     target,
+    other_targets,
     command,
     mount,
     isolate,
@@ -748,13 +755,38 @@ def shell(
     otherwise bst may respond to them instead. e.g.
 
     \b
-        bst shell example.bst -- df -h
+        bst shell base.bst -- df -h
 
     Use the --build option to create a temporary sysroot for
     building the element instead.
 
+    Use the --with option to stage the artifacts of other elements
+    into the temporary sysroot to make them available to run e.g.
+
+    \b
+        bst shell --with base.bst example.bst -- cat example.txt
+
     If no COMMAND is specified, the default is to attempt
     to run an interactive shell.
+
+    # Examples:
+
+    \b
+        # Attempt to run an interactive shell with example.bst
+        bst shell example.bst
+        # Attempt to run an df -h with example.bst
+        bst shell example.bst -- df h
+        # In a workspace directory, attempt to shell into the workspace element
+        bst shell
+        # Attempt to run cat from base.bst to read example.txt from example.bst
+        bst shell --with base.bst example.bst -- cat example.txt
+        # Attempt to run an interactive shell with the sources and all dependencies of example.bst
+        bst shell --build example.bst
+
+    For all examples on this page:
+    - example.bst is a simple import element with no dependencies
+       that imports a file called example.txt
+    - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).
     """
 
     # Buildtree can only be used with build shells
@@ -776,6 +808,7 @@ def shell(
                 target,
                 scope,
                 app.shell_prompt,
+                other_targets=other_targets,
                 mounts=mounts,
                 isolate=isolate,
                 command=command,

src/buildstream/_stream.py

@@ -243,6 +243,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     #    target: The name of the element to run the shell for
     #    scope: The scope for the shell, only BUILD or RUN are valid (_Scope)
     #    prompt: A function to return the prompt to display in the shell
+    #    other_targets (Iterable[str]): The name of other elements to stage in the shell
     #    unique_id: (str): A unique_id to use to lookup an Element instance
     #    mounts: Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
@@ -262,6 +263,7 @@ def shell(
         scope: int,
         prompt: Callable[[Element], str],
         *,
+        other_targets: Iterable[str] = (),
         unique_id: Optional[str] = None,
         mounts: Optional[List[_HostMount]] = None,
         isolate: bool = False,
@@ -357,8 +359,46 @@ def shell(
             self._fetch([element])
             _pipeline.assert_sources_cached(self._context, [element])
 
+        # Load the other targets
+        try:
+            other_elements = self.load_selection(
+                other_targets,
+                selection=_PipelineSelection.RUN,
+                load_artifacts=True,
+                connect_artifact_cache=True,
+                connect_source_cache=True,
+                artifact_remotes=artifact_remotes,
+                source_remotes=source_remotes,
+                ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                ignore_project_source_remotes=ignore_project_source_remotes,
+            )
+        except StreamError as e:
+            if e.reason == "deps-not-supported":
+                raise StreamError(
+                    "Only buildtrees are supported with artifact names",
+                    detail="Use the --build and --use-buildtree options to shell into a cached build tree",
+                    reason="only-buildtrees-supported",
+                ) from e
+            raise
+
+        self.query_cache(other_elements)
+        self._pull_missing_artifacts(other_elements)
+        missing_deps = [dep for dep in _pipeline.dependencies(other_elements, _Scope.RUN) if not dep._cached()]
+        if missing_deps:
+            raise StreamError(
+                "Elements need to be built or downloaded before staging a shell environment",
+                detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+                reason="shell-missing-deps",
+            )
+
         return element._shell(
-            scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree
+            scope,
+            mounts=mounts,
+            isolate=isolate,
+            prompt=prompt(element),
+            command=command,
+            usebuildtree=usebuildtree,
+            other_elements=other_elements,
         )
 
     # build()

src/buildstream/element.py

@@ -62,6 +62,9 @@
 ---------------
 """
 
+# For 3.7+ support, not necessary and deprecated in 3.14+
+from __future__ import annotations
+
 import os
 import re
 import stat
@@ -74,6 +77,7 @@
 from threading import Lock
 from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence
 
+
 from pyroaring import BitMap  # pylint: disable=no-name-in-module
 
 from . import _yaml
@@ -90,7 +94,7 @@
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _HostMount
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -2058,9 +2062,20 @@ def _push(self):
     #    prompt (str): A suitable prompt string for PS1
     #    command (list): An argv to launch in the sandbox
     #    usebuildtree (bool): Use the buildtree as its source
+    #    other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox
     #
     # Returns: Exit code
-    def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False):
+    def _shell(
+        self,
+        scope: int | None = None,
+        *,
+        mounts: List[_HostMount] | None = None,
+        isolate: bool = False,
+        prompt: str | None = None,
+        command: List[str] | None = None,
+        usebuildtree: bool = False,
+        other_elements: List[Element] | None = None,
+    ):
 
         with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox:
             environment = sandbox._get_configured_environment() or self.get_environment()
@@ -2076,6 +2091,17 @@ def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command
             if prompt is not None:
                 environment["PS1"] = prompt
 
+            with self.timed_activity("Staging other_targets", silent_nested=True), self.__collect_overlaps(sandbox):
+                self.stage_dependency_artifacts(sandbox, other_elements)
+
+            if other_elements:
+                # Stage artifacts from other_elements into the sandbox.
+                for element in other_elements:
+                    # Stage deps in the sandbox root
+                    with element.timed_activity("Integrating sandbox"), sandbox.batch():
+                        for dep in element._dependencies(_Scope.RUN):
+                            dep.integrate(sandbox)
+
             # Special configurations for non-isolated sandboxes
             if not isolate:
 

tests/integration/shell.py

@@ -16,12 +16,15 @@
 # pylint: disable=redefined-outer-name
 
 import os
+from typing import Dict, List, Tuple
 import uuid
 
+
 import pytest
 
 from buildstream import _yaml
 from buildstream._testing import cli_integration as cli  # pylint: disable=unused-import
+from buildstream._testing.runcli import CliIntegration
 from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN
 from buildstream.exceptions import ErrorDomain
 from buildstream import utils
@@ -46,18 +49,35 @@
 #    mount (tuple): A (host, target) tuple for the `--mount` option
 #    element (str): The element to build and run a shell with
 #    isolate (bool): Whether to pass --isolate to `bst shell`
-#
-def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False):
+#    other_elements (list(str)): Other elements to stage in the sandbox
+def execute_shell(
+    cli: CliIntegration,
+    project: str,
+    command: List[str],
+    *,
+    config: None | Dict = None,
+    mount: Tuple[str, str] | None = None,
+    element: str = "base.bst",
+    isolate: bool = False,
+    other_elements: List[str] | None = None
+):
     # Ensure the element is built
     result = cli.run_project_config(project=project, project_config=config, args=["build", element])
     assert result.exit_code == 0
+    if other_elements is not None:
+        for other_element in other_elements:
+            result = cli.run_project_config(project=project, project_config=config, args=["build", other_element])
+            assert result.exit_code == 0
 
     args = ["shell"]
     if isolate:
         args += ["--isolate"]
     if mount is not None:
         host_path, target_path = mount
         args += ["--mount", host_path, target_path]
+    if other_elements is not None:
+        for other_element in other_elements:
+            args += ["--with", other_element]
     args += [element, "--", *command]
 
     return cli.run_project_config(project=project, project_config=config, args=args)
@@ -86,6 +106,28 @@ def test_executable(cli, datafiles):
     assert result.output == "Horseys!\n"
 
 
+# Test staging and running additional targets in the shell of the main target for debugging.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_with_other_targets(cli, datafiles):
+    project = str(datafiles)
+
+    # Show we can't cat in a shell for manual/import-file.bst
+    result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst")
+    assert (
+        result.exit_code == -1
+    ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies"
+
+    # Show we can now cat with base.bst in a shell for manual/import-file.bst
+    result = execute_shell(
+        cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"]
+    )
+    assert (
+        result.exit_code == 0
+    ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat"
+    assert result.output == "This is a test\n"
+
+
 # Test shell environment variable explicit assignments
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)