diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index e9e6ddfb9..08b473ccc 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -20,6 +20,7 @@ import shutil import click from .. import _yaml +from .._frontend.app import App from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope @@ -377,8 +378,6 @@ def cli(context, **kwargs): user preferences configuration file. """ - from .app import App - # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) @@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_): metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) +@click.option( + "--with", + "other_targets", + type=click.Path(readable=False), + multiple=True, + help="A additional target to stage into an element's sandbox environment", +) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", @@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_): @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( - app, + app: App, target, + other_targets, command, mount, isolate, @@ -748,13 +755,38 @@ def shell( otherwise bst may respond to them instead. e.g. \b - bst shell example.bst -- df -h + bst shell base.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. + Use the --with option to stage the artifacts of other elements + into the temporary sysroot to make them available to run e.g. + + \b + bst shell --with base.bst example.bst -- cat example.txt + If no COMMAND is specified, the default is to attempt to run an interactive shell. + + # Examples: + + \b + # Attempt to run an interactive shell with example.bst + bst shell example.bst + # Attempt to run an df -h with example.bst + bst shell example.bst -- df h + # In a workspace directory, attempt to shell into the workspace element + bst shell + # Attempt to run cat from base.bst to read example.txt from example.bst + bst shell --with base.bst example.bst -- cat example.txt + # Attempt to run an interactive shell with the sources and all dependencies of example.bst + bst shell --build example.bst + + For all examples on this page: + - example.bst is a simple import element with no dependencies + that imports a file called example.txt + - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). """ # Buildtree can only be used with build shells @@ -776,6 +808,7 @@ def shell( target, scope, app.shell_prompt, + other_targets=other_targets, mounts=mounts, isolate=isolate, command=command, diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 913bc3f17..82e6a271d 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -243,6 +243,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell + # other_targets (Iterable[str]): The name of other elements to stage in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds @@ -262,6 +263,7 @@ def shell( scope: int, prompt: Callable[[Element], str], *, + other_targets: Iterable[str] = (), unique_id: Optional[str] = None, mounts: Optional[List[_HostMount]] = None, isolate: bool = False, @@ -357,8 +359,46 @@ def shell( self._fetch([element]) _pipeline.assert_sources_cached(self._context, [element]) + # Load the other targets + try: + other_elements = self.load_selection( + other_targets, + selection=_PipelineSelection.RUN, + load_artifacts=True, + connect_artifact_cache=True, + connect_source_cache=True, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) + except StreamError as e: + if e.reason == "deps-not-supported": + raise StreamError( + "Only buildtrees are supported with artifact names", + detail="Use the --build and --use-buildtree options to shell into a cached build tree", + reason="only-buildtrees-supported", + ) from e + raise + + self.query_cache(other_elements) + self._pull_missing_artifacts(other_elements) + missing_deps = [dep for dep in _pipeline.dependencies(other_elements, _Scope.RUN) if not dep._cached()] + if missing_deps: + raise StreamError( + "Elements need to be built or downloaded before staging a shell environment", + detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))), + reason="shell-missing-deps", + ) + return element._shell( - scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree + scope, + mounts=mounts, + isolate=isolate, + prompt=prompt(element), + command=command, + usebuildtree=usebuildtree, + other_elements=other_elements, ) # build() diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 6ac1ff1eb..d195ffc15 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,6 +62,9 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations + import os import re import stat @@ -74,6 +77,7 @@ from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence + from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml @@ -90,7 +94,7 @@ from .sandbox import _SandboxFlags, SandboxCommandError from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote -from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey +from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _HostMount from ._artifact import Artifact from ._elementproxy import ElementProxy from ._elementsources import ElementSources @@ -2058,9 +2062,20 @@ def _push(self): # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source + # other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox # # Returns: Exit code - def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False): + def _shell( + self, + scope: int | None = None, + *, + mounts: List[_HostMount] | None = None, + isolate: bool = False, + prompt: str | None = None, + command: List[str] | None = None, + usebuildtree: bool = False, + other_elements: List[Element] | None = None, + ): with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox: environment = sandbox._get_configured_environment() or self.get_environment() @@ -2076,6 +2091,17 @@ def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command if prompt is not None: environment["PS1"] = prompt + with self.timed_activity("Staging other_targets", silent_nested=True), self.__collect_overlaps(sandbox): + self.stage_dependency_artifacts(sandbox, other_elements) + + if other_elements: + # Stage artifacts from other_elements into the sandbox. + for element in other_elements: + # Stage deps in the sandbox root + with element.timed_activity("Integrating sandbox"), sandbox.batch(): + for dep in element._dependencies(_Scope.RUN): + dep.integrate(sandbox) + # Special configurations for non-isolated sandboxes if not isolate: diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..4d0b4ce6b 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,12 +16,15 @@ # pylint: disable=redefined-outer-name import os +from typing import Dict, List, Tuple import uuid + import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,11 +49,25 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` -# -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +# other_elements (list(str)): Other elements to stage in the sandbox +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: None | Dict = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + other_elements: List[str] | None = None +): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 + if other_elements is not None: + for other_element in other_elements: + result = cli.run_project_config(project=project, project_config=config, args=["build", other_element]) + assert result.exit_code == 0 args = ["shell"] if isolate: @@ -58,6 +75,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] + if other_elements is not None: + for other_element in other_elements: + args += ["--with", other_element] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) @@ -86,6 +106,28 @@ def test_executable(cli, datafiles): assert result.output == "Horseys!\n" +# Test staging and running additional targets in the shell of the main target for debugging. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_with_other_targets(cli, datafiles): + project = str(datafiles) + + # Show we can't cat in a shell for manual/import-file.bst + result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst") + assert ( + result.exit_code == -1 + ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies" + + # Show we can now cat with base.bst in a shell for manual/import-file.bst + result = execute_shell( + cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"] + ) + assert ( + result.exit_code == 0 + ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat" + assert result.output == "This is a test\n" + + # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR)