diff --git a/CHANGELOG.md b/CHANGELOG.md index f15ddd44..3f29e005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Added `cloudsmith credential-helper install maven`, `install gradle`, and `install sbt`. They set up the Cloudsmith Maven tools, which get repository credentials from `cloudsmith credential-helper generic`. Each command installs the newest release from the public `cloudsmith/maven-tools` repository, for every build of the user. - - `maven` puts the extension in `~/.m2/repository` and adds it to `~/.m2/extensions.xml`, which Maven 3.10 and later read. When `mvn -v` reports an earlier Maven, the command shows a warning. With `--local`, the command adds the extension to `.mvn/extensions.xml` of the Maven project in the current directory instead. With `--mavenrc`, it adds a block to `~/.mavenrc` instead, which the `mvn` and `mvnw` scripts of all Maven versions read. `--mavenrc` does not work on Windows or when the path of the local Maven repository contains a space or a `:`. In these cases the command stops and tells you to use `--local` or Maven 3.10. - - `gradle` writes an init script to `~/.gradle/init.d` (or `$GRADLE_USER_HOME/init.d`), which applies the plugin to every Gradle build. - - `sbt` writes a global plugin to `~/.sbt/1.0/plugins`, which adds the plugin to every sbt 1 build. With `--local`, it writes `project/cloudsmith.sbt` of the sbt project in the current directory instead. + - `maven` puts the extension in the local repository that Maven reports (`~/.m2/repository` when the command cannot run `mvn`) and adds it to `~/.m2/extensions.xml`, which Maven 3.10 and later read. When `mvn -v` reports an earlier Maven, the command shows a warning. With `--local`, the command adds the extension to `.mvn/extensions.xml` of the Maven project in the current directory instead. With `--mavenrc`, it adds a block to `~/.mavenrc` instead, which the `mvn` and `mvnw` scripts of all Maven versions read. `--mavenrc` does not work on Windows or when the path of the local Maven repository contains a space or a `:`. In these cases the command stops and tells you to use `--local` or Maven 3.10. + - `gradle` writes an init script to `init.d` of the Gradle user home that `gradle` reports (`$GRADLE_USER_HOME` or `~/.gradle` when the command cannot run `gradle`), which applies the plugin to every Gradle build. + - `sbt` writes a global plugin to the global plugins directory that `sbt` reports (`~/.sbt/1.0/plugins` when the command cannot run `sbt`), which adds the plugin to every sbt 1 build. With `--local`, it writes `project/cloudsmith.sbt` of the sbt project in the current directory instead. + - To find these locations, the commands run Maven, Gradle or sbt, which can take a few seconds. The commands show a message while they wait. When Maven, Gradle or sbt reports a relative path for these locations, the command stops, because each build would look in its own directory. + - `credential-helper list` checks only the default locations of Gradle and sbt, so a location that `gradle.user.home` or `sbt.global.base` moves shows as not installed. - The commands add to existing files and keep their other content. `uninstall` removes only what `install` added. - Added `skills` to `cloudsmith upstream`. Use `cloudsmith upstream skills list`, `create`, `update`, and `delete` to manage AI Skills upstreams. An AI Skills upstream caches skills and plugins from a GitHub repository. diff --git a/cloudsmith_cli/cli/commands/credential_helper/manage.py b/cloudsmith_cli/cli/commands/credential_helper/manage.py index 8459ac51..87980080 100644 --- a/cloudsmith_cli/cli/commands/credential_helper/manage.py +++ b/cloudsmith_cli/cli/commands/credential_helper/manage.py @@ -9,6 +9,7 @@ from __future__ import annotations import sys +from functools import partial from urllib.parse import urlsplit import click @@ -25,6 +26,7 @@ MavenrcNotSupported, ) from cloudsmith_cli.credential_helpers.maven_tools import ( + BuildToolError, ConfigFileConflict, ProjectNotFound, ) @@ -438,6 +440,7 @@ def install_cmd( if getattr(installer, "KEEPS_INSTALL_TIME_SETTINGS", False): install_kwargs["launcher_env"] = _launcher_settings_from_cli(ctx) + install_kwargs.update(_progress_option(installer)) try: actions = installer.install(**install_kwargs) @@ -451,6 +454,7 @@ def install_cmd( ConfigFileConflict, UnsafeLauncherValue, MavenrcNotSupported, + BuildToolError, ) as exc: raise click.ClickException(str(exc)) except OSError as exc: @@ -550,8 +554,14 @@ def uninstall_cmd( bin_dir=bin_dir, dry_run=dry_run, **_helper_options(ctx, helper), + **_progress_option(installer), ) - except (ConfigFileParseError, ProjectNotFound, ConfigFileConflict) as exc: + except ( + ConfigFileParseError, + ProjectNotFound, + ConfigFileConflict, + BuildToolError, + ) as exc: raise click.ClickException(str(exc)) except OSError as exc: raise click.ClickException( @@ -578,6 +588,13 @@ def uninstall_cmd( click.secho(f" {warning}" if dry_run else warning, err=True, fg="yellow") +def _progress_option(installer) -> dict: + """Return the callback that says why the command waits, for an installer that asks a build tool.""" + if getattr(installer, "ASKS_BUILD_TOOL", False): + return {"progress": partial(click.echo, err=True)} + return {} + + # --------------------------------------------------------------------------- # list # --------------------------------------------------------------------------- @@ -593,6 +610,11 @@ def list_cmd(ctx, opts) -> None: Shows which helpers are available, whether their launcher binary is present on PATH, and which registry hosts they are configured for. + For gradle and sbt, list checks only the default locations: init.d in + $GRADLE_USER_HOME or ~/.gradle, and ~/.sbt/1.0/plugins. Install and + uninstall ask Gradle and sbt where they read plugins. If gradle.user.home + or sbt.global.base moves that location, list shows "not installed". + Example: \b diff --git a/cloudsmith_cli/cli/tests/test_credential_helper_maven_tools.py b/cloudsmith_cli/cli/tests/test_credential_helper_maven_tools.py index fd753023..67764cbe 100644 --- a/cloudsmith_cli/cli/tests/test_credential_helper_maven_tools.py +++ b/cloudsmith_cli/cli/tests/test_credential_helper_maven_tools.py @@ -9,7 +9,11 @@ import errno import os +import shutil import subprocess +import sys +import tempfile +import time from contextlib import nullcontext from pathlib import Path from unittest.mock import Mock, patch @@ -24,7 +28,11 @@ from ...credential_helpers.maven import extensions, mavenrc from ...credential_helpers.maven import installer as maven_installer from ...credential_helpers.maven.installer import MavenInstaller, MavenrcNotSupported -from ...credential_helpers.maven_tools import ConfigFileConflict, ProjectNotFound +from ...credential_helpers.maven_tools import ( + ConfigFileConflict, + ProjectNotFound, + BuildToolError, +) from ...credential_helpers.sbt.installer import SbtInstaller EXTENSION_PATH = "io/cloudsmith/cloudsmith-maven-extension" @@ -90,6 +98,25 @@ def fake_download(artifact_path, destination, session=None): return downloads +@pytest.fixture(autouse=True) +def tool_runs(monkeypatch): + """Run as if no build tool is on PATH, and as if each one reports nothing.""" + commands = [] + which = shutil.which + + def fake_output(command, cwd=None): + commands.append((command, cwd)) + return "" + + monkeypatch.setattr(maven_tools, "tool_output", fake_output) + monkeypatch.setattr( + shutil, + "which", + lambda name: None if name in ("mvn", "gradle", "sbt") else which(name), + ) + return commands + + @pytest.fixture(autouse=True) def _maven_3_10(monkeypatch): """Run as if `mvn -v` reports Maven 3.10.0.""" @@ -725,6 +752,23 @@ def _user_extensions(home: Path) -> Path: return home / ".m2" / "extensions.xml" +def _skip_unless_it_runs(tool: str) -> None: + executable = shutil.which(tool) + if ( + executable is None + or subprocess.run( + [executable, "--version"], capture_output=True, timeout=120, check=False + ).returncode + ): + pytest.skip(f"{tool} is not installed or does not run") + + +def _jar(repository: Path) -> Path: + return ( + repository / EXTENSION_PATH / "0.2.0" / "cloudsmith-maven-extension-0.2.0.jar" + ) + + def test_maven_install_puts_the_extension_in_the_local_repository( home, fake_repository, outside_projects ): @@ -794,6 +838,220 @@ def test_maven_install_marks_the_files_as_installed( ) +def test_maven_install_uses_the_local_repository_that_maven_reports( + home, fake_repository, outside_projects, monkeypatch +): + """The JAR goes in the repository from the debug output of Maven, and --mavenrc loads it there.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: ( + "[DEBUG] Using manager EnhancedLocalRepositoryManager" + f" with priority 10.0 for {home / 'repo'}\n" + ), + ) + + MavenInstaller().install(mavenrc=True) + + assert _jar(home / "repo").exists() + text = (home / ".mavenrc").read_text(encoding="utf-8") + assert f"-Dmaven.ext.class.path={_jar(home / 'repo')}" in text + + +# The parameter replaces the autouse fake, so the real tool_output runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_tool_output_has_stdout_and_stderr_without_colors(tool_runs): + """A build tool can log to stderr and in color, so the output has both streams without colors.""" + script = "import sys; print('\\x1b[1;36mDEBUG\\x1b[m out'); print('err', file=sys.stderr)" + + output = maven_tools.tool_output([sys.executable, "-c", script]) + + assert sorted(output.split()) == ["DEBUG", "err", "out"] + + +@pytest.mark.windows +@pytest.mark.parametrize( + ("interrupt", "error"), + [ + (subprocess.TimeoutExpired("tool", 1), BuildToolError), + (KeyboardInterrupt(), KeyboardInterrupt), + ], +) +# The parameter replaces the autouse fake, so the real tool_output runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_tool_output_stops_the_tool_and_its_children( + tool_runs, interrupt, error, tmp_path, monkeypatch +): + """A launcher script starts a JVM, so a timeout or Ctrl+C stops the child too.""" + communicate = subprocess.Popen.communicate + tools = [] + + def interrupted_communicate(self, data=None, timeout=None): + if tools: # A later process, such as taskkill, stops the tool. + return communicate(self, data, timeout) + tools.append(self) + time.sleep(1) + raise interrupt + + monkeypatch.setattr(subprocess.Popen, "communicate", interrupted_communicate) + alive_file = tmp_path / "alive" + child = f"import pathlib, time; time.sleep(2); pathlib.Path({str(alive_file)!r}).touch()" + script = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, '-c', {child!r}])\n" + "time.sleep(60)\n" + ) + + with pytest.raises(error): + maven_tools.tool_output([sys.executable, "-c", script]) + monkeypatch.setattr(subprocess.Popen, "communicate", communicate) + time.sleep(2.5) + + assert not alive_file.exists() + + +@pytest.mark.parametrize("relative_path", ["repo", ".", "../repo"]) +def test_maven_install_with_a_relative_local_repository_fails( + relative_path, home, fake_repository, outside_projects, monkeypatch +): + """Without local, a relative repository depends on the directory of each build, so the install stops.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: ( + "[DEBUG] Using manager EnhancedLocalRepositoryManager" + f" with priority 10.0 for {cwd / relative_path}\n" + ), + ) + + with pytest.raises(BuildToolError): + MavenInstaller().install() + + assert not fake_repository + + +def test_maven_install_without_a_second_answer_from_maven_uses_the_default( + home, fake_repository, outside_projects, monkeypatch +): + """When the second Maven run reports nothing, the install uses ~/.m2/repository.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + outputs = iter( + [ + "[DEBUG] Using manager EnhancedLocalRepositoryManager" + f" with priority 10.0 for {home / 'repo'}\n", + "", + ] + ) + monkeypatch.setattr( + maven_tools, "tool_output", lambda command, cwd=None: next(outputs) + ) + + actions = MavenInstaller().install() + + assert _jar(home / ".m2" / "repository").exists() + assert actions[0].startswith("WARNING: mvn did not report its local repository") + + +@pytest.mark.parametrize( + ("installer", "tool"), [(GradleInstaller, "gradle"), (SbtInstaller, "sbt")] +) +def test_install_warns_when_the_build_tool_reports_nothing( + installer, tool, home, fake_repository, monkeypatch +): + """A build tool on PATH that reports nothing gets the default location and a warning.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + + actions = installer().install() + + assert actions[0] == ( + f"WARNING: {tool} did not report its {installer.reported}, " + f"so the command uses {installer().path()}." + ) + assert installer().path().exists() + + +# The parameter replaces the autouse fake, so the real tool_output runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_tool_output_runs_a_tool_on_a_relative_path_in_another_directory( + tool_runs, tmp_path, monkeypatch +): + """PATH can have a relative directory, so the tool path does not depend on cwd.""" + monkeypatch.chdir(Path(sys.executable).parent) + other_dir = tmp_path / "other" + other_dir.mkdir() + + output = maven_tools.tool_output( + [os.path.join(".", Path(sys.executable).name), "-c", "print('ran')"], + other_dir, + ) + + assert output.strip() == "ran" + + +@pytest.mark.parametrize( + ("local", "wrapper"), + [(False, True), (True, False), (True, True)], +) +def test_maven_install_asks_the_maven_that_the_build_uses( + local, + wrapper, + home, + fake_repository, + maven_project, + tool_runs, + monkeypatch, +): + """With local, the Maven wrapper runs in the current module; else mvn runs outside the project. No goal runs.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + (maven_project / ".mvn").mkdir() + mvnw = maven_project / "mvnw" + if wrapper: + mvnw.write_text("", encoding="utf-8") + module = maven_project / "module" + module.mkdir() + monkeypatch.chdir(module) + + MavenInstaller().install(local=local) + + [(command, cwd)] = tool_runs + executable = str(mvnw) if local and wrapper else "/bin/mvn" + assert command == [ + executable, + "-X", + "-o", + "-B", + "cloudsmith-local-repository-probe", + ] + assert (cwd == module) is local + + +@pytest.mark.windows +# The parameter replaces the autouse fake, so the real mvn runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_maven_install_uses_the_local_repository_of_the_real_maven( + tool_runs, home, fake_repository, outside_projects, tmp_path, monkeypatch +): + """The real mvn reports the repository on stderr too, and a project around the temporary directory does not count.""" + _skip_unless_it_runs("mvn") + project = tmp_path / "project" + (project / ".mvn").mkdir(parents=True) + (project / ".mvn" / "maven.config").write_text( + f"-Dmaven.repo.local={tmp_path / 'project-repo'}\n", encoding="utf-8" + ) + monkeypatch.setattr(tempfile, "tempdir", str(project)) + monkeypatch.setenv( + "MAVEN_OPTS", + f"-Dmaven.repo.local={tmp_path / 'repo'}" + " -Dorg.slf4j.simpleLogger.logFile=System.err", + ) + + MavenInstaller().install() + + assert _jar(tmp_path / "repo").exists() + + def test_maven_install_local_turns_on_the_extension_in_the_project( home, fake_repository, maven_project ): @@ -1040,6 +1298,40 @@ def test_gradle_install_uses_gradle_user_home( assert (tmp_path / "gradle-home" / "init.d" / "cloudsmith.init.gradle.kts").exists() +def test_gradle_install_and_uninstall_use_the_user_home_that_gradle_reports( + home, fake_repository, monkeypatch +): + """The init script goes in init.d of the Gradle user home that Gradle reports.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: f"CLOUDSMITH_GRADLE_USER_HOME={home / 'reported'}\n", + ) + init_script = home / "reported" / "init.d" / "cloudsmith.init.gradle.kts" + + GradleInstaller().install() + assert init_script.exists() + + GradleInstaller().uninstall() + assert not init_script.exists() + + +@pytest.mark.windows +# The parameter replaces the autouse fake, so the real gradle runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_gradle_install_uses_the_user_home_of_the_real_gradle( + tool_runs, home, fake_repository, tmp_path, monkeypatch +): + """The real gradle reports gradle.user.home from GRADLE_OPTS.""" + _skip_unless_it_runs("gradle") + monkeypatch.setenv("GRADLE_OPTS", f"-Dgradle.user.home={tmp_path / 'gradle-home'}") + + GradleInstaller().install() + + assert (tmp_path / "gradle-home" / "init.d" / "cloudsmith.init.gradle.kts").exists() + + def test_gradle_uninstall_and_status(home, fake_repository): """Uninstall removes the init script, and status follows it.""" script = home / ".gradle" / "init.d" / "cloudsmith.init.gradle.kts" @@ -1117,6 +1409,85 @@ def test_sbt_install_writes_a_global_plugin(home, fake_repository): assert any(str(plugin_file) in action for action in actions) +def test_sbt_install_and_uninstall_use_the_global_plugins_that_sbt_reports( + home, fake_repository, monkeypatch +): + """The plugin file goes in the global plugins directory that sbt 1 reports.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: ( + f"CLOUDSMITH_SBT_GLOBAL_PLUGINS={home / 'reported'}\n" + ), + ) + plugin_file = home / "reported" / "cloudsmith.sbt" + + SbtInstaller().install() + assert plugin_file.exists() + + SbtInstaller().uninstall() + assert not plugin_file.exists() + + +def test_sbt_install_with_a_relative_global_base_and_a_broken_plugin_fails( + home, fake_repository, monkeypatch +): + """sbt can name a relative global plugins directory, which changes with the directory of each build.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: ( + "[info] welcome to sbt 1.13.0 (Java)\n" + "[info] loading global plugins from relative/plugins\n" + ), + ) + + with pytest.raises(BuildToolError): + SbtInstaller().install() + + +@pytest.mark.parametrize( + ("sbt_version", "uses_reported"), [("1.13.0", True), ("2.0.0", False)] +) +def test_sbt_install_finds_the_global_plugins_when_a_plugin_fails_to_load( + sbt_version, uses_reported, home, fake_repository, monkeypatch +): + """sbt names the global plugins directory before it fails to load a plugin there; sbt 2 uses its own.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd=None: ( + f"[info] welcome to sbt {sbt_version} (Java)\n" + f"[info] loading global plugins from {home / 'reported'}\n" + "[error] Not found\n" + ), + ) + + SbtInstaller().install() + + reported_file = home / "reported" / "cloudsmith.sbt" + assert reported_file.exists() is uses_reported + assert SbtInstaller().path().exists() is not uses_reported + + +@pytest.mark.windows +# The parameter replaces the autouse fake, so the real sbt runs. +@pytest.mark.parametrize("tool_runs", [None]) +def test_sbt_install_uses_the_global_base_of_the_real_sbt( + tool_runs, home, fake_repository, tmp_path, monkeypatch +): + """The real sbt reports sbt.global.base from SBT_OPTS.""" + _skip_unless_it_runs("sbt") + monkeypatch.setenv("SBT_OPTS", f"-Dsbt.global.base={tmp_path / 'sbt-base'}") + + SbtInstaller().install() + + assert (tmp_path / "sbt-base" / "plugins" / "cloudsmith.sbt").exists() + + def test_sbt_install_local_writes_a_project_plugin( home, fake_repository, outside_projects ): @@ -1327,6 +1698,44 @@ def test_cli_with_a_config_file_that_install_did_not_write_fails_cleanly( assert str(plugin_file) in result.output +@pytest.mark.parametrize( + ("command_name", "helper", "message"), + [ + ("install", "maven", "Asking mvn for its local repository."), + ("install", "gradle", "Asking gradle for its Gradle user home."), + ("uninstall", "sbt", "Asking sbt for its global plugins directory."), + ], +) +def test_cli_says_why_it_waits_for_the_build_tool( + command_name, helper, message, home, fake_repository, outside_projects, monkeypatch +): + """A build tool can take some seconds to start, so the command says what it waits for.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + + result = _invoke(command_name, [helper]) + + assert result.exit_code == 0 + assert f"{message} This can take a few seconds." in result.output + + +@pytest.mark.parametrize("command_name", ["install", "uninstall"]) +def test_cli_with_a_relative_tool_path_fails_cleanly( + command_name, home, fake_repository, outside_projects, monkeypatch +): + """A relative Gradle user home stops the command with the reason, without a traceback.""" + monkeypatch.setattr(maven_installer.shutil, "which", lambda name: f"/bin/{name}") + monkeypatch.setattr( + maven_tools, + "tool_output", + lambda command, cwd: f"CLOUDSMITH_GRADLE_USER_HOME={cwd / 'rel'}\n", + ) + + result = _invoke(command_name, ["gradle"]) + + assert result.exit_code == 1 + assert "relative path" in result.output + + @pytest.mark.parametrize( ("command_name", "helper", "option"), [ diff --git a/cloudsmith_cli/credential_helpers/gradle/installer.py b/cloudsmith_cli/credential_helpers/gradle/installer.py index ec362119..329f43d5 100644 --- a/cloudsmith_cli/credential_helpers/gradle/installer.py +++ b/cloudsmith_cli/credential_helpers/gradle/installer.py @@ -4,10 +4,13 @@ from __future__ import annotations import os +import re from pathlib import Path +from .. import maven_tools from ..maven_tools import REPOSITORY_URL, ConfigFileInstaller +INIT_SCRIPT_NAME = "cloudsmith.init.gradle.kts" INIT_SCRIPT = """\ // Written by `cloudsmith credential-helper install gradle`. initscript {{ @@ -23,17 +26,38 @@ pluginManager.apply(io.cloudsmith.gradle.CloudsmithPlugin::class.java) }} """ +_PROBE_INIT_SCRIPT = ( + 'println("CLOUDSMITH_GRADLE_USER_HOME=${gradle.gradleUserHomeDir.absolutePath}")\n' +) +_USER_HOME_LINE = re.compile(r"^CLOUDSMITH_GRADLE_USER_HOME=(.+)$", re.MULTILINE) class GradleInstaller(ConfigFileInstaller): name = "gradle" summary = "Gradle plugin that gets credentials from the Cloudsmith CLI" artifact_path = "io/cloudsmith/cloudsmith-gradle-plugin" + tool = "gradle" + reported = "Gradle user home" + relative_message = ( + "The Gradle user home is a relative path, so each build looks for the init " + "script in its own directory. Set an absolute path for gradle.user.home." + ) def path(self) -> Path: gradle_user_home = os.environ.get("GRADLE_USER_HOME") base = Path(gradle_user_home) if gradle_user_home else Path.home() / ".gradle" - return base / "init.d" / "cloudsmith.init.gradle.kts" + return base / "init.d" / INIT_SCRIPT_NAME + + def reported_path(self, executable: str, cwd: Path) -> Path | None: + """Return the init script in the Gradle user home that an init script prints.""" + probe = cwd / "probe.init.gradle" + probe.write_text(_PROBE_INIT_SCRIPT, encoding="utf-8") + output = maven_tools.tool_output( + [executable, "-q", "--offline", "--no-daemon", "-I", str(probe), "help"], + cwd, + ) + match = _USER_HOME_LINE.search(output) + return cwd / match.group(1) / "init.d" / INIT_SCRIPT_NAME if match else None def render(self, version: str) -> str: return INIT_SCRIPT.format(repository_url=REPOSITORY_URL, version=version) diff --git a/cloudsmith_cli/credential_helpers/maven/installer.py b/cloudsmith_cli/credential_helpers/maven/installer.py index fc3c741c..99b8ad79 100644 --- a/cloudsmith_cli/credential_helpers/maven/installer.py +++ b/cloudsmith_cli/credential_helpers/maven/installer.py @@ -17,7 +17,6 @@ import os import re import shutil -import subprocess from functools import partial from pathlib import Path from typing import TYPE_CHECKING @@ -35,7 +34,10 @@ EXTENSION_PATH = f"io/cloudsmith/{extensions.ARTIFACT_ID}" FIRST_MAVEN_WITH_USER_EXTENSIONS = (3, 10) -MAVEN_VERSION_TIMEOUT_SECONDS = 60 +_LOCAL_REPOSITORY_LOG = re.compile( + r"^\[DEBUG\] Using manager \S+ with priority \S+ for (.+)$", re.MULTILINE +) +_UNKNOWN_PHASE = "cloudsmith-local-repository-probe" INSTEAD_OF_MAVENRC = ( "Run `cloudsmith credential-helper install maven --local` in each project " "(Maven 3.8 and later), or upgrade to Maven 3.10 or later and run " @@ -75,8 +77,8 @@ def _mavenrc_file() -> Path: return Path.home() / ".mavenrc" -def _version_dir(version: str) -> Path: - return Path.home() / ".m2" / "repository" / EXTENSION_PATH / version +def _version_dir(repository: Path, version: str) -> Path: + return repository / EXTENSION_PATH / version def _read(path: Path) -> str | None: @@ -91,20 +93,64 @@ def _maven_version() -> str | None: mvn = shutil.which("mvn") if not mvn: return None - try: - result = subprocess.run( - [mvn, "-v"], - capture_output=True, - text=True, - timeout=MAVEN_VERSION_TIMEOUT_SECONDS, - check=False, - ) - except (OSError, subprocess.TimeoutExpired): - return None - match = re.search(r"Apache Maven (\d+(?:\.\d+)+)", result.stdout) + match = re.search( + r"Apache Maven (\d+(?:\.\d+)+)", maven_tools.tool_output([mvn, "-v"]) + ) return match.group(1) if match else None +def _maven_executable(local: bool) -> str | None: + """Return the Maven wrapper of the project with ``local``, else ``mvn`` on ``PATH``.""" + if local: + wrapper = _project_dir() / ("mvnw.cmd" if is_windows() else "mvnw") + if wrapper.is_file(): + return str(wrapper) + return shutil.which("mvn") + + +def _reported_repository(mvn: str, cwd: Path) -> Path | None: + """Return the local repository that Maven logs when it runs in ``cwd``. + + Maven logs the repository, then stops on a phase that does not exist before it + runs a plugin, also for a default goal or goals in ``MAVEN_ARGS``. + """ + output = maven_tools.tool_output([mvn, "-X", "-o", "-B", _UNKNOWN_PHASE], cwd) + match = _LOCAL_REPOSITORY_LOG.search(output) + return cwd / match.group(1) if match else None + + +def _local_repository( + local: bool, progress: Callable[[str], None] | None +) -> tuple[Path, list[str]]: + """Return the local repository that Maven reports, or ``~/.m2/repository``, and any warning. + + With ``local``, Maven runs in the current directory, as a build from there does. + Without ``local``, Maven runs outside projects to skip the project ``.mvn``. + """ + default = Path.home() / ".m2" / "repository" + mvn = _maven_executable(local) + if mvn is None: + return default, [] + tool = Path(mvn).name + if progress: + progress(maven_tools.waiting_message(tool, "local repository")) + if local: + repository = _reported_repository(mvn, maven_tools.current_dir()) + else: + repository = maven_tools.ask_outside_projects( + partial(_reported_repository, mvn), + "The local repository of Maven is a relative path, so each build looks " + "for the extension in its own directory. Set an absolute path for the " + "local repository, or run " + "`cloudsmith credential-helper install maven --local` in each project.", + ) + if repository is None: + return default, [ + maven_tools.no_answer_warning(tool, "local repository", default) + ] + return repository, [] + + def _old_maven_warning() -> list[str]: version = _maven_version() if version is None: @@ -182,6 +228,8 @@ class MavenInstaller: summary = "Maven extension that gets credentials from the Cloudsmith CLI" # The install command passes a session with its proxy and TLS settings. downloads_files = True + # The install command passes a callback that shows the waiting message. + ASKS_BUILD_TOOL = True def install( self, @@ -190,23 +238,26 @@ def install( local: bool = False, mavenrc: bool = False, session: requests.Session | None = None, + progress: Callable[[str], None] | None = None, **_options, ) -> list[str]: - """Put the extension in ``~/.m2/repository`` and turn it on.""" + """Put the extension in the local repository of Maven and turn it on.""" project_file = _project_extensions_file() if local else None if mavenrc and is_windows(): raise MavenrcNotSupported( f"mvn.cmd on Windows does not read ~/.mavenrc. {INSTEAD_OF_MAVENRC}" ) + repository, actions = _local_repository(local, progress) version = maven_tools.latest_version(EXTENSION_PATH, session) - jar = _version_dir(version) / f"{extensions.ARTIFACT_ID}-{version}.jar" + version_dir = _version_dir(repository, version) + jar = version_dir / f"{extensions.ARTIFACT_ID}-{version}.jar" if mavenrc and any(char in f" \t\n{os.pathsep}" for char in str(jar)): raise MavenrcNotSupported( "The mvn script splits MAVEN_OPTS at spaces and Java splits the " f"class path at {os.pathsep}, so they cannot load {jar}. " f"{INSTEAD_OF_MAVENRC}" ) - actions = self._download(version, dry_run, session) + actions += self._download(version_dir, version, dry_run, session) if mavenrc: return ( actions @@ -226,9 +277,12 @@ def install( ) def _download( - self, version: str, dry_run: bool, session: requests.Session | None + self, + version_dir: Path, + version: str, + dry_run: bool, + session: requests.Session | None, ) -> list[str]: - version_dir = _version_dir(version) file_names = [ f"{extensions.ARTIFACT_ID}-{version}.{file_type}" for file_type in ("jar", "pom") diff --git a/cloudsmith_cli/credential_helpers/maven_tools.py b/cloudsmith_cli/credential_helpers/maven_tools.py index f04f35b4..562b88d0 100644 --- a/cloudsmith_cli/credential_helpers/maven_tools.py +++ b/cloudsmith_cli/credential_helpers/maven_tools.py @@ -13,18 +13,27 @@ import os import re import shutil +import signal import stat +import subprocess +import tempfile +from functools import partial from pathlib import Path from typing import TYPE_CHECKING from xml.etree import ElementTree from ..core.session import create_requests_session +from .launchers import is_windows if TYPE_CHECKING: + from collections.abc import Callable + import requests REPOSITORY_URL = "https://dl.cloudsmith.io/public/cloudsmith/maven-tools/maven/" TIMEOUT_SECONDS = 30 +TOOL_TIMEOUT_SECONDS = 60 +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m") class ProjectNotFound(Exception): @@ -46,6 +55,91 @@ class ConfigFileConflict(Exception): """The config file has content that the installer did not write.""" +class BuildToolError(Exception): + """The build tool did not report a path that every build uses.""" + + +def _stop(process: subprocess.Popen[str]) -> None: + """Stop the tool and the processes that it started, such as the JVM of a launcher script.""" + if is_windows(): + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(process.pid)], + capture_output=True, + check=False, + ) + else: + # The tool can finish between the timeout and the kill. + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def tool_output(command: list[str], cwd: Path | None = None) -> str: + """Return what the build tool prints, without colors, or an empty string when it does not run.""" + try: + process = subprocess.Popen( + # PATH can have a relative directory, and the tool runs in another one. + [os.path.abspath(command[0]), *command[1:]], + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + start_new_session=True, + ) + except OSError: + return "" + with process: + try: + output, _ = process.communicate(timeout=TOOL_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired as exc: + _stop(process) + raise BuildToolError( + f"{Path(command[0]).name} did not finish in {TOOL_TIMEOUT_SECONDS} " + "seconds. Make sure that it runs, and then run the command again." + ) from exc + except KeyboardInterrupt: + # The tool runs in its own session, so Ctrl+C does not reach it. + _stop(process) + raise + return _ANSI_ESCAPE.sub("", output) + + +def ask_outside_projects( + ask: Callable[[Path], Path | None], relative_message: str +) -> Path | None: + """Return the path that *ask* gets from the build tool in an empty directory. + + An empty ``.mvn`` and ``settings.gradle`` make each directory a project root, + so Maven and Gradle do not read a project above the temporary directory. + The tool runs again in a subdirectory. A different answer means a relative + path, which changes with the directory of each build. No second answer means + that the tool failed. + """ + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as empty_dir: + nested_dir = Path(empty_dir) / "nested" + for project_dir in (Path(empty_dir), nested_dir): + (project_dir / ".mvn").mkdir(parents=True) + (project_dir / "settings.gradle").write_text("", encoding="utf-8") + answer = ask(Path(empty_dir)) + nested_answer = ask(nested_dir) if answer else None + if nested_answer and nested_answer != answer: + raise BuildToolError(relative_message) + return nested_answer + + +def waiting_message(tool: str, reported: str) -> str: + """Return the message to show while the build tool starts, which can take some seconds.""" + return f"Asking {tool} for its {reported}. This can take a few seconds." + + +def no_answer_warning(tool: str, reported: str, default: Path) -> str: + """Return the warning for a build tool that ran but did not report a location.""" + return ( + f"WARNING: {tool} did not report its {reported}, so the command uses {default}." + ) + + def _get(artifact_path: str, session: requests.Session | None) -> bytes: response = (session or create_requests_session()).get( REPOSITORY_URL + artifact_path, timeout=TIMEOUT_SECONDS @@ -108,6 +202,12 @@ class ConfigFileInstaller: artifact_path = "" # The install command passes a session with its proxy and TLS settings. downloads_files = True + # The install and uninstall commands pass a callback that shows the waiting message. + ASKS_BUILD_TOOL = True + # The build tool that reports where it reads the config file for every build. + tool = "" + reported = "" + relative_message = "" def path(self) -> Path: """Return the config file that loads the plugin for every build.""" @@ -117,6 +217,27 @@ def local_path(self) -> Path: """Return the config file that loads the plugin for the current project.""" raise NotImplementedError + def reported_path(self, executable: str, cwd: Path) -> Path | None: + """Return the config file in the location that the build tool reports in *cwd*.""" + raise NotImplementedError + + def _path_for_every_build( + self, progress: Callable[[str], None] | None + ) -> tuple[Path, list[str]]: + """Return the config file that the build tool reports, and any warning.""" + executable = shutil.which(self.tool) if self.tool else None + if executable is None: + return self.path(), [] + if progress: + progress(waiting_message(self.tool, self.reported)) + path = ask_outside_projects( + partial(self.reported_path, executable), self.relative_message + ) + if path is None: + default = self.path() + return default, [no_answer_warning(self.tool, self.reported, default)] + return path, [] + def render(self, version: str) -> str: """Return the content of the config file for *version*.""" raise NotImplementedError @@ -146,29 +267,39 @@ def install( dry_run: bool = False, local: bool = False, session: requests.Session | None = None, + progress: Callable[[str], None] | None = None, **_options, ) -> list[str]: - path = self.local_path() if local else self.path() + path, warnings = ( + (self.local_path(), []) if local else self._path_for_every_build(progress) + ) version = latest_version(self.artifact_path, session) text = self.render(version) if self._read_own_file(path) == text: - return [f"{path} already uses version {version} (no change)"] + return [*warnings, f"{path} already uses version {version} (no change)"] if dry_run: - return [f"would write {path} for version {version}"] + return [*warnings, f"would write {path} for version {version}"] path.parent.mkdir(parents=True, exist_ok=True) write_atomically(path, text.encode("utf-8")) - return [f"wrote {path} for version {version}"] + return [*warnings, f"wrote {path} for version {version}"] def uninstall( - self, *, dry_run: bool = False, local: bool = False, **_options + self, + *, + dry_run: bool = False, + local: bool = False, + progress: Callable[[str], None] | None = None, + **_options, ) -> list[str]: - path = self.local_path() if local else self.path() + path, warnings = ( + (self.local_path(), []) if local else self._path_for_every_build(progress) + ) if self._read_own_file(path) is None: - return [f"{path} not found (nothing to remove)"] + return [*warnings, f"{path} not found (nothing to remove)"] if dry_run: - return [f"would remove {path}"] + return [*warnings, f"would remove {path}"] path.unlink() - return [f"removed {path}"] + return [*warnings, f"removed {path}"] def status(self) -> dict: paths = [self.path()] diff --git a/cloudsmith_cli/credential_helpers/sbt/installer.py b/cloudsmith_cli/credential_helpers/sbt/installer.py index 99266f34..1c3451b2 100644 --- a/cloudsmith_cli/credential_helpers/sbt/installer.py +++ b/cloudsmith_cli/credential_helpers/sbt/installer.py @@ -3,8 +3,10 @@ from __future__ import annotations +import re from pathlib import Path +from .. import maven_tools from ..maven_tools import ( REPOSITORY_URL, ConfigFileInstaller, @@ -17,16 +19,50 @@ resolvers += "cloudsmith-maven-tools" at "{repository_url}" addSbtPlugin("io.cloudsmith" % "sbt-cloudsmith" % "{version}") """ +_PROBE_BUILD = """\ +val cloudsmithGlobalPlugins = taskKey[Unit]("Print the global plugins directory of sbt 1.") +cloudsmithGlobalPlugins := { + val s = state.value + if (sbtBinaryVersion.value.startsWith("1.")) + println("CLOUDSMITH_SBT_GLOBAL_PLUGINS=" + + BuildPaths.getGlobalPluginsDirectory(s, BuildPaths.getGlobalBase(s)).getAbsolutePath) +} +""" +_GLOBAL_PLUGINS_LINE = re.compile(r"^CLOUDSMITH_SBT_GLOBAL_PLUGINS=(.+)$", re.MULTILINE) +_SBT_1_LINE = re.compile(r"^\[info\] welcome to sbt 1\.", re.MULTILINE) +_LOADING_GLOBAL_PLUGINS_LINE = re.compile( + r"^\[info\] loading global plugins from (.+)$", re.MULTILINE +) class SbtInstaller(ConfigFileInstaller): name = "sbt" summary = "sbt plugin that gets credentials from the Cloudsmith CLI" artifact_path = "io/cloudsmith/sbt-cloudsmith_2.12_1.0" + tool = "sbt" + reported = "global plugins directory" + relative_message = ( + "The global base of sbt is a relative path, so each build looks for the " + "plugin in its own directory. Set an absolute path for sbt.global.base, or " + "run `cloudsmith credential-helper install sbt --local` in each project." + ) def path(self) -> Path: return Path.home() / ".sbt" / "1.0" / "plugins" / "cloudsmith.sbt" + def reported_path(self, executable: str, cwd: Path) -> Path | None: + """Return the plugin file in the global plugins directory that sbt 1 prints.""" + (cwd / "build.sbt").write_text(_PROBE_BUILD, encoding="utf-8") + output = maven_tools.tool_output( + [executable, "-batch", "-no-colors", "cloudsmithGlobalPlugins"], cwd + ) + match = _GLOBAL_PLUGINS_LINE.search(output) + if match is None and _SBT_1_LINE.search(output): + # sbt names the directory before it fails to load a plugin in it. + match = _LOADING_GLOBAL_PLUGINS_LINE.search(output) + # A relative path is relative to cwd, so the second probe gets another one. + return cwd / match.group(1) / "cloudsmith.sbt" if match else None + def local_path(self) -> Path: # sbt loads every .sbt file in project/, so the build's own files stay as they are. project_dir = current_dir()