From e62bee53df1912615802cdf2d4444d24c836ed6b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 18 Oct 2026 18:48:37 +0100 Subject: [PATCH 1/2] Let fetch.Resolver take per-ecosystem base URLs Callers that never register a registry can now point the hard-coded fallback patterns at a mirror with WithBaseURLs. Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- README.md | 9 +++++++ fetch/resolver.go | 59 +++++++++++++++++++++++++++++++++++------- fetch/resolver_test.go | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 047185d..3fba18c 100644 --- a/README.md +++ b/README.md @@ -486,6 +486,15 @@ info, _ := resolver.Resolve(ctx, "npm", "lodash", "4.17.21") // info.URL = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" ``` +`WithBaseURLs` points those patterns at a mirror instead of the public host. Ecosystems left out of the map keep their defaults: + +```go +resolver := fetch.NewResolver(fetch.WithBaseURLs(map[string]string{ + "npm": "https://npm.mycompany.com", + "cargo": "https://crates.mycompany.com/crates", +})) +``` + ## Private Registries PURLs with a `repository_url` qualifier automatically use that URL: diff --git a/fetch/resolver.go b/fetch/resolver.go index 985fca5..0988ab8 100644 --- a/fetch/resolver.go +++ b/fetch/resolver.go @@ -28,16 +28,54 @@ type Registry interface { URLs() client.URLBuilder } +// defaultBaseURLs are the public registry hosts used when no registry is +// registered for an ecosystem and WithBaseURLs does not override them. +var defaultBaseURLs = map[string]string{ + "npm": "https://registry.npmjs.org", + "cargo": "https://static.crates.io/crates", + "gem": "https://rubygems.org", + "golang": "https://proxy.golang.org", + "hex": "https://repo.hex.pm", + "pub": "https://pub.dev", + "maven": "https://repo1.maven.org/maven2", + "nuget": "https://api.nuget.org", +} + // Resolver determines download URLs for package artifacts. type Resolver struct { registries map[string]Registry + baseURLs map[string]string +} + +// ResolverOption configures a Resolver. +type ResolverOption func(*Resolver) + +// WithBaseURLs overrides the base URL used for ecosystems resolved without a +// registered registry, keyed by ecosystem. Ecosystems not in the map keep +// the public registry host. +func WithBaseURLs(baseURLs map[string]string) ResolverOption { + return func(r *Resolver) { + for ecosystem, baseURL := range baseURLs { + if baseURL != "" { + r.baseURLs[ecosystem] = strings.TrimRight(baseURL, "/") + } + } + } } // NewResolver creates a new URL resolver. -func NewResolver() *Resolver { - return &Resolver{ +func NewResolver(opts ...ResolverOption) *Resolver { + r := &Resolver{ registries: make(map[string]Registry), + baseURLs: make(map[string]string, len(defaultBaseURLs)), + } + for ecosystem, baseURL := range defaultBaseURLs { + r.baseURLs[ecosystem] = baseURL + } + for _, opt := range opts { + opt(r) } + return r } // RegisterRegistry adds a registry for URL resolution. @@ -96,6 +134,7 @@ func (r *Resolver) ResolveWithOptions( // when no registry client is configured. func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*ArtifactInfo, error) { var url, filename string + base := r.baseURLs[ecosystem] switch ecosystem { case "npm": @@ -103,28 +142,28 @@ func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*Art if idx := strings.LastIndex(name, "/"); idx >= 0 { shortName = name[idx+1:] } - url = fmt.Sprintf("https://registry.npmjs.org/%s/-/%s-%s.tgz", name, shortName, version) + url = fmt.Sprintf("%s/%s/-/%s-%s.tgz", base, name, shortName, version) filename = fmt.Sprintf("%s-%s.tgz", shortName, version) case "cargo": - url = fmt.Sprintf("https://static.crates.io/crates/%s/%s-%s.crate", name, name, version) + url = fmt.Sprintf("%s/%s/%s-%s.crate", base, name, name, version) filename = fmt.Sprintf("%s-%s.crate", name, version) case "gem": - url = fmt.Sprintf("https://rubygems.org/downloads/%s-%s.gem", name, version) + url = fmt.Sprintf("%s/downloads/%s-%s.gem", base, name, version) filename = fmt.Sprintf("%s-%s.gem", name, version) case "golang": encoded := encodeGoModule(name) - url = fmt.Sprintf("https://proxy.golang.org/%s/@v/%s.zip", encoded, version) + url = fmt.Sprintf("%s/%s/@v/%s.zip", base, encoded, version) filename = fmt.Sprintf("%s@%s.zip", lastPathComponent(name), version) case "hex": - url = fmt.Sprintf("https://repo.hex.pm/tarballs/%s-%s.tar", name, version) + url = fmt.Sprintf("%s/tarballs/%s-%s.tar", base, name, version) filename = fmt.Sprintf("%s-%s.tar", name, version) case "pub": - url = fmt.Sprintf("https://pub.dev/packages/%s/versions/%s.tar.gz", name, version) + url = fmt.Sprintf("%s/packages/%s/versions/%s.tar.gz", base, name, version) filename = fmt.Sprintf("%s-%s.tar.gz", name, version) case "maven": @@ -134,13 +173,13 @@ func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*Art return nil, fmt.Errorf("invalid maven name format, expected group:artifact") } group = strings.ReplaceAll(group, ".", "/") - url = fmt.Sprintf("https://repo1.maven.org/maven2/%s/%s/%s/%s-%s.jar", group, artifact, version, artifact, version) + url = fmt.Sprintf("%s/%s/%s/%s/%s-%s.jar", base, group, artifact, version, artifact, version) filename = fmt.Sprintf("%s-%s.jar", artifact, version) case "nuget": // NuGet package IDs are case-insensitive, use lowercase lowername := strings.ToLower(name) - url = fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/%s/%s.%s.nupkg", lowername, version, lowername, version) + url = fmt.Sprintf("%s/v3-flatcontainer/%s/%s/%s.%s.nupkg", base, lowername, version, lowername, version) filename = fmt.Sprintf("%s.%s.nupkg", lowername, version) default: diff --git a/fetch/resolver_test.go b/fetch/resolver_test.go index 7cf03c3..2734624 100644 --- a/fetch/resolver_test.go +++ b/fetch/resolver_test.go @@ -94,6 +94,59 @@ func TestResolveWithoutRegistry(t *testing.T) { } } +func TestResolveWithBaseURLs(t *testing.T) { + r := NewResolver(WithBaseURLs(map[string]string{ + "npm": "https://npm.internal/", + "cargo": "https://crates.internal/crates", + "gem": "https://gems.internal", + "golang": "https://goproxy.internal", + "hex": "https://hex.internal", + "pub": "https://pub.internal", + "maven": "https://maven.internal/releases", + "nuget": "https://nuget.internal", + })) + + tests := []struct { + ecosystem string + name string + version string + wantURL string + }{ + {"npm", "@babel/core", "7.23.0", "https://npm.internal/@babel/core/-/core-7.23.0.tgz"}, + {"cargo", "serde", "1.0.193", "https://crates.internal/crates/serde/serde-1.0.193.crate"}, + {"gem", "rails", "7.1.2", "https://gems.internal/downloads/rails-7.1.2.gem"}, + {"golang", "github.com/Azure/azure-sdk-for-go", "v68.0.0", "https://goproxy.internal/github.com/!azure/azure-sdk-for-go/@v/v68.0.0.zip"}, + {"hex", "phoenix", "1.7.10", "https://hex.internal/tarballs/phoenix-1.7.10.tar"}, + {"pub", "flutter", "3.16.0", "https://pub.internal/packages/flutter/versions/3.16.0.tar.gz"}, + {"maven", "com.google.guava:guava", "32.1.3-jre", "https://maven.internal/releases/com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar"}, + {"nuget", "Newtonsoft.Json", "13.0.3", "https://nuget.internal/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg"}, + } + + for _, tt := range tests { + t.Run(tt.ecosystem, func(t *testing.T) { + info, err := r.Resolve(context.Background(), tt.ecosystem, tt.name, tt.version) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if info.URL != tt.wantURL { + t.Errorf("URL = %q, want %q", info.URL, tt.wantURL) + } + }) + } +} + +func TestResolveWithBaseURLsKeepsDefaultsForOtherEcosystems(t *testing.T) { + r := NewResolver(WithBaseURLs(map[string]string{"npm": "https://npm.internal", "cargo": ""})) + + info, err := r.Resolve(context.Background(), "cargo", "serde", "1.0.193") + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if want := "https://static.crates.io/crates/serde/serde-1.0.193.crate"; info.URL != want { + t.Errorf("URL = %q, want %q", info.URL, want) + } +} + func TestResolveUnsupportedEcosystem(t *testing.T) { r := NewResolver() From da877f0fda107719913c23270b8608a90c79f34d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 18 Oct 2026 18:52:00 +0100 Subject: [PATCH 2/2] Treat ecosystem aliases as one ecosystem in fetch.Resolver Names are normalised to PURL types, so rubygems/gem and packagist/composer resolve the same way. Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- fetch/resolver.go | 9 ++++--- fetch/resolver_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/fetch/resolver.go b/fetch/resolver.go index 0988ab8..2f5633e 100644 --- a/fetch/resolver.go +++ b/fetch/resolver.go @@ -9,6 +9,7 @@ import ( "strings" "unicode" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" "github.com/git-pkgs/registries/client" ) @@ -57,7 +58,7 @@ func WithBaseURLs(baseURLs map[string]string) ResolverOption { return func(r *Resolver) { for ecosystem, baseURL := range baseURLs { if baseURL != "" { - r.baseURLs[ecosystem] = strings.TrimRight(baseURL, "/") + r.baseURLs[purl.EcosystemToPURLType(ecosystem)] = strings.TrimRight(baseURL, "/") } } } @@ -78,9 +79,10 @@ func NewResolver(opts ...ResolverOption) *Resolver { return r } -// RegisterRegistry adds a registry for URL resolution. +// RegisterRegistry adds a registry for URL resolution. Ecosystem names are +// matched by PURL type, so "rubygems" and "gem" are the same ecosystem. func (r *Resolver) RegisterRegistry(reg Registry) { - r.registries[reg.Ecosystem()] = reg + r.registries[purl.EcosystemToPURLType(reg.Ecosystem())] = reg } // ArtifactInfo contains information about a downloadable artifact. @@ -111,6 +113,7 @@ func (r *Resolver) ResolveWithOptions( version string, options ResolveOptions, ) (*ArtifactInfo, error) { + ecosystem = purl.EcosystemToPURLType(ecosystem) reg, ok := r.registries[ecosystem] if !ok { info, err := r.resolveWithoutRegistry(ecosystem, name, version) diff --git a/fetch/resolver_test.go b/fetch/resolver_test.go index 2734624..443d923 100644 --- a/fetch/resolver_test.go +++ b/fetch/resolver_test.go @@ -354,3 +354,64 @@ func TestFilenameFromURL(t *testing.T) { } } } + +type namedRegistry struct { + fakeRegistry + ecosystem string +} + +func (n *namedRegistry) Ecosystem() string { return n.ecosystem } + +func TestResolveEcosystemAliases(t *testing.T) { + for _, ecosystem := range []string{"gem", "rubygems", "RubyGems"} { + t.Run(ecosystem, func(t *testing.T) { + info, err := NewResolver().Resolve(context.Background(), ecosystem, "rake", "13.2.1") + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if want := "https://rubygems.org/downloads/rake-13.2.1.gem"; info.URL != want { + t.Errorf("URL = %q, want %q", info.URL, want) + } + }) + } +} + +func TestResolveRegisteredEcosystemAliases(t *testing.T) { + want := "https://repo.packagist.org/dist/monolog/monolog-3.5.0.zip" + for _, tt := range []struct{ registered, requested string }{ + {"composer", "packagist"}, + {"packagist", "composer"}, + {"composer", "composer"}, + } { + t.Run(tt.registered+"/"+tt.requested, func(t *testing.T) { + r := NewResolver() + r.RegisterRegistry(&namedRegistry{ + ecosystem: tt.registered, + fakeRegistry: fakeRegistry{versions: []registries.Version{{ + Number: "3.5.0", + Metadata: map[string]any{"download_url": want}, + }}}, + }) + + info, err := r.Resolve(context.Background(), tt.requested, "monolog/monolog", "3.5.0") + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if info.URL != want { + t.Errorf("URL = %q, want %q", info.URL, want) + } + }) + } +} + +func TestWithBaseURLsAcceptsEcosystemAliases(t *testing.T) { + r := NewResolver(WithBaseURLs(map[string]string{"rubygems": "https://gems.internal"})) + + info, err := r.Resolve(context.Background(), "gem", "rake", "13.2.1") + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if want := "https://gems.internal/downloads/rake-13.2.1.gem"; info.URL != want { + t.Errorf("URL = %q, want %q", info.URL, want) + } +}