diff --git a/README.md b/README.md index 083f18b..013190f 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,8 @@ therefore come from upstream's database, not from the proxy's own vulnerability data, and versions withheld by [cooldown](#version-cooldown) are not excluded from the report. +To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). + ### Cargo Create or edit `~/.cargo/config.toml`: @@ -344,6 +346,8 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` +To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). + ### Conan (C/C++) Add the proxy as a remote: diff --git a/config.example.yaml b/config.example.yaml index a34462a..d29b4ba 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -140,6 +140,11 @@ upstream: # gate on publish age, e.g. Yarn's npmMinimalAgeGate. Default: false. # npm_full_metadata: true + # npm packages served only from a private registry, by name pattern. + # Matching packages are never looked up on the npm registry above. + # npm_routes: + # "@example/*": "https://npm.example.com/registry" + # Cargo sparse index URL cargo: "https://index.crates.io" @@ -185,6 +190,11 @@ upstream: # Packagist repository URL composer_repository: "https://repo.packagist.org" + # Composer packages served only from a private repository, by name pattern. + # Matching packages are never looked up on composer_repository. + # composer_routes: + # "example/*": "https://composer.example.com/packages" + # Conan registry URL conan: "https://center.conan.io" diff --git a/docs/configuration.md b/docs/configuration.md index 331db95..675d45a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -229,6 +229,36 @@ upstream: `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. +### Private package routes + +Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream: + +```yaml +upstream: + composer_routes: + "example/*": "https://packages.example.com/composer" + npm_routes: + "@example/*": "https://packages.example.com/npm" + allow_private_hosts: + - "packages.example.com" + auth: + "https://packages.example.com": + type: bearer + token: "${PRIVATE_REGISTRY_TOKEN}" +``` + +- Patterns use Go's [`path.Match`](https://pkg.go.dev/path#Match) syntax and are matched case-insensitively. `*` does not cross `/`, so `example/*` matches `example/library` but not `example-fork/library`. The longest matching pattern wins. +- A Composer route URL is the repository base that serves `/p2/{vendor}/{package}.json`. A route also covers the package's dev versions in `/p2/{vendor}/{package}~dev.json`; patterns are matched against the package name without `~dev`. +- An npm route URL is the registry base that serves `/{package}`. Tarball URLs in its packuments must be on the same host and below the same path. +- A routed package is not looked up anywhere else. If the private registry does not have it, the client gets a 404. If the registry fails or rejects the credentials, the client gets a 502. A package with the same name on the public registry is never served in its place (dependency confusion). +- Metadata and artifacts of routed packages are cached under keys that include the route's URL. Entries cached from the public registry before a route was added are never served for a routed package. +- With cooldown enabled, the publish time of a routed npm version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own. +- Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. +- Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. +- Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- npm audit and signing-key requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. +- Routes can only be set in the configuration file, not with environment variables. + Helm HTTP repositories and additional OCI registries are configured as named maps: ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index 2df21d8..fc314bd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,6 +66,7 @@ import ( "net" "net/url" "os" + "path" "path/filepath" "sort" "strconv" @@ -512,6 +513,13 @@ type UpstreamConfig struct { // Default: false (the abbreviated format is preferred). NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"` + // NPMRoutes maps package name patterns to npm registry URLs. A package + // whose name matches a pattern is fetched only from that registry, never + // from upstream.npm, so a public package cannot stand in for it. Patterns + // use path.Match syntax, e.g. "@example/*"; the longest matching pattern + // wins. + NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -573,6 +581,14 @@ type UpstreamConfig struct { // Default: https://repo.packagist.org ComposerRepository string `json:"composer_repository" yaml:"composer_repository"` + // ComposerRoutes maps package name patterns to Composer repository URLs + // (the base that serves /p2/{vendor}/{package}.json). A package whose + // name matches a pattern is fetched only from that repository, never + // from upstream.composer_repository, so a public package cannot stand in + // for it. Patterns use path.Match syntax, e.g. "example/*"; the longest + // matching pattern wins. + ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` + // Conan is the upstream Conan registry URL. // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` @@ -692,6 +708,12 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil { return err } + if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil { + return err + } + if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil { + return err + } if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } @@ -784,6 +806,18 @@ func validateNamedUpstreams(field string, upstreams map[string]string) error { return nil } +func validatePackageRoutes(field string, routes map[string]string) error { + for pattern, upstreamURL := range routes { + if _, err := path.Match(pattern, ""); pattern == "" || err != nil { + return fmt.Errorf("invalid %s pattern %q", field, pattern) + } + if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil { + return err + } + } + return nil +} + func parseAuthURL(value string) (*url.URL, error) { parsed, err := url.Parse(value) if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c1cc2ff..5566afd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1302,6 +1302,61 @@ func TestValidateUpstreamAuthURLs(t *testing.T) { }) } +func TestValidatePackageRoutes(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr bool + }{ + { + name: "valid Composer and npm routes", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com/packages"} + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "https://npm.example.com/registry"} + }, + }, + { + name: "Composer route URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "composer.example.com"} + }, + wantErr: true, + }, + { + name: "npm route URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "npm.example.com"} + }, + wantErr: true, + }, + { + name: "Composer route pattern is malformed", + modify: func(cfg *Config) { + cfg.Upstream.ComposerRoutes = map[string]string{"acme/[": "https://composer.example.com"} + }, + wantErr: true, + }, + { + name: "npm route pattern is empty", + modify: func(cfg *Config) { + cfg.Upstream.NPMRoutes = map[string]string{"": "https://npm.example.com"} + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + tt.modify(cfg) + err := cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) + } + }) + } +} + func TestValidateNamedUpstreams(t *testing.T) { tests := []struct { name string diff --git a/internal/handler/composer.go b/internal/handler/composer.go index da7a957..8608ed5 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -19,6 +19,7 @@ const ( composerMinified = "composer/2.0" composerUnset = "__unset" composerDevReset = "~dev" + composerDevFile = "~dev" // suffix of the metadata file holding dev versions vendorPackageParts = 2 ) @@ -30,6 +31,7 @@ type ComposerHandler struct { upstreamURL string repoURL string proxyURL string + routes packageRoutes } // NewComposerHandler creates a new Composer protocol handler. @@ -51,6 +53,13 @@ func NewComposerHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, repoUR return h } +// WithPackageRoutes sends packages matching a pattern to a dedicated Composer +// repository instead of the default one. See packageRoutes. +func (h *ComposerHandler) WithPackageRoutes(routes map[string]string) *ComposerHandler { + h.routes = newPackageRoutes(routes) + return h +} + // Routes returns the HTTP handler for Composer requests. func (h *ComposerHandler) Routes() http.Handler { mux := http.NewServeMux() @@ -86,6 +95,18 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ _ = json.NewEncoder(w).Encode(index) } +// sourceFor returns the repository serving a package and the key its metadata +// is cached under. packageName may carry the ~dev suffix of the file holding +// the package's dev versions: routes match the package without it, so its dev +// versions come from the same repository, while the cache key keeps it so the +// two files stay apart. +func (h *ComposerHandler) sourceFor(packageName string) (repoURL, cacheKey string) { + if route, routed := h.routes.match(strings.TrimSuffix(packageName, composerDevFile)); routed { + return route.url, route.cacheKey(packageName) + } + return h.repoURL, packageName +} + // handlePackageMetadata proxies and rewrites package metadata. func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { // Parse path: /p2/{vendor}/{package}.json @@ -102,15 +123,16 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R h.proxy.Logger.Info("composer metadata request", "package", packageName) - upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) + repoURL, cacheKey := h.sourceFor(packageName) + upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) - if rewritten, ok := h.proxy.storedRewrite("composer", packageName, h.proxyURL, packageName); ok { + if rewritten, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { w.Header().Set(headerContentType, "application/json") _, _ = w.Write(rewritten) return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", packageName, upstreamURL) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", cacheKey, upstreamURL) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -121,7 +143,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, packageName, body, h.rewriteMetadata) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata) if err != nil { if r.Context().Err() != nil { return // the client left while waiting on a shared rewrite @@ -192,6 +214,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v packagePURL := canonicalPackagePURL("composer", packageName) upstream, written := newComposerFields(), newComposerFields() devReset, first := false, true + _, routed := h.routes.match(packageName) out.WriteByte('[') err := forEachJSONElement(versions, func(entry []byte) error { @@ -225,6 +248,9 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v out.WriteString(`"` + composerDevReset + `",`) devReset = false } + if routed { + disableDefaultNotification(upstream) + } h.writeVersion(out, packageName, version, upstream, written) return nil }) @@ -296,6 +322,17 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } +// disableDefaultNotification stops Composer from reporting installs of a routed +// package to the default repository's notify-batch URL, which would disclose +// the package name to it. Composer only falls back to notify-batch when a +// version has no notification-url of its own, and skips an empty one. Setting +// it on the expanded fields keeps it set for the versions that inherit them. +func disableDefaultNotification(fields *composerFields) { + if fields.get("notification-url") == nil { + fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) + } +} + // rewriteDist returns a version's dist object with its url pointing at this // proxy. A dist without a string url is returned unchanged. func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { @@ -475,7 +512,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) "package", packageName, "version", version, "download_url", downloadURL) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL) + cacheFilename := h.routes.cacheFilename(packageName, filename) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, cacheFilename, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -497,8 +535,9 @@ func isDevVersion(version string) bool { // file, tagged releases from the regular file; the other file is included as a // fallback so an unexpected classification still resolves. func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string { - stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg) - dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg) + repoURL, _ := h.sourceFor(vendor + "/" + pkg) + stable := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) + dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", repoURL, vendor, pkg) if isDevVersion(version) { return []string{dev, stable} diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go new file mode 100644 index 0000000..5a65440 --- /dev/null +++ b/internal/handler/composer_routes_test.go @@ -0,0 +1,302 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// composerUpstreamRecorder serves canned Composer metadata and records which +// paths were requested. +type composerUpstreamRecorder struct { + *httptest.Server + mu sync.Mutex + requests []string +} + +func newComposerUpstream(t *testing.T, documents map[string]string, status int) *composerUpstreamRecorder { + t.Helper() + u := &composerUpstreamRecorder{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.requests = append(u.requests, r.URL.Path) + u.mu.Unlock() + if status != http.StatusOK { + w.WriteHeader(status) + return + } + body, ok := documents[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *composerUpstreamRecorder) requested() []string { + u.mu.Lock() + defer u.mu.Unlock() + return append([]string(nil), u.requests...) +} + +const routedComposerMetadata = `{ + "packages": { + "example/library": [ + {"name": "example/library", "version": "1.0.0", "dist": {"url": "https://private.example.com/archives/library-1.0.0.zip", "type": "zip"}} + ] + } +}` + +const publicComposerMetadata = `{ + "packages": { + "example/library": [ + {"name": "example/library", "version": "9.9.9", "dist": {"url": "https://public.example.com/library-9.9.9.zip", "type": "zip"}} + ] + } +}` + +func newRoutedComposerHandler(t *testing.T, proxy *Proxy, public, private *composerUpstreamRecorder) *ComposerHandler { + t.Helper() + return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). + WithPackageRoutes(map[string]string{"example/*": private.URL + "/group/packages/composer"}) +} + +func serveComposer(h *ComposerHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func TestComposerRoutedMetadataComesFromRouteOnly(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package: %v", got) + } + + var metadata map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &metadata); err != nil { + t.Fatalf("parsing response: %v", err) + } + versions := metadata["packages"].(map[string]any)["example/library"].([]any) + version := versions[0].(map[string]any) + if version["version"] != "1.0.0" { + t.Errorf("version = %v, want the routed upstream's 1.0.0", version["version"]) + } + wantDist := "http://proxy.test/composer/files/example/library/1.0.0/library-1.0.0.zip" + if dist := version["dist"].(map[string]any)["url"]; dist != wantDist { + t.Errorf("dist url = %v, want %q", dist, wantDist) + } + if notification, ok := version["notification-url"]; !ok || notification != "" { + t.Errorf("notification-url = %v (present %t), want empty so Composer does not report routed installs to notify-batch", notification, ok) + } +} + +func TestComposerRoutedMetadataDoesNotFallBack(t *testing.T) { + tests := []struct { + name string + privateStatus int + wantStatus int + }{ + {"package missing on route", http.StatusOK, http.StatusNotFound}, + {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, + {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{}, tt.privateStatus) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + + if w.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package: %v", got) + } + }) + } +} + +func TestComposerRoutedMetadataIgnoresDefaultRepositoryCache(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) + if w := serveComposer(unrouted, "/p2/example/library.json"); w.Code != http.StatusOK { + t.Fatalf("priming the default repository cache: status %d", w.Code) + } + + w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/p2/example/library.json") + + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served metadata cached from the default repository: %s", w.Body.String()) + } + if len(private.requested()) == 0 { + t.Error("routed upstream was not queried") + } +} + +func TestComposerUnroutedMetadataUsesDefaultRepository(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/other/library.json": strings.ReplaceAll(publicComposerMetadata, "example/", "other/")}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/other/library.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := private.requested(); len(got) != 0 { + t.Errorf("route was queried for an unrouted package: %v", got) + } + if strings.Contains(w.Body.String(), "notification-url") { + t.Errorf("unrouted package metadata gained a notification-url: %s", w.Body.String()) + } +} + +func TestComposerRoutedDownloadResolvesFromRoute(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("archive")), + ContentType: "application/zip", + } + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/files/example/library/1.0.0/library-1.0.0.zip") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if want := "https://private.example.com/archives/library-1.0.0.zip"; artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed download: %v", got) + } +} + +func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { + public := newComposerUpstream(t, map[string]string{"/p2/example/library.json": strings.ReplaceAll(publicComposerMetadata, "9.9.9", "1.0.0")}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": routedComposerMetadata}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("public archive")), + ContentType: "application/zip", + } + + unrouted := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL) + if w := serveComposer(unrouted, "/files/example/library/1.0.0/library-1.0.0.zip"); w.Code != http.StatusOK { + t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) + } + + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("private archive")), + ContentType: "application/zip", + } + w := serveComposer(newRoutedComposerHandler(t, proxy, public, private), "/files/example/library/1.0.0/library-1.0.0.zip") + + if w.Body.String() != "private archive" { + t.Errorf("routed download served %q, want the routed upstream's archive", w.Body.String()) + } +} + +// TestComposerRoutedMinifiedMetadataDisablesNotification checks that every +// version of a routed package, as Composer expands minified metadata, has an +// empty notification-url: versions that inherit their fields, versions after a +// ~dev reset, and versions after one that set its own URL and then unset it. +func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { + minified := `{"minified":"composer/2.0","packages":{"example/library":[` + + `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + + `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + + `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + + `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + + `"~dev",` + + `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` + public := newComposerUpstream(t, map[string]string{}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library.json": minified}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := newRoutedComposerHandler(t, proxy, public, private) + + w := serveComposer(h, "/p2/example/library.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) + } + want := map[string]string{ + "1.0.0": "", + "1.1.0": "", + "1.2.0": "https://private.example.com/notify", // the route's own URL is kept + "1.3.0": "", + "dev-main": "", + } + versions := composerVersions(t, w.Body.Bytes(), "example/library") + if len(versions) != len(want) { + t.Fatalf("versions = %v, want %d", versionNames(versions), len(want)) + } + for _, v := range versions { + name, _ := v["version"].(string) + if got, ok := v["notification-url"]; !ok || got != want[name] { + t.Errorf("%s: notification-url = %v (present %t), want %q", name, got, ok, want[name]) + } + } +} + +// TestComposerExactRouteCoversDevMetadata is a regression test: Packagist +// serves a package's dev versions from a separate {name}~dev.json file, and an +// exact route for the package must cover that file too. Matching the route +// against the name with ~dev still attached sent the private package's dev +// metadata to the default repository. +func TestComposerExactRouteCoversDevMetadata(t *testing.T) { + const devMetadata = `{"packages":{"example/library":[{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/archives/main.zip","type":"zip"}}]}}` + public := newComposerUpstream(t, map[string]string{"/p2/example/library~dev.json": publicComposerMetadata}, http.StatusOK) + private := newComposerUpstream(t, map[string]string{"/group/packages/composer/p2/example/library~dev.json": devMetadata}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", public.URL, public.URL). + WithPackageRoutes(map[string]string{"example/library": private.URL + "/group/packages/composer"}) + + w := serveComposer(h, "/p2/example/library~dev.json") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body: %s", w.Code, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default repository was queried for a routed package's dev metadata: %v", got) + } + if got, want := private.requested(), []string{"/group/packages/composer/p2/example/library~dev.json"}; strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("route requests = %v, want %v", got, want) + } + if !strings.Contains(w.Body.String(), "/composer/files/example/library/dev-main/") { + t.Errorf("dev metadata was not rewritten through the proxy: %s", w.Body.String()) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 31c5341..5c8fbd7 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -38,6 +38,7 @@ type NPMHandler struct { proxy *Proxy upstreamURL string proxyURL string // URL where this proxy is hosted + routes packageRoutes } // NewNPMHandler creates a new npm protocol handler. @@ -53,6 +54,13 @@ func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { } } +// WithPackageRoutes sends packages matching a pattern to a dedicated npm +// registry instead of the default one. See packageRoutes. +func (h *NPMHandler) WithPackageRoutes(routes map[string]string) *NPMHandler { + h.routes = newPackageRoutes(routes) + return h +} + // Routes returns the HTTP handler for npm requests. // Mount this at /npm on your router. func (h *NPMHandler) Routes() http.Handler { @@ -156,6 +164,15 @@ func (h *NPMHandler) handleSecurity(w http.ResponseWriter, r *http.Request) { h.proxy.relayResponse(w, r, resp, nil) } +// sourceFor returns the registry serving a package and the key its metadata is +// cached under. +func (h *NPMHandler) sourceFor(packageName string) (registryURL, cacheKey string) { + if route, routed := h.routes.match(packageName); routed { + return route.url, route.cacheKey(packageName) + } + return h.upstreamURL, packageName +} + // handlePackageMetadata proxies package metadata from upstream and rewrites tarball URLs. func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { packageName := h.extractPackageName(r) @@ -166,7 +183,8 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques h.proxy.Logger.Info("npm metadata request", "package", packageName) - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) + registryURL, cacheKey := h.sourceFor(packageName) + upstreamURL := fmt.Sprintf("%s/%s", registryURL, url.PathEscape(packageName)) // Prefer the smaller abbreviated packument format but include application/json // as a fallback so upstreams that reject the abbreviated type (e.g. JFrog @@ -180,14 +198,14 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } - if rewritten, ok := h.proxy.storedRewrite("npm", packageName, h.proxyURL, packageName); ok { + if rewritten, ok := h.proxy.storedRewrite("npm", cacheKey, h.proxyURL, cacheKey); ok { w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) _, _ = w.Write(rewritten) return } - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, accept) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, "package not found") @@ -198,7 +216,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, packageName, body, func(b []byte) ([]byte, error) { + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, cacheKey, body, func(b []byte) ([]byte, error) { return h.rewriteMetadata(packageName, b) }) if err != nil { @@ -585,9 +603,10 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) return } + registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := h.upstreamURL + "/" + url.PathEscape(packageName) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + upstreamURL := registryURL + "/" + url.PathEscape(packageName) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, contentTypeJSON) return body, err }) if h.versionInCooldown(packageName, version, metadata) { @@ -597,7 +616,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - result, err := h.getTarball(r, packageName, version, filename, metadata) + result, err := h.getTarball(r, registryURL, packageName, version, filename, metadata) if err != nil { switch { case errors.Is(err, ErrUpstreamNotFound): @@ -614,21 +633,22 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } -func (h *NPMHandler) getTarball(r *http.Request, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { - if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, filename); err != nil || cached != nil { +func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { + cacheFilename := h.routes.cacheFilename(packageName, filename) + if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err } - downloadURL := fmt.Sprintf("%s/%s/-/%s", h.upstreamURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) + downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) body, err := metadata() if err == nil { if tarball := npmVersionTarball(body, version); tarball != "" { - downloadURL, err = h.validateTarballURL(tarball) + downloadURL, err = validateTarballURL(tarball, registryURL) if err != nil { return nil, err } } } - return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, filename, downloadURL) + return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, cacheFilename, downloadURL) } // npmVersionTarball returns versions[version].dist.tarball from a packument, @@ -639,12 +659,12 @@ func npmVersionTarball(body []byte, version string) string { return tarball } -func (h *NPMHandler) validateTarballURL(raw string) (string, error) { +func validateTarballURL(raw, registryURL string) (string, error) { tarball, err := url.Parse(raw) if err != nil { return "", fmt.Errorf("parsing npm tarball URL: %w", err) } - upstream, err := url.Parse(h.upstreamURL) + upstream, err := url.Parse(registryURL) if err != nil { return "", fmt.Errorf("parsing npm upstream URL: %w", err) } @@ -673,14 +693,25 @@ func (h *NPMHandler) validateTarballURL(raw string) (string, error) { // fetched and parsed at most once per version. A version with no usable // publish time is allowed through, matching how applyCooldownFiltering // treats it. +// +// The versions row is keyed by the plain version PURL, which does not say +// which registry the version came from, so routed packages leave it alone: +// their publish time always comes from the route's own metadata, cached under +// the route. Otherwise a time read from the default registry would be trusted +// for a private version of the same name, a private time would hold back the +// public version, and moving a route to another upstream would keep the old +// one's times. func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } + _, routed := h.routes.match(packageName) versionPURL := canonicalVersionPURL("npm", packageName, version) - if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) + if !routed { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) + } } body, err := metadata() @@ -705,9 +736,11 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL("npm", packageName), publishedAt); err != nil { - h.proxy.Logger.Warn("cooldown: could not store npm publish time", - "package", packageName, "version", version, "error", err) + if !routed { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL("npm", packageName), publishedAt); err != nil { + h.proxy.Logger.Warn("cooldown: could not store npm publish time", + "package", packageName, "version", version, "error", err) + } } return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt) diff --git a/internal/handler/npm_routes_test.go b/internal/handler/npm_routes_test.go new file mode 100644 index 0000000..a2ffb38 --- /dev/null +++ b/internal/handler/npm_routes_test.go @@ -0,0 +1,296 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +// npmUpstreamRecorder serves canned packuments and records which paths were +// requested. +type npmUpstreamRecorder struct { + *httptest.Server + mu sync.Mutex + requests []string +} + +func newNPMUpstream(t *testing.T, packuments map[string]func(baseURL string) string, status int) *npmUpstreamRecorder { + t.Helper() + u := &npmUpstreamRecorder{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + u.requests = append(u.requests, r.URL.Path) + u.mu.Unlock() + if status != http.StatusOK { + w.WriteHeader(status) + return + } + packument, ok := packuments[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, packument(u.URL)) + })) + t.Cleanup(u.Close) + return u +} + +func (u *npmUpstreamRecorder) requested() []string { + u.mu.Lock() + defer u.mu.Unlock() + return append([]string(nil), u.requests...) +} + +func routedTestPackument(version, tarballBase string) func(string) string { + return func(baseURL string) string { + return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + + `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}}}` + } +} + +const routedNPMPath = "/registry/@example/widgets" + +func newRoutedNPMHandler(proxy *Proxy, public, private *npmUpstreamRecorder) *NPMHandler { + return NewNPMHandler(proxy, "http://proxy.test", public.URL). + WithPackageRoutes(map[string]string{"@example/*": private.URL + "/registry"}) +} + +func TestNPMRoutedMetadataComesFromRouteOnly(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served the default registry's metadata: %s", w.Body.String()) + } + wantTarball := "http://proxy.test/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" + if !strings.Contains(w.Body.String(), wantTarball) { + t.Errorf("tarball not rewritten to %q: %s", wantTarball, w.Body.String()) + } +} + +func TestNPMRoutedMetadataDoesNotFallBack(t *testing.T) { + tests := []struct { + name string + privateStatus int + wantStatus int + }{ + {"package missing on route", http.StatusOK, http.StatusNotFound}, + {"route rejects credentials", http.StatusUnauthorized, http.StatusBadGateway}, + {"route unavailable", http.StatusInternalServerError, http.StatusBadGateway}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, tt.privateStatus) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example%2fwidgets", nil, nil) + + if w.Code != tt.wantStatus { + t.Errorf("status = %d, want %d", w.Code, tt.wantStatus) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed package: %v", got) + } + }) + } +} + +func TestNPMRoutedMetadataIgnoresDefaultRegistryCache(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("9.9.9", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + + if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example%2fwidgets", nil, nil); w.Code != http.StatusOK { + t.Fatalf("priming the default registry cache: status %d", w.Code) + } + + w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example%2fwidgets", nil, nil) + + if strings.Contains(w.Body.String(), "9.9.9") { + t.Errorf("routed package served metadata cached from the default registry: %s", w.Body.String()) + } +} + +func TestNPMUnroutedMetadataUsesDefaultRegistry(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/lodash": routedTestPackument("4.17.21", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/lodash", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := private.requested(); len(got) != 0 { + t.Errorf("route was queried for an unrouted package: %v", got) + } +} + +func TestNPMRoutedDownloadResolvesFromRoute(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := newRoutedNPMHandler(proxy, public, private) + + w := serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if want := private.URL + "/registry/@example/widgets/-/widgets-1.0.0.tgz"; artifactFetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", artifactFetcher.fetchedURL, want) + } + if got := public.requested(); len(got) != 0 { + t.Errorf("default registry was queried for a routed download: %v", got) + } +} + +func TestNPMRoutedDownloadWithoutMetadataStaysOnRoute(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{}, http.StatusInternalServerError) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := newRoutedNPMHandler(proxy, public, private) + + serveNPM(t, h, http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if !strings.HasPrefix(artifactFetcher.fetchedURL, private.URL+"/registry/") { + t.Errorf("fetched URL = %q, want one on the route %q", artifactFetcher.fetchedURL, private.URL+"/registry/") + } +} + +func TestNPMRoutedDownloadIgnoresDefaultRegistryArtifact(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": routedTestPackument("1.0.0", "")}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: routedTestPackument("1.0.0", "/registry")}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("public package")), + ContentType: "application/gzip", + } + + if w := serveNPM(t, NewNPMHandler(proxy, "http://proxy.test", public.URL), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil); w.Code != http.StatusOK { + t.Fatalf("priming the artifact cache: status %d; body: %s", w.Code, w.Body.String()) + } + + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("private package")), + ContentType: "application/gzip", + } + w := serveNPM(t, newRoutedNPMHandler(proxy, public, private), http.MethodGet, "/@example/widgets/-/widgets-1.0.0.tgz", nil, nil) + + if w.Body.String() != "private package" { + t.Errorf("routed download served %q, want the routed registry's package", w.Body.String()) + } +} + +// timedTestPackument is routedTestPackument with a publish time for the +// version, published age ago. +func timedTestPackument(version, tarballBase string, age time.Duration) func(string) string { + published := time.Now().Add(-age).UTC().Format(time.RFC3339) + return func(baseURL string) string { + return `{"name":"@example/widgets","dist-tags":{"latest":"` + version + `"},"versions":{"` + version + + `":{"version":"` + version + `","dist":{"tarball":"` + baseURL + tarballBase + `/@example/widgets/-/widgets-` + version + `.tgz"}}},` + + `"time":{"` + version + `":"` + published + `"}}` + } +} + +// TestNPMRoutedCooldownUsesRoutePublishTimes is a regression test: the +// download cooldown check stored publish times under the package's plain +// version PURL, so a time read from one registry was trusted for the same +// version from another. A route's version published an hour ago must stay in +// a seven-day cooldown even when the default registry's version of the same +// name is a month old, whichever was downloaded first, and changing a route's +// upstream must not carry over the old one's times. +func TestNPMRoutedCooldownUsesRoutePublishTimes(t *testing.T) { + const tarball = "/@example/widgets/-/widgets-1.0.0.tgz" + month, hour := 30*24*time.Hour, time.Hour + + type step struct { + handler string // "public", "route" or "other route" + wantStatus int + } + tests := []struct { + name string + steps []step + }{ + {"route added after a public download", []step{ + {"public", http.StatusOK}, + {"route", http.StatusNotFound}, + }}, + {"public download after a routed one", []step{ + {"route", http.StatusNotFound}, + {"public", http.StatusOK}, + }}, + {"route moved to another upstream", []step{ + {"other route", http.StatusOK}, + {"route", http.StatusNotFound}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + public := newNPMUpstream(t, map[string]func(string) string{"/@example/widgets": timedTestPackument("1.0.0", "", month)}, http.StatusOK) + private := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", hour)}, http.StatusOK) + old := newNPMUpstream(t, map[string]func(string) string{routedNPMPath: timedTestPackument("1.0.0", "/registry", month)}, http.StatusOK) + proxy, _, _, artifactFetcher := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handlers := map[string]*NPMHandler{ + "public": NewNPMHandler(proxy, "http://proxy.test", public.URL), + "route": newRoutedNPMHandler(proxy, public, private), + "other route": newRoutedNPMHandler(proxy, public, old), + } + + for _, s := range tt.steps { + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + w := serveNPM(t, handlers[s.handler], http.MethodGet, tarball, nil, nil) + if w.Code != s.wantStatus { + t.Errorf("%s download: status %d, want %d; body: %s", s.handler, w.Code, s.wantStatus, w.Body.String()) + } + } + }) + } +} diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go new file mode 100644 index 0000000..8af7099 --- /dev/null +++ b/internal/handler/package_routes.go @@ -0,0 +1,74 @@ +package handler + +import ( + "crypto/sha256" + "encoding/hex" + "path" + "sort" + "strings" +) + +// routeIDLength is the number of hex characters of the upstream URL digest +// that qualify cache entries of a routed package. +const routeIDLength = 16 + +// packageRoutes sends packages whose names match a pattern to a dedicated +// upstream instead of the ecosystem's default registry. A matched package is +// never looked up anywhere else, so a public package with the same name cannot +// replace it. +type packageRoutes struct { + routes []packageRoute // longest pattern first +} + +type packageRoute struct { + pattern string + url string + id string +} + +func newPackageRoutes(patterns map[string]string) packageRoutes { + routes := make([]packageRoute, 0, len(patterns)) + for pattern, upstreamURL := range patterns { + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + digest := sha256.Sum256([]byte(upstreamURL)) + routes = append(routes, packageRoute{ + pattern: strings.ToLower(pattern), + url: upstreamURL, + id: hex.EncodeToString(digest[:])[:routeIDLength], + }) + } + sort.Slice(routes, func(i, j int) bool { + if len(routes[i].pattern) != len(routes[j].pattern) { + return len(routes[i].pattern) > len(routes[j].pattern) + } + return routes[i].pattern < routes[j].pattern + }) + return packageRoutes{routes: routes} +} + +// match returns the route for a package name, if any pattern matches it. +func (r packageRoutes) match(packageName string) (packageRoute, bool) { + name := strings.ToLower(packageName) + for _, route := range r.routes { + if ok, _ := path.Match(route.pattern, name); ok { + return route, true + } + } + return packageRoute{}, false +} + +// cacheKey qualifies a package's metadata cache key with the route's upstream, +// so entries cached from the default registry, or from a route's previous +// upstream, are never served for a routed package. +func (r packageRoute) cacheKey(packageName string) string { + return "routes/" + r.id + "/" + packageName +} + +// cacheFilename qualifies a routed package's artifact filename the same way as +// cacheKey, and leaves other packages' filenames unchanged. +func (r packageRoutes) cacheFilename(packageName, filename string) string { + if route, routed := r.match(packageName); routed { + return route.id + "/" + filename + } + return filename +} diff --git a/internal/handler/package_routes_test.go b/internal/handler/package_routes_test.go new file mode 100644 index 0000000..dcb6f5f --- /dev/null +++ b/internal/handler/package_routes_test.go @@ -0,0 +1,60 @@ +package handler + +import "testing" + +func TestPackageRoutesMatch(t *testing.T) { + routes := newPackageRoutes(map[string]string{ + "@example/*": "https://npm.example.com/scoped/", + "@example/tool-*": "https://npm.example.com/tools", + "example/*": "https://composer.example.com/packages", + "single-package": "https://registry.example.com/single", + }) + + tests := []struct { + name string + pkg string + wantURL string + wantOK bool + }{ + {"scoped npm package", "@example/widgets", "https://npm.example.com/scoped", true}, + {"longest pattern wins", "@example/tool-kit", "https://npm.example.com/tools", true}, + {"composer vendor", "example/library", "https://composer.example.com/packages", true}, + {"exact name", "single-package", "https://registry.example.com/single", true}, + {"names are case-insensitive", "Example/Library", "https://composer.example.com/packages", true}, + {"wildcard does not cross slash", "example/library/extra", "", false}, + {"similar vendor does not match", "example-fork/library", "", false}, + {"unrouted package", "lodash", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + route, ok := routes.match(tt.pkg) + if ok != tt.wantOK { + t.Fatalf("match(%q) ok = %t, want %t", tt.pkg, ok, tt.wantOK) + } + if ok && route.url != tt.wantURL { + t.Errorf("match(%q) url = %q, want %q", tt.pkg, route.url, tt.wantURL) + } + }) + } +} + +func TestPackageRoutesCacheKeysAreDistinct(t *testing.T) { + routes := newPackageRoutes(map[string]string{ + "example/*": "https://one.example.com", + "other-example/*": "https://two.example.com", + }) + + first, _ := routes.match("example/library") + second, _ := routes.match("other-example/library") + + if first.cacheKey("example/library") == "example/library" { + t.Error("routed cache key must differ from the unrouted package name") + } + if first.cacheKey("example/library") == second.cacheKey("example/library") { + t.Error("routes with different upstreams must not share cache keys") + } + if containsPathTraversal(first.cacheKey("example/library")) { + t.Error("routed cache key must be usable as a storage path") + } +} diff --git a/internal/server/package_routes_test.go b/internal/server/package_routes_test.go new file mode 100644 index 0000000..57bf946 --- /dev/null +++ b/internal/server/package_routes_test.go @@ -0,0 +1,53 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +func TestMountProtocolHandlersAppliesPackageRoutes(t *testing.T) { + routed := make(chan string, 2) + private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + routed <- r.URL.Path + http.NotFound(w, r) + })) + defer private.Close() + + cfg := config.Default() + cfg.BaseURL = "http://proxy.test" + cfg.Upstream.Composer = "http://default.invalid" + cfg.Upstream.ComposerRepository = "http://default.invalid" + cfg.Upstream.NPM = "http://default.invalid" + cfg.Upstream.ComposerRoutes = map[string]string{"example/*": private.URL + "/composer"} + cfg.Upstream.NPMRoutes = map[string]string{"@example/*": private.URL + "/npm"} + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(nil, nil, fetch.NewFetcher(), fetch.NewResolver(), logger) + proxy.HTTPClient = private.Client() + s := &Server{cfg: cfg, logger: logger} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + for path, wantUpstreamPath := range map[string]string{ + "/composer/p2/example/library.json": "/composer/p2/example/library.json", + "/npm/@example%2fwidgets": "/npm/@example/widgets", + } { + r.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, path, nil)) + select { + case got := <-routed: + if got != wantUpstreamPath { + t.Errorf("%s reached route upstream at %q, want %q", path, got, wantUpstreamPath) + } + default: + t.Errorf("%s did not reach the configured route", path) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 66ed14b..380892a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -385,7 +385,8 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM) + npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM). + WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( proxy, s.cfg.BaseURL, @@ -425,7 +426,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.BaseURL, s.cfg.Upstream.Composer, s.cfg.Upstream.ComposerRepository, - ) + ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN)