diff --git a/README.md b/README.md index 56dfd315..5840d55d 100644 --- a/README.md +++ b/README.md @@ -609,6 +609,24 @@ Commit a `mise.lock` and install with `mise install --locked` so pinned installs need no API call at all. Add a bearer token for `https://api.github.com` under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. +Tools mise downloads from elsewhere, such as Maven from `archive.apache.org`, +can be cached the same way by declaring their version-pinned paths in +`upstream.generic_artifacts` (see [configuration](docs/configuration.md)): + +```yaml +upstream: + generic: + apache: "https://archive.apache.org" + generic_artifacts: + apache: + - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' +``` + +```toml +[settings.url_replacements] +"https://archive.apache.org/" = "http://localhost:8080/generic/apache/" +``` + ## Configuration The proxy can be configured via: diff --git a/config.example.yaml b/config.example.yaml index 49b81efd..85c9a8ae 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -243,6 +243,15 @@ upstream: # generic: # github: "https://github.com" # github-api: "https://api.github.com" + # apache: "https://archive.apache.org" + + # Version-pinned paths of generic upstreams, cached immutably like GitHub + # release assets. Each pattern must match the whole path after + # /generic/{name}/ and have `version` and `file` named groups; an optional + # `name` group names the package. + # generic_artifacts: + # apache: + # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path diff --git a/docs/configuration.md b/docs/configuration.md index 8261ebae..0b936940 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -276,7 +276,28 @@ Only configured upstreams are reachable, so this is not an open HTTP proxy. Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is -down. Every other path is served through the metadata cache (`cache_metadata` +down. + +Other upstreams can declare their own version-pinned paths in +`generic_artifacts`. Each pattern is a regular expression that must match the +whole path after `/generic/{name}/`, with `version` and `file` named groups +(single path segments) and an optional `name` group for the package name, +which otherwise is the upstream name. Matching downloads are cached like +release assets and listed under the `generic` ecosystem: + +```yaml +upstream: + generic: + apache: "https://archive.apache.org" + generic_artifacts: + apache: + - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' +``` + +Only list paths whose content never changes for a given version: a match is +never revalidated. + +Every other path is served through the metadata cache (`cache_metadata` must be enabled for offline fallback): fresh within `metadata_ttl`, then revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with a `Warning: 110` header when the upstream fails, refuses or rate-limits the diff --git a/internal/config/config.go b/internal/config/config.go index 54f826f5..3cae9ec2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,6 +66,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -637,11 +638,21 @@ type UpstreamConfig struct { // Generic maps names to plain HTTP upstream base URLs, served at // /generic/{name}/. The remaining request path and query string are // appended to the upstream URL. GitHub release asset paths - // ({owner}/{repo}/releases/download/{tag}/{asset}) are cached in the - // artifact cache; everything else goes through the metadata cache. + // ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching + // GenericArtifacts are cached in the artifact cache; everything else + // goes through the metadata cache. // Example: {"github": "https://github.com", "github-api": "https://api.github.com"}. Generic map[string]string `json:"generic" yaml:"generic"` + // GenericArtifacts maps a Generic upstream name to regular expressions for + // its version-pinned download paths. A request path after + // /generic/{name}/ that one of them matches in full is cached in the + // artifact cache like a GitHub release asset. Each pattern needs `version` + // and `file` named groups; an optional `name` group names the package, + // which otherwise is the upstream name. + // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. + GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -694,6 +705,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } + if err := u.validateGenericArtifacts(); err != nil { + return err + } if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { return err } @@ -708,6 +722,36 @@ func (u *UpstreamConfig) Validate() error { return nil } +func (u *UpstreamConfig) validateGenericArtifacts() error { + for name, patterns := range u.GenericArtifacts { + if _, found := u.Generic[name]; !found { + return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name) + } + for _, pattern := range patterns { + if _, err := CompileGenericArtifactPattern(pattern); err != nil { + return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err) + } + } + } + return nil +} + +// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern +// so that it only matches a whole path, and checks it has the `version` and +// `file` groups. +func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) { + re, err := regexp.Compile(`^(?:` + pattern + `)$`) + if err != nil { + return nil, err + } + for _, group := range []string{"version", "file"} { + if re.SubexpIndex(group) < 0 { + return nil, fmt.Errorf("missing (?P<%s>...) group", group) + } + } + return re, nil +} + // debianReservedRepositoryNames are the upstream.debian archive's own root // paths, which a repository of the same name would shadow. var debianReservedRepositoryNames = []string{"pool", "dists"} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b1818cea..b7d40922 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1327,6 +1327,40 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "generic artifact patterns", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{ + "apache": {`dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)`}, + } + }, + }, + { + name: "generic artifact patterns for an unknown upstream", + modify: func(cfg *Config) { + cfg.Upstream.GenericArtifacts = map[string][]string{ + "apache": {`(?P[^/]+)/(?P[^/]+)`}, + } + }, + wantErr: true, + }, + { + name: "generic artifact pattern does not compile", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+/(?P[^/]+)`}} + }, + wantErr: true, + }, + { + name: "generic artifact pattern without a file group", + modify: func(cfg *Config) { + cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"} + cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P[^/]+)/[^/]+`}} + }, + wantErr: true, + }, { name: "generic upstream URL is not absolute", modify: func(cfg *Config) { diff --git a/internal/handler/generic.go b/internal/handler/generic.go index a2458b28..67ae94ad 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -1,11 +1,15 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" + "fmt" "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/config" ) const ( @@ -33,7 +37,8 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // fixed URL shapes, such as mise's aqua backend fetching GitHub release // assets, and is pointed at by URL-rewriting settings on the client. // -// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}) are +// Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and +// paths matching the upstream's configured artifact patterns, are // version-pinned and cached in the shared artifact cache, so they keep being // served when the upstream is unreachable. Every other path is served through // the metadata cache: fresh within the metadata TTL, revalidated with the @@ -41,19 +46,36 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // or refuses the request. That covers API responses such as // api.github.com/repos/{owner}/{repo}/releases/tags/{tag}. type GenericHandler struct { - proxy *Proxy - repositories map[string]string + proxy *Proxy + repositories map[string]string + artifactPatterns map[string][]*regexp.Regexp } // NewGenericHandler creates a generic HTTP download proxy handler. -func NewGenericHandler(proxy *Proxy, repositories map[string]string) *GenericHandler { +// artifactPatterns maps upstream names to upstream.generic_artifacts patterns; +// a pattern that does not compile is logged and skipped. +func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPatterns ...map[string][]string) *GenericHandler { h := &GenericHandler{ - proxy: proxy, - repositories: make(map[string]string, len(repositories)), + proxy: proxy, + repositories: make(map[string]string, len(repositories)), + artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") } + if len(artifactPatterns) > 0 { + for name, patterns := range artifactPatterns[0] { + for _, pattern := range patterns { + re, err := config.CompileGenericArtifactPattern(pattern) + if err != nil { + proxy.Logger.Warn("skipping generic artifact pattern", + "repository", name, "pattern", pattern, "error", err) + continue + } + h.artifactPatterns[name] = append(h.artifactPatterns[name], re) + } + } + } return h } @@ -80,8 +102,8 @@ func (h *GenericHandler) Routes() http.Handler { return } - if asset, ok := parseGitHubReleaseAsset(rest); ok { - h.handleReleaseAsset(w, r, repository, upstreamURL, rest, asset) + if artifact, ok := h.parseArtifact(repository, rest); ok { + h.handleArtifact(w, r, repository, upstreamURL, rest, artifact) return } @@ -89,6 +111,54 @@ func (h *GenericHandler) Routes() http.Handler { }) } +// genericArtifact is the identity of a version-pinned download. +type genericArtifact struct { + name string + version string + filename string +} + +// parseArtifact returns the identity of a version-pinned download path: one +// matching the upstream's artifact patterns, else a GitHub release asset. +func (h *GenericHandler) parseArtifact(repository, path string) (genericArtifact, bool) { + for _, pattern := range h.artifactPatterns[repository] { + if artifact, ok := matchArtifactPattern(pattern, repository, path); ok { + return artifact, true + } + } + if asset, ok := parseGitHubReleaseAsset(path); ok { + return genericArtifact{ + name: asset.owner + "/" + asset.repo, + version: asset.tag, + filename: asset.filename, + }, true + } + return genericArtifact{}, false +} + +// matchArtifactPattern extracts the artifact identity from a path matched by +// pattern. The version and file must be single, non-empty path segments; the +// name defaults to the upstream name. +func matchArtifactPattern(pattern *regexp.Regexp, repository, path string) (genericArtifact, bool) { + matches := pattern.FindStringSubmatch(path) + if matches == nil { + return genericArtifact{}, false + } + artifact := genericArtifact{ + name: repository, + version: matches[pattern.SubexpIndex("version")], + filename: matches[pattern.SubexpIndex("file")], + } + if i := pattern.SubexpIndex("name"); i >= 0 && matches[i] != "" { + artifact.name = matches[i] + } + if artifact.version == "" || artifact.filename == "" || + strings.Contains(artifact.version, "/") || strings.Contains(artifact.filename, "/") { + return genericArtifact{}, false + } + return artifact, true +} + // githubReleaseAsset is the identity of a version-pinned release download. type githubReleaseAsset struct { owner string @@ -112,21 +182,26 @@ func parseGitHubReleaseAsset(path string) (githubReleaseAsset, bool) { }, true } -// handleReleaseAsset fetches and caches a version-pinned release asset in the -// artifact cache. The configured upstream name is part of the cache identity -// so two upstreams serving the same path never share bytes. -func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, asset githubReleaseAsset) { - name := asset.owner + "/" + asset.repo +// handleArtifact fetches and caches a version-pinned download in the artifact +// cache. The upstream, full path and query identify the cached bytes, since +// distinct downloads can have the same captured name, version and filename. +func (h *GenericHandler) handleArtifact(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, artifact genericArtifact) { downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + asset.filename + if r.URL.RawQuery != "" { + downloadURL += "?" + r.URL.RawQuery + } + cacheFilename := repository + "/" + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery) + "/" + artifact.filename - h.proxy.Logger.Info("generic release asset download", - "repository", repository, "name", name, "version", asset.tag, "filename", asset.filename) + h.proxy.Logger.Info("generic artifact download", + "repository", repository, "name", artifact.name, "version", artifact.version, "filename", artifact.filename) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), genericEcosystem, name, asset.tag, cacheFilename, downloadURL) + result, err := h.legacyArtifact(r.Context(), repository, downloadURL, artifact) + if err == nil && result == nil { + result, err = h.proxy.GetOrFetchArtifactFromURL( + r.Context(), genericEcosystem, artifact.name, artifact.version, cacheFilename, downloadURL) + } if err != nil { - h.proxy.serveArtifactError(w, err, "failed to fetch release asset") + h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } @@ -136,6 +211,25 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque ServeArtifactRequest(w, r, result) } +// legacyArtifact returns the artifact cached under the unhashed +// {repository}/{filename} key, provided its recorded upstream URL is +// downloadURL, so a different path or query sharing that key is not served. +func (h *GenericHandler) legacyArtifact(ctx context.Context, repository, downloadURL string, artifact genericArtifact) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(genericEcosystem, artifact.name, artifact.version) + if err != nil { + return nil, err + } + filename := repository + "/" + artifact.filename + record, err := h.proxy.DB.GetArtifact(versionPURL, filename) + if err != nil { + return nil, fmt.Errorf("checking artifact cache: %w", err) + } + if record == nil || record.UpstreamURL != downloadURL { + return nil, nil + } + return h.proxy.checkCache(ctx, pkgPURL, versionPURL, filename) +} + // handleMetadata serves any other path through the metadata cache. The query // string is forwarded and is part of the cache identity. A fixed Accept header // keeps all clients on one cached representation. @@ -146,14 +240,14 @@ func (h *GenericHandler) handleMetadata(w http.ResponseWriter, r *http.Request, } h.proxy.ProxyCached(w, r, target, genericEcosystem, - h.metadataCacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) + h.cacheKey(repository, upstreamURL, path, r.URL.RawQuery), genericAcceptAny) } -// metadataCacheKey derives the metadata cache key from the upstream name, its +// cacheKey derives the cache key from the upstream name, its // URL, the request path and query. Hashing the identity keeps distinct // upstreams from sharing entries and drops cached entries when an upstream is // repointed, mirroring APKHandler.metadataCacheKey. -func (h *GenericHandler) metadataCacheKey(repository, upstreamURL, path, query string) string { +func (h *GenericHandler) cacheKey(repository, upstreamURL, path, query string) string { identity := repository + "\x00" + upstreamURL + "\x00" + path + "\x00" + query digest := sha256.Sum256([]byte(identity)) return hex.EncodeToString(digest[:]) diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index e67a6b10..3e45fb94 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -1,13 +1,16 @@ package handler import ( + "context" "net/http" "net/http/httptest" + "regexp" "strings" "sync/atomic" "testing" "time" + "github.com/git-pkgs/proxy/internal/config" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/registries/fetch" ) @@ -51,6 +54,179 @@ func TestParseGitHubReleaseAsset(t *testing.T) { } } +const testMavenPattern = `dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)` + +func TestMatchArtifactPattern(t *testing.T) { + named, err := config.CompileGenericArtifactPattern(testMavenPattern) + if err != nil { + t.Fatal(err) + } + unnamed, err := config.CompileGenericArtifactPattern(`dist/v(?P[^/]+)/(?P.+)`) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + pattern *regexp.Regexp + path string + want genericArtifact + ok bool + }{ + { + "name group", + named, + "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, + true, + }, + { + "name defaults to upstream", + unnamed, + "dist/v20.15.0/node-v20.15.0-linux-x64.tar.xz", + genericArtifact{name: "apache", version: "20.15.0", filename: "node-v20.15.0-linux-x64.tar.xz"}, + true, + }, + // Patterns match whole paths only. + {"prefix only", named, "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz/extra", genericArtifact{}, false}, + {"suffix only", named, "mirror/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, + {"directory listing", named, "dist/maven/maven-3/3.9.9/binaries/", genericArtifact{}, false}, + // A file group spanning segments would nest storage paths. + {"file with slash", unnamed, "dist/v20.15.0/win-x64/node.exe", genericArtifact{}, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := matchArtifactPattern(tt.pattern, "apache", tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("matchArtifactPattern(%q) = (%+v, %v), want (%+v, %v)", tt.path, got, ok, tt.want, tt.ok) + } + }) + } +} + +func TestGenericHandler_ArtifactPatternIsCachedAndOtherPathsAreNot(t *testing.T) { + const ( + artifactPath = "/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz" + listingPath = "/dist/maven/maven-3/3.9.9/binaries/" + ) + tarball := []byte("maven tarball bytes") + var available atomic.Bool + available.Store(true) + var artifactRequests, listingRequests atomic.Int32 + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + switch r.URL.Path { + case artifactPath: + artifactRequests.Add(1) + _, _ = w.Write(tarball) + case listingPath: + listingRequests.Add(1) + _, _ = w.Write([]byte("listing")) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + proxy, db, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, + map[string]string{"apache": upstream.URL}, + map[string][]string{"apache": {testMavenPattern}}) + + w := serveGenericRequest(h, "/apache"+artifactPath) + if w.Code != http.StatusOK || w.Body.String() != string(tarball) { + t.Fatalf("status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) + } + + pkg, err := db.GetPackageByEcosystemName(genericEcosystem, "maven") + if err != nil || pkg == nil { + t.Fatalf("package generic/maven = %v, %v; want a cached package", pkg, err) + } + versions, err := db.GetVersionsByPackagePURL(pkg.PURL) + if err != nil || len(versions) != 1 || !strings.HasSuffix(versions[0].PURL, "@3.9.9") { + t.Fatalf("versions = %+v, %v; want one @3.9.9", versions, err) + } + + available.Store(false) + w = serveGenericRequest(h, "/apache"+artifactPath) + if w.Code != http.StatusOK || w.Body.String() != string(tarball) { + t.Fatalf("cached: status = %d, body = %q, want 200 %q", w.Code, w.Body.String(), tarball) + } + if got := artifactRequests.Load(); got != 1 { + t.Errorf("artifact upstream requests = %d, want 1", got) + } + + // Paths the pattern does not match keep going through the metadata route. + available.Store(true) + for range 2 { + serveGenericRequest(h, "/apache"+listingPath) + } + if got := listingRequests.Load(); got != 2 { + t.Errorf("listing upstream requests = %d, want 2 with metadata caching off", got) + } +} + +func TestGenericHandler_ConfiguredArtifactsKeepDistinctPathsAndQueries(t *testing.T) { + for _, tt := range []struct { + name string + targets []string + }{ + {"platform paths", []string{ + "/releases/tool/1.0.0/linux/tool.tar.gz", + "/releases/tool/1.0.0/darwin/tool.tar.gz", + }}, + {"query parameters", []string{ + "/releases/tool/1.0.0/linux/tool.tar.gz?token=example&variant=a%2Bb", + "/releases/tool/1.0.0/linux/tool.tar.gz?token=other&variant=a+b", + "/releases/tool/1.0.0/linux/tool.tar.gz", + }}, + } { + t.Run(tt.name, func(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + // Echo the exact target so dropped or changed queries fail too. + _, _ = w.Write([]byte(r.URL.RequestURI())) + })) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewGenericHandler(proxy, + map[string]string{"downloads": upstream.URL}, + map[string][]string{"downloads": {`releases/(?Ptool)/(?P[^/]+)/(linux|darwin)/(?P[^/]+)`}}) + + for pass := range 2 { + for _, target := range tt.targets { + w := serveGenericRequest(h, "/downloads"+target) + if w.Code != http.StatusOK || w.Body.String() != target { + t.Fatalf("pass %d, target %q: status = %d, body = %q; want 200 %q", + pass, target, w.Code, w.Body.String(), target) + } + } + upstream.Close() + } + if got := requests.Load(); got != int32(len(tt.targets)) { + t.Errorf("upstream requests = %d, want %d", got, len(tt.targets)) + } + }) + } +} + func TestGenericHandler_RejectsUnknownUpstreamAndBadPaths(t *testing.T) { h := NewGenericHandler(testProxy(), map[string]string{"github": "https://github.com"}) @@ -160,6 +336,54 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing } } +func TestGenericHandler_ServesReleaseAssetCachedUnderUnhashedKey(t *testing.T) { + asset := []byte("jq binary bytes") + var available atomic.Bool + available.Store(true) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + _, _ = w.Write(asset) + })) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + // Cache the asset the way releases before hashed keys did. + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), genericEcosystem, + "jqlang/jq", "jq-1.7.1", "github/jq-linux-amd64", upstream.URL+testReleaseAssetPath) + if err != nil { + t.Fatalf("priming legacy entry: %v", err) + } + if result.Reader != nil { + _ = result.Reader.Close() + } + available.Store(false) + + h := NewGenericHandler(proxy, map[string]string{"github": upstream.URL}) + w := serveGenericRequest(h, "/github"+testReleaseAssetPath) + if w.Code != http.StatusOK || w.Body.String() != string(asset) { + t.Fatalf("legacy entry: status = %d, body = %q; want 200 %q", w.Code, w.Body.String(), asset) + } + + w = serveGenericRequest(h, "/github"+testReleaseAssetPath+"?x=1") + if w.Code != http.StatusBadGateway { + t.Errorf("with query: status = %d, want 502", w.Code) + } + + repointed := NewGenericHandler(proxy, map[string]string{"github": upstream.URL + "/mirror"}) + w = serveGenericRequest(repointed, "/github"+testReleaseAssetPath) + if w.Code != http.StatusBadGateway { + t.Errorf("repointed upstream: status = %d, want 502", w.Code) + } +} + func TestGenericHandler_ReleaseAssetNotFoundIsNotCached(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) diff --git a/internal/server/server.go b/internal/server/server.go index 7494b94d..8dcdec89 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -445,7 +445,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.DebianRepositories, ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) - genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))