Cache configured version-pinned paths of generic upstreams

README.md

@@ -609,6 +609,24 @@ Commit a `mise.lock` and install with `mise install --locked` so pinned
 installs need no API call at all. Add a bearer token for `https://api.github.com`
 under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit.
 
+Tools mise downloads from elsewhere, such as Maven from `archive.apache.org`,
+can be cached the same way by declaring their version-pinned paths in
+`upstream.generic_artifacts` (see [configuration](docs/configuration.md)):
+
+```yaml
+upstream:
+  generic:
+    apache: "https://archive.apache.org"
+  generic_artifacts:
+    apache:
+      - 'dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)'
+```
+
+```toml
+[settings.url_replacements]
+"https://archive.apache.org/" = "http://localhost:8080/generic/apache/"
+```
+
 ## Configuration
 
 The proxy can be configured via:

config.example.yaml

@@ -243,6 +243,15 @@ upstream:
   # generic:
   #   github: "https://github.com"
   #   github-api: "https://api.github.com"
+  #   apache: "https://archive.apache.org"
+
+  # Version-pinned paths of generic upstreams, cached immutably like GitHub
+  # release assets. Each pattern must match the whole path after
+  # /generic/{name}/ and have `version` and `file` named groups; an optional
+  # `name` group names the package.
+  # generic_artifacts:
+  #   apache:
+  #     - 'dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)'
 
   # Authentication for upstream registries
   # Keys are absolute URL scopes. Scheme, host, effective port, and path

docs/configuration.md

@@ -276,7 +276,28 @@ Only configured upstreams are reachable, so this is not an open HTTP proxy.
 Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are
 version-pinned GitHub release assets: they are stored in the artifact cache
 and served from it without revalidation, including while the upstream is
-down. Every other path is served through the metadata cache (`cache_metadata`
+down.
+
+Other upstreams can declare their own version-pinned paths in
+`generic_artifacts`. Each pattern is a regular expression that must match the
+whole path after `/generic/{name}/`, with `version` and `file` named groups
+(single path segments) and an optional `name` group for the package name,
+which otherwise is the upstream name. Matching downloads are cached like
+release assets and listed under the `generic` ecosystem:
+
+```yaml
+upstream:
+  generic:
+    apache: "https://archive.apache.org"
+  generic_artifacts:
+    apache:
+      - 'dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)'
+```
+
+Only list paths whose content never changes for a given version: a match is
+never revalidated.
+
+Every other path is served through the metadata cache (`cache_metadata`
 must be enabled for offline fallback): fresh within `metadata_ttl`, then
 revalidated with the upstream's `ETag`/`Last-Modified`, and served stale with
 a `Warning: 110` header when the upstream fails, refuses or rate-limits the

internal/config/config.go

@@ -66,6 +66,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -637,11 +638,21 @@ type UpstreamConfig struct {
 	// Generic maps names to plain HTTP upstream base URLs, served at
 	// /generic/{name}/. The remaining request path and query string are
 	// appended to the upstream URL. GitHub release asset paths
-	// ({owner}/{repo}/releases/download/{tag}/{asset}) are cached in the
-	// artifact cache; everything else goes through the metadata cache.
+	// ({owner}/{repo}/releases/download/{tag}/{asset}) and paths matching
+	// GenericArtifacts are cached in the artifact cache; everything else
+	// goes through the metadata cache.
 	// Example: {"github": "https://github.com", "github-api": "https://api.github.com"}.
 	Generic map[string]string `json:"generic" yaml:"generic"`
 
+	// GenericArtifacts maps a Generic upstream name to regular expressions for
+	// its version-pinned download paths. A request path after
+	// /generic/{name}/ that one of them matches in full is cached in the
+	// artifact cache like a GitHub release asset. Each pattern needs `version`
+	// and `file` named groups; an optional `name` group names the package,
+	// which otherwise is the upstream name.
+	// Example: {"apache": ["dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)"]}.
+	GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"`
+
 	// Auth configures authentication for upstream registries.
 	// Keys are absolute URL scopes matched by scheme, host, effective port,
 	// and path-segment prefix.
@@ -694,6 +705,9 @@ func (u *UpstreamConfig) Validate() error {
 	if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil {
 		return err
 	}
+	if err := u.validateGenericArtifacts(); err != nil {
+		return err
+	}
 	if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil {
 		return err
 	}
@@ -708,6 +722,36 @@ func (u *UpstreamConfig) Validate() error {
 	return nil
 }
 
+func (u *UpstreamConfig) validateGenericArtifacts() error {
+	for name, patterns := range u.GenericArtifacts {
+		if _, found := u.Generic[name]; !found {
+			return fmt.Errorf("invalid upstream.generic_artifacts name %q: no upstream.generic entry", name)
+		}
+		for _, pattern := range patterns {
+			if _, err := CompileGenericArtifactPattern(pattern); err != nil {
+				return fmt.Errorf("invalid upstream.generic_artifacts.%s pattern %q: %w", name, pattern, err)
+			}
+		}
+	}
+	return nil
+}
+
+// CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern
+// so that it only matches a whole path, and checks it has the `version` and
+// `file` groups.
+func CompileGenericArtifactPattern(pattern string) (*regexp.Regexp, error) {
+	re, err := regexp.Compile(`^(?:` + pattern + `)$`)
+	if err != nil {
+		return nil, err
+	}
+	for _, group := range []string{"version", "file"} {
+		if re.SubexpIndex(group) < 0 {
+			return nil, fmt.Errorf("missing (?P<%s>...) group", group)
+		}
+	}
+	return re, nil
+}
+
 // debianReservedRepositoryNames are the upstream.debian archive's own root
 // paths, which a repository of the same name would shadow.
 var debianReservedRepositoryNames = []string{"pool", "dists"}

internal/config/config_test.go

@@ -1327,6 +1327,40 @@ func TestValidateNamedUpstreams(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "generic artifact patterns",
+			modify: func(cfg *Config) {
+				cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"}
+				cfg.Upstream.GenericArtifacts = map[string][]string{
+					"apache": {`dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)`},
+				}
+			},
+		},
+		{
+			name: "generic artifact patterns for an unknown upstream",
+			modify: func(cfg *Config) {
+				cfg.Upstream.GenericArtifacts = map[string][]string{
+					"apache": {`(?P<version>[^/]+)/(?P<file>[^/]+)`},
+				}
+			},
+			wantErr: true,
+		},
+		{
+			name: "generic artifact pattern does not compile",
+			modify: func(cfg *Config) {
+				cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"}
+				cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P<version>[^/]+/(?P<file>[^/]+)`}}
+			},
+			wantErr: true,
+		},
+		{
+			name: "generic artifact pattern without a file group",
+			modify: func(cfg *Config) {
+				cfg.Upstream.Generic = map[string]string{"apache": "https://archive.apache.org"}
+				cfg.Upstream.GenericArtifacts = map[string][]string{"apache": {`(?P<version>[^/]+)/[^/]+`}}
+			},
+			wantErr: true,
+		},
 		{
 			name: "generic upstream URL is not absolute",
 			modify: func(cfg *Config) {