Cache Conan 2 recipe and package archives

internal/handler/conan.go

@@ -1,7 +1,6 @@
 package handler
 
 import (
-	"fmt"
 	"net/http"
 	"strings"
 )
@@ -41,13 +40,12 @@ func (h *ConanHandler) Routes() http.Handler {
 	mux.HandleFunc("GET /v1/ping", h.handlePing)
 	mux.HandleFunc("GET /v2/ping", h.handlePing)
 
-	// Recipe file downloads (cache these)
-	mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile)
-	mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile)
-
-	// Package file downloads (cache these)
-	mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile)
-	mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile)
+	// Recipe and package file downloads, as built by the Conan 2 client
+	// (ClientV2Router.recipe_file and package_file). The v1 protocol hands out
+	// signed absolute URLs on the upstream host, so its downloads never pass
+	// through here.
+	mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/files/{filename}", h.handleRecipeFile)
+	mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/packages/{pkgref}/revisions/{pkgrev}/files/{filename}", h.handlePackageFile)
 
 	// Proxy all other endpoints (metadata, search, etc.) with caching
 	mux.HandleFunc("GET /", h.proxyCached)
@@ -70,25 +68,19 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request)
 	revision := r.PathValue("revision")
 	filename := r.PathValue("filename")
 
-	// Only cache specific files
+	// Only the archives are artifacts; the rest goes through the metadata cache.
 	if !h.shouldCacheFile(filename) {
-		h.proxyUpstream(w, r)
+		h.proxyCached(w, r)
 		return
 	}
 
-	// Conan package name format: name/version@user/channel
-	packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel)
-
 	h.proxy.Logger.Info("conan recipe download",
 		"name", name, "version", version, "user", user, "channel", channel, "filename", filename)
 
 	upstreamURL := h.upstreamURL + r.URL.Path
+	storageFilename := conanStorageFilename(user, channel, revision+"_"+filename)
 
-	// Use revision as part of version for storage
-	storageVersion := fmt.Sprintf("%s_%s", version, revision)
-	storageFilename := fmt.Sprintf("recipe_%s", filename)
-
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch file")
 		return
@@ -108,24 +100,19 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
 	pkgrev := r.PathValue("pkgrev")
 	filename := r.PathValue("filename")
 
-	// Only cache specific files
+	// Only the archives are artifacts; the rest goes through the metadata cache.
 	if !h.shouldCacheFile(filename) {
-		h.proxyUpstream(w, r)
+		h.proxyCached(w, r)
 		return
 	}
 
-	packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel)
-
 	h.proxy.Logger.Info("conan package download",
 		"name", name, "version", version, "pkgref", pkgref, "filename", filename)
 
 	upstreamURL := h.upstreamURL + r.URL.Path
+	storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename)
 
-	// Use revision and package ref as part of version for storage
-	storageVersion := fmt.Sprintf("%s_%s_%s_%s", version, revision, pkgref, pkgrev)
-	storageFilename := fmt.Sprintf("package_%s", filename)
-
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch file")
 		return
@@ -134,13 +121,24 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
 	ServeArtifactRequest(w, r, result)
 }
 
+// conanStorageFilename names a cached archive. Archives are stored under
+// pkg:conan/{name}@{version}, so the file name carries the revisions and, when
+// set, user and channel, to keep every recipe and binary apart.
+func conanStorageFilename(user, channel, file string) string {
+	if user == "_" && channel == "_" {
+		return file
+	}
+	return user + "@" + channel + "_" + file
+}
+
 // shouldCacheFile returns true if the file should be cached.
 func (h *ConanHandler) shouldCacheFile(filename string) bool {
-	// Cache the large archive files
+	// Cache the large archive files. Since Conan 2.25 they can also be xz or
+	// zstd (core.upload:compression_format).
 	cacheFiles := []string{
-		"conan_sources.tgz",
-		"conan_export.tgz",
-		"conan_package.tgz",
+		"conan_sources.tgz", "conan_sources.txz", "conan_sources.tzst",
+		"conan_export.tgz", "conan_export.txz", "conan_export.tzst",
+		"conan_package.tgz", "conan_package.txz", "conan_package.tzst",
 	}
 
 	for _, f := range cacheFiles {

internal/handler/conan_test.go

@@ -6,7 +6,9 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
 const testProxyURL = "http://localhost:8080"
@@ -28,6 +30,13 @@ func TestConanShouldCacheFile(t *testing.T) {
 		{"conan_sources.tgz", true},
 		{"conan_export.tgz", true},
 		{"conan_package.tgz", true},
+		{"conan_sources.txz", true},
+		{"conan_export.txz", true},
+		{"conan_package.txz", true},
+		{"conan_sources.tzst", true},
+		{"conan_export.tzst", true},
+		{"conan_package.tzst", true},
+		{"conan_package.tar.gz", false},
 		{"conanfile.py", false},
 		{"conanmanifest.txt", false},
 		{"conaninfo.txt", false},
@@ -225,7 +234,7 @@ func TestConanRecipeFileNonCacheable(t *testing.T) {
 		proxyURL:    "http://proxy.local",
 	}
 
-	req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/recipe/conanfile.py", nil)
+	req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/files/conanfile.py", nil)
 	req.SetPathValue("name", "zlib")
 	req.SetPathValue("version", "1.2.13")
 	req.SetPathValue("user", "_")
@@ -260,7 +269,7 @@ func TestConanPackageFileNonCacheable(t *testing.T) {
 		proxyURL:    "http://proxy.local",
 	}
 
-	req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/package/pkgref1/pkgrev1/conaninfo.txt", nil)
+	req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/packages/pkgref1/revisions/pkgrev1/files/conaninfo.txt", nil)
 	req.SetPathValue("name", "zlib")
 	req.SetPathValue("version", "1.2.13")
 	req.SetPathValue("user", "_")
@@ -283,6 +292,173 @@ func TestConanPackageFileNonCacheable(t *testing.T) {
 	}
 }
 
+// The Conan 2 client downloads recipe and package files over these routes;
+// the archives must be stored as artifacts and served from the cache after.
+func TestConanV2FileRoutesCacheArchives(t *testing.T) {
+	tests := []struct {
+		name     string
+		path     string
+		filename string
+	}{
+		{"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz",
+			"rrev1_conan_sources.tgz"},
+		{"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz",
+			"rrev1_pkgid1_prev1_conan_package.tgz"},
+		{"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz",
+			"rrev1_conan_export.txz"},
+		{"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst",
+			"rrev1_pkgid1_prev1_conan_package.tzst"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			proxy, db, _, fetcher := setupTestProxy(t)
+			upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				t.Errorf("archive %s was proxied instead of fetched as an artifact", r.URL.Path)
+				http.Error(w, "unexpected", http.StatusInternalServerError)
+			}))
+			defer upstream.Close()
+			proxy.HTTPClient = upstream.Client()
+
+			h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL)
+			srv := httptest.NewServer(h.Routes())
+			defer srv.Close()
+
+			for i := range 2 {
+				fetcher.artifact = artifactBody("archive bytes")
+				fetcher.fetchCalled = false
+
+				resp, err := http.Get(srv.URL + tt.path)
+				if err != nil {
+					t.Fatalf("request %d failed: %v", i+1, err)
+				}
+				body, _ := io.ReadAll(resp.Body)
+				_ = resp.Body.Close()
+
+				if resp.StatusCode != http.StatusOK || string(body) != "archive bytes" {
+					t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "archive bytes")
+				}
+				if fetcher.fetchCalled != (i == 0) {
+					t.Errorf("request %d: fetched = %v, want %v", i+1, fetcher.fetchCalled, i == 0)
+				}
+				if i == 0 && fetcher.fetchedURL != upstream.URL+tt.path {
+					t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, upstream.URL+tt.path)
+				}
+			}
+
+			recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", tt.filename)
+		})
+	}
+}
+
+// References that differ only in user and channel share pkg:conan/zlib@1.3.1
+// but must not share files, even with the same recipe revision.
+func TestConanUserChannelKeptApart(t *testing.T) {
+	proxy, db, _, fetcher := setupTestProxy(t)
+	h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid")
+	srv := httptest.NewServer(h.Routes())
+	defer srv.Close()
+
+	for _, ref := range []struct{ path, body, filename string }{
+		{"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz",
+			"center sources", "rrev1_conan_sources.tgz"},
+		{"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz",
+			"acme sources", "acme@stable_rrev1_conan_sources.tgz"},
+	} {
+		fetcher.artifact = artifactBody(ref.body)
+		fetcher.fetchCalled = false
+
+		resp, err := http.Get(srv.URL + ref.path)
+		if err != nil {
+			t.Fatalf("GET %s failed: %v", ref.path, err)
+		}
+		body, _ := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+
+		if string(body) != ref.body || !fetcher.fetchCalled {
+			t.Errorf("GET %s: body = %q, fetched = %v, want %q from upstream", ref.path, body, fetcher.fetchCalled, ref.body)
+		}
+		recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", ref.filename)
+	}
+}
+
+// A denylist entry for a Conan version blocks its recipe and binary archives.
+func TestConanDenylistMatchesVersion(t *testing.T) {
+	proxy, _, _, fetcher := setupTestProxy(t)
+	setTestDenylist(t, proxy, "pkg:conan/zlib@1.3.1")
+	h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid")
+	srv := httptest.NewServer(h.Routes())
+	defer srv.Close()
+
+	for _, path := range []string{
+		"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz",
+		"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz",
+	} {
+		fetcher.artifact = artifactBody("archive bytes")
+		resp, err := http.Get(srv.URL + path)
+		if err != nil {
+			t.Fatalf("GET %s failed: %v", path, err)
+		}
+		_ = resp.Body.Close()
+		if resp.StatusCode != http.StatusForbidden {
+			t.Errorf("GET %s: status = %d, want %d", path, resp.StatusCode, http.StatusForbidden)
+		}
+	}
+	if fetcher.fetchCalled {
+		t.Error("denied archive was fetched from upstream")
+	}
+}
+
+// The small files next to the archives keep going through the metadata cache.
+func TestConanV2FileRoutesKeepSmallFilesAsMetadata(t *testing.T) {
+	for name, path := range map[string]string{
+		"recipe":  "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conanmanifest.txt",
+		"package": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conaninfo.txt",
+	} {
+		t.Run(name, func(t *testing.T) {
+			var requests atomic.Int32
+			upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				requests.Add(1)
+				if r.URL.Path != path {
+					http.NotFound(w, r)
+					return
+				}
+				w.Header().Set("Content-Type", "text/plain")
+				_, _ = w.Write([]byte("small file"))
+			}))
+			defer upstream.Close()
+
+			proxy, _, _, fetcher := setupTestProxy(t)
+			proxy.CacheMetadata = true
+			proxy.MetadataTTL = time.Hour
+			proxy.HTTPClient = upstream.Client()
+
+			h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL)
+			srv := httptest.NewServer(h.Routes())
+			defer srv.Close()
+
+			for i := range 2 {
+				resp, err := http.Get(srv.URL + path)
+				if err != nil {
+					t.Fatalf("request %d failed: %v", i+1, err)
+				}
+				body, _ := io.ReadAll(resp.Body)
+				_ = resp.Body.Close()
+				if resp.StatusCode != http.StatusOK || string(body) != "small file" {
+					t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "small file")
+				}
+			}
+
+			if got := requests.Load(); got != 1 {
+				t.Errorf("upstream requests = %d, want 1 (second served from the metadata cache)", got)
+			}
+			if fetcher.fetchCalled {
+				t.Error("small file was fetched as an artifact")
+			}
+		})
+	}
+}
+
 func TestConanRoutes(t *testing.T) {
 	h := &ConanHandler{
 		proxy:       conanTestProxy(),

internal/handler/download_test.go

@@ -1163,7 +1163,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) {
 	srv := httptest.NewServer(h.Routes())
 	defer srv.Close()
 
-	resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz")
+	resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz")
 	if err != nil {
 		t.Fatalf("request failed: %v", err)
 	}
@@ -1173,7 +1173,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) {
 		t.Error("expected fetcher to be called on cache miss")
 	}
 
-	want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz"
+	want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz"
 	if fetcher.fetchedURL != want {
 		t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want)
 	}
@@ -1190,7 +1190,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) {
 	srv := httptest.NewServer(h.Routes())
 	defer srv.Close()
 
-	resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz")
+	resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz")
 	if err != nil {
 		t.Fatalf("request failed: %v", err)
 	}
@@ -1200,7 +1200,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) {
 		t.Error("expected fetcher to be called on cache miss")
 	}
 
-	want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz"
+	want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz"
 	if fetcher.fetchedURL != want {
 		t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want)
 	}

internal/handler/notfound_ecosystems_test.go

@@ -31,7 +31,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) {
 			func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }},
 		{"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2",
 			func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }},
-		{"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz",
+		{"conan", "/v2/conans/zlib/1.3.1/_/_/revisions/0/files/conan_sources.tgz",
 			func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }},
 		{"gem", "/gems/rails-7.1.0.gem",
 			func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }},

internal/handler/relay_test.go

@@ -50,7 +50,7 @@ func relayTestRoutes(proxy *Proxy, upstream string) http.Handler {
 func TestRelayRoutes(t *testing.T) {
 	for _, route := range []string{
 		"/upstream", "/file", "/metadata", "/nuget/query",
-		"/conan/v2/files/demo/1.0/user/stable/rev/recipe/other.txt",
+		"/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt",
 		"/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies",
 		"/gem/info/demo", "/conda/conda-forge/noarch/repodata.json",
 		"/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift",