From 5158899380129fafc6a0567f63cfa5756e006960 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 18 Oct 2026 19:07:56 +0100 Subject: [PATCH 1/2] Decode anaconda.org responses and publish conda artifacts owner is an object and upload_time a string in the real API, so FetchVersions failed on every package. Each file is now an Artifact with its subdir, build and arch. Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- internal/conda/conda.go | 96 ++++++++++++++++++++++++++---------- internal/conda/conda_test.go | 77 ++++++++++++++++++++++++++--- 2 files changed, 141 insertions(+), 32 deletions(-) diff --git a/internal/conda/conda.go b/internal/conda/conda.go index 62f9848..b4e162f 100644 --- a/internal/conda/conda.go +++ b/internal/conda/conda.go @@ -4,6 +4,7 @@ package conda import ( "context" "fmt" + "path" "strings" "time" @@ -74,25 +75,33 @@ type packageResponse struct { Versions []string `json:"versions"` LatestVersion string `json:"latest_version"` Files []fileInfo `json:"files"` - Owner string `json:"owner"` + Owner ownerInfo `json:"owner"` PublicAccess bool `json:"public_access"` } +type ownerInfo struct { + Login string `json:"login"` + Name string `json:"name"` +} + type fileInfo struct { - Version string `json:"version"` - Basename string `json:"basename"` - Attrs fileAttrs `json:"attrs"` - UploadTime int64 `json:"upload_time"` - MD5 string `json:"md5"` - SHA256 string `json:"sha256"` - Size int64 `json:"size"` - Ndownloads int64 `json:"ndownloads"` + Version string `json:"version"` + Basename string `json:"basename"` + DownloadURL string `json:"download_url"` + Attrs fileAttrs `json:"attrs"` + UploadTime string `json:"upload_time"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Ndownloads int64 `json:"ndownloads"` } type fileAttrs struct { Depends []string `json:"depends"` Arch string `json:"arch"` Platform string `json:"platform"` + Subdir string `json:"subdir"` + Build string `json:"build"` BuildNumber int `json:"build_number"` } @@ -141,7 +150,7 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package LatestVersion: resp.LatestVersion, Metadata: map[string]any{ "channel": channel, - "owner": resp.Owner, + "owner": resp.Owner.Login, "doc_url": resp.DocURL, "license_url": resp.LicenseURL, }, @@ -167,20 +176,11 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi // Build version info from files versionMap := make(map[string]*core.Version) for _, f := range resp.Files { - if _, exists := versionMap[f.Version]; !exists { - var publishedAt time.Time - if f.UploadTime > 0 { - publishedAt = time.Unix(f.UploadTime, 0) - } - - var integrity string - if f.SHA256 != "" { - integrity = "sha256-" + f.SHA256 - } else if f.MD5 != "" { - integrity = "md5-" + f.MD5 - } - - versionMap[f.Version] = &core.Version{ + integrity := fileIntegrity(f) + ver, exists := versionMap[f.Version] + if !exists { + publishedAt, _ := time.Parse(uploadTimeLayout, f.UploadTime) + ver = &core.Version{ Number: f.Version, PublishedAt: publishedAt, Integrity: integrity, @@ -189,6 +189,10 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi "downloads": f.Ndownloads, }, } + versionMap[f.Version] = ver + } + if artifact, ok := fileArtifact(f, integrity); ok { + ver.Artifacts = append(ver.Artifacts, artifact) } } @@ -205,6 +209,45 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi return versions, nil } +// uploadTimeLayout is the format of upload_time in anaconda.org file records, +// e.g. "2026-07-17 01:07:03.989000+00:00". +const uploadTimeLayout = "2006-01-02 15:04:05.999999-07:00" + +func fileIntegrity(f fileInfo) string { + if f.SHA256 != "" { + return "sha256-" + f.SHA256 + } + if f.MD5 != "" { + return "md5-" + f.MD5 + } + return "" +} + +// fileArtifact describes one build of a version. anaconda.org returns +// scheme-relative download URLs ("//api.anaconda.org/download/..."). +func fileArtifact(f fileInfo, integrity string) (core.Artifact, bool) { + downloadURL := f.DownloadURL + if downloadURL == "" { + return core.Artifact{}, false + } + if strings.HasPrefix(downloadURL, "//") { + downloadURL = "https:" + downloadURL + } + return core.Artifact{ + URL: downloadURL, + Filename: path.Base(f.Basename), + Integrity: integrity, + Size: f.Size, + Metadata: map[string]any{ + "subdir": f.Attrs.Subdir, + "build": f.Attrs.Build, + "build_number": f.Attrs.BuildNumber, + "arch": f.Attrs.Arch, + "platform": f.Attrs.Platform, + }, + }, true +} + func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { channel, pkgName := parsePackageName(name) if channel == "" { @@ -270,12 +313,13 @@ func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Ma return nil, err } - if resp.Owner == "" { + if resp.Owner.Login == "" { return nil, nil } return []core.Maintainer{{ - Login: resp.Owner, + Login: resp.Owner.Login, + Name: resp.Owner.Name, }}, nil } diff --git a/internal/conda/conda_test.go b/internal/conda/conda_test.go index cccea40..418a6ed 100644 --- a/internal/conda/conda_test.go +++ b/internal/conda/conda_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/git-pkgs/registries/internal/core" ) @@ -44,7 +45,7 @@ func TestFetchPackage(t *testing.T) { License: "BSD-3-Clause", HomeURL: "https://www.numpy.org", DevURL: "https://github.com/numpy/numpy", - Owner: "conda-forge", + Owner: ownerInfo{Login: "conda-forge"}, LatestVersion: "1.26.0", Versions: []string{"1.26.0", "1.25.2", "1.24.0"}, } @@ -88,7 +89,7 @@ func TestFetchPackageWithChannel(t *testing.T) { resp := packageResponse{ Name: "samtools", Summary: "Tools for manipulating next-gen sequencing data", - Owner: "bioconda", + Owner: ownerInfo{Login: "bioconda"}, } _ = json.NewEncoder(w).Encode(resp) })) @@ -114,9 +115,9 @@ func TestFetchVersions(t *testing.T) { Name: "pandas", Versions: []string{"2.1.0", "2.0.3", "1.5.3"}, Files: []fileInfo{ - {Version: "2.1.0", UploadTime: 1699900000, SHA256: "abc123"}, - {Version: "2.0.3", UploadTime: 1689100000, SHA256: "def456"}, - {Version: "1.5.3", UploadTime: 1678300000, MD5: "ghi789"}, + {Version: "2.1.0", UploadTime: "2023-11-13 18:26:40.000000+00:00", SHA256: "abc123"}, + {Version: "2.0.3", UploadTime: "2023-07-11 18:26:40.000000+00:00", SHA256: "def456"}, + {Version: "1.5.3", UploadTime: "2023-03-08 18:26:40.000000+00:00", MD5: "ghi789"}, }, } _ = json.NewEncoder(w).Encode(resp) @@ -198,7 +199,7 @@ func TestFetchMaintainers(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { resp := packageResponse{ Name: "scipy", - Owner: "conda-forge", + Owner: ownerInfo{Login: "conda-forge"}, } _ = json.NewEncoder(w).Encode(resp) })) @@ -270,3 +271,67 @@ func TestEcosystem(t *testing.T) { t.Errorf("expected ecosystem 'conda', got %q", reg.Ecosystem()) } } + +// anacondaZlib is trimmed from https://api.anaconda.org/package/conda-forge/zlib. +const anacondaZlib = `{ + "name": "zlib", + "owner": {"login": "conda-forge", "name": "conda-forge", "company": "", "url": ""}, + "versions": ["1.3.1", "1.2.13"], + "license": "Zlib", + "files": [ + {"version": "1.3.1", "basename": "linux-64/zlib-1.3.1-hb9d3cd8_2.conda", + "download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/linux-64/zlib-1.3.1-hb9d3cd8_2.conda", + "upload_time": "2024-09-17 07:32:19.214000+00:00", "sha256": "aaa", "size": 92286, + "attrs": {"subdir": "linux-64", "arch": "x86_64", "platform": "linux", "build": "hb9d3cd8_2", "build_number": 2, "depends": ["libgcc >=13"]}}, + {"version": "1.3.1", "basename": "osx-arm64/zlib-1.3.1-h8359307_2.conda", + "download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/osx-arm64/zlib-1.3.1-h8359307_2.conda", + "upload_time": "2024-09-17 07:35:01.100000+00:00", "sha256": "bbb", "size": 77606, + "attrs": {"subdir": "osx-arm64", "arch": "arm64", "platform": "osx", "build": "h8359307_2", "build_number": 2, "depends": []}}, + {"version": "1.2.13", "basename": "linux-64/zlib-1.2.13-h4ab18f5_6.conda", + "download_url": "//api.anaconda.org/download/conda-forge/zlib/1.2.13/linux-64/zlib-1.2.13-h4ab18f5_6.conda", + "upload_time": "2024-06-05 12:00:00.000000+00:00", "md5": "ccc", "size": 90000, + "attrs": {"subdir": "linux-64", "arch": "x86_64", "platform": "linux", "build": "h4ab18f5_6", "build_number": 6, "depends": []}} + ] +}` + +func TestFetchVersionsAnacondaResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(anacondaZlib)) + })) + defer server.Close() + + reg := New(server.URL, core.DefaultClient()) + versions, err := reg.FetchVersions(context.Background(), "zlib") + if err != nil { + t.Fatalf("FetchVersions failed: %v", err) + } + if len(versions) != 2 { + t.Fatalf("expected 2 versions, got %d", len(versions)) + } + + v := versions[0] + if want := time.Date(2024, 9, 17, 7, 32, 19, 214000000, time.UTC); !v.PublishedAt.Equal(want) { + t.Errorf("PublishedAt = %v, want %v", v.PublishedAt, want) + } + if len(v.Artifacts) != 2 { + t.Fatalf("expected 2 artifacts for 1.3.1, got %d", len(v.Artifacts)) + } + a := v.Artifacts[1] + if want := "https://api.anaconda.org/download/conda-forge/zlib/1.3.1/osx-arm64/zlib-1.3.1-h8359307_2.conda"; a.URL != want { + t.Errorf("URL = %q, want %q", a.URL, want) + } + if a.Filename != "zlib-1.3.1-h8359307_2.conda" || a.Integrity != "sha256-bbb" || a.Size != 77606 { + t.Errorf("artifact = %+v", a) + } + if a.Metadata["subdir"] != "osx-arm64" || a.Metadata["build"] != "h8359307_2" || a.Metadata["arch"] != "arm64" { + t.Errorf("metadata = %+v", a.Metadata) + } + if versions[1].Artifacts[0].Integrity != "md5-ccc" { + t.Errorf("1.2.13 integrity = %q, want md5-ccc", versions[1].Artifacts[0].Integrity) + } + + maintainers, err := reg.FetchMaintainers(context.Background(), "zlib") + if err != nil || len(maintainers) != 1 || maintainers[0].Login != "conda-forge" { + t.Errorf("FetchMaintainers = %+v, %v", maintainers, err) + } +} From 209dd7e94084867750588f6e87e0fec577652a3d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 18 Oct 2026 19:07:56 +0100 Subject: [PATCH 2/2] Select artifacts by PURL qualifiers in fetch.Resolver Claude-Session: https://claude.ai/code/session_01NxsA75HBjdWQKcdx7FDTDc --- README.md | 8 +++ fetch/qualifiers_test.go | 103 +++++++++++++++++++++++++++++++++++++++ fetch/resolver.go | 25 ++++++++++ 3 files changed, 136 insertions(+) create mode 100644 fetch/qualifiers_test.go diff --git a/README.md b/README.md index 047185d..f30d16e 100644 --- a/README.md +++ b/README.md @@ -478,6 +478,14 @@ info, err := resolver.Resolve(ctx, "cargo", "serde", "1.0.0") // info.Integrity may be populated for metadata-resolved URLs ``` +When a version has several files, `ResolveWithOptions` picks one by PURL qualifiers. `arch`, `distro`, `subdir` and `build` are matched against artifact metadata, and an artifact that doesn't record a qualifier isn't ruled out by it. `repository_url` and `channel` are ignored: register a registry for that URL instead. + +```go +info, err := resolver.ResolveWithOptions(ctx, "conda", "zlib", "1.3.1", fetch.ResolveOptions{ + Qualifiers: map[string]string{"subdir": "osx-arm64"}, +}) +``` + The resolver also works without a registered registry for ecosystems with predictable URL patterns: ```go diff --git a/fetch/qualifiers_test.go b/fetch/qualifiers_test.go new file mode 100644 index 0000000..dfdbd3c --- /dev/null +++ b/fetch/qualifiers_test.go @@ -0,0 +1,103 @@ +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/conda" +) + +const anacondaZlib = `{"name": "zlib", "owner": {"login": "conda-forge"}, "versions": ["1.3.1"], "files": [ + {"version": "1.3.1", "basename": "linux-64/zlib-1.3.1-hb9d3cd8_2.conda", + "download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/linux-64/zlib-1.3.1-hb9d3cd8_2.conda", + "upload_time": "2024-09-17 07:32:19.214000+00:00", "sha256": "aaa", + "attrs": {"subdir": "linux-64", "arch": "x86_64", "platform": "linux", "build": "hb9d3cd8_2"}}, + {"version": "1.3.1", "basename": "osx-arm64/zlib-1.3.1-h8359307_2.conda", + "download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/osx-arm64/zlib-1.3.1-h8359307_2.conda", + "upload_time": "2024-09-17 07:35:01.100000+00:00", "sha256": "bbb", + "attrs": {"subdir": "osx-arm64", "arch": "arm64", "platform": "osx", "build": "h8359307_2"}} +]}` + +func TestResolveCondaQualifiers(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/package/conda-forge/zlib" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(anacondaZlib)) + })) + defer server.Close() + + reg, err := registries.New("conda", server.URL, registries.DefaultClient()) + if err != nil { + t.Fatal(err) + } + r := NewResolver() + r.RegisterRegistry(reg) + + linux := "zlib-1.3.1-hb9d3cd8_2.conda" + osx := "zlib-1.3.1-h8359307_2.conda" + tests := []struct { + name string + qualifiers map[string]string + want string + }{ + {"none", nil, linux}, + {"subdir", map[string]string{"subdir": "osx-arm64"}, osx}, + {"build", map[string]string{"build": "hb9d3cd8_2"}, linux}, + {"arch", map[string]string{"arch": "arm64"}, osx}, + {"subdir and build", map[string]string{"subdir": "osx-arm64", "build": "h8359307_2"}, osx}, + {"repository_url and channel ignored", map[string]string{"repository_url": "https://example.com", "channel": "bioconda", "subdir": "osx-arm64"}, osx}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, err := r.ResolveWithOptions(context.Background(), "conda", "zlib", "1.3.1", ResolveOptions{Qualifiers: tt.qualifiers}) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if info.Filename != tt.want { + t.Errorf("Filename = %q, want %q", info.Filename, tt.want) + } + }) + } + + for _, q := range []map[string]string{ + {"subdir": "win-64"}, + {"subdir": "osx-arm64", "build": "hb9d3cd8_2"}, + } { + _, err := r.ResolveWithOptions(context.Background(), "conda", "zlib", "1.3.1", ResolveOptions{Qualifiers: q}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("qualifiers %v: err = %v, want ErrNoMatchingArtifact", q, err) + } + } +} + +func TestResolveQualifiersSkipArtifactsWithoutTheField(t *testing.T) { + r := NewResolver() + r.RegisterRegistry(&fakeRegistry{versions: []registries.Version{{ + Number: "1.0.0", + Artifacts: []registries.Artifact{ + {URL: "https://example.com/pkg-1.0.0-bookworm-amd64.deb", Metadata: map[string]any{"distro": "bookworm", "arch": "amd64"}}, + {URL: "https://example.com/pkg-1.0.0-trixie.deb", Metadata: map[string]any{"distro": "trixie"}}, + }, + }}}) + + info, err := r.ResolveWithOptions(context.Background(), "fake", "pkg", "1.0.0", + ResolveOptions{Qualifiers: map[string]string{"distro": "trixie", "arch": "amd64"}}) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if want := "pkg-1.0.0-trixie.deb"; info.Filename != want { + t.Errorf("Filename = %q, want %q", info.Filename, want) + } + + _, err = r.ResolveWithOptions(context.Background(), "fake", "pkg", "1.0.0", + ResolveOptions{Qualifiers: map[string]string{"distro": "sid"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Errorf("err = %v, want ErrNoMatchingArtifact", err) + } +} diff --git a/fetch/resolver.go b/fetch/resolver.go index 985fca5..ee83b6a 100644 --- a/fetch/resolver.go +++ b/fetch/resolver.go @@ -57,8 +57,17 @@ type ArtifactInfo struct { type ResolveOptions struct { Filename string Integrity string + // Qualifiers are PURL qualifiers. Those in selectionQualifiers pick among + // the artifacts a registry lists for a version; an artifact that does not + // record a qualifier is not ruled out by it. repository_url and channel + // choose the registry, which is up to the caller, and are ignored here. + Qualifiers map[string]string } +// selectionQualifiers are the PURL qualifiers matched against artifact +// metadata of the same name. +var selectionQualifiers = []string{"arch", "distro", "subdir", "build"} + // Resolve returns the download URL and filename for a package artifact. func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) { return r.ResolveWithOptions(ctx, ecosystem, name, version, ResolveOptions{}) @@ -207,6 +216,9 @@ func selectArtifact(candidates []registries.Artifact, options ResolveOptions) (* if options.Integrity != "" && !integrityMatches(candidate.Integrity, options.Integrity) { continue } + if !qualifiersMatch(candidate.Metadata, options.Qualifiers) { + continue + } if err := checkMetadataURL(candidate.URL); err != nil { return nil, err } @@ -220,6 +232,19 @@ func selectArtifact(candidates []registries.Artifact, options ResolveOptions) (* return nil, ErrNoMatchingArtifact } +func qualifiersMatch(metadata map[string]any, qualifiers map[string]string) bool { + for _, key := range selectionQualifiers { + want := qualifiers[key] + if want == "" { + continue + } + if got, ok := metadata[key].(string); ok && got != "" && got != want { + return false + } + } + return true +} + func matchSingleArtifact(info *ArtifactInfo, options ResolveOptions, err error) (*ArtifactInfo, error) { if err != nil { return nil, err