Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,14 @@ info, err := resolver.Resolve(ctx, "cargo", "serde", "1.0.0")
// info.Integrity may be populated for metadata-resolved URLs
```

When a version has several files, `ResolveWithOptions` picks one by PURL qualifiers. `arch`, `distro`, `subdir` and `build` are matched against artifact metadata, and an artifact that doesn't record a qualifier isn't ruled out by it. `repository_url` and `channel` are ignored: register a registry for that URL instead.

```go
info, err := resolver.ResolveWithOptions(ctx, "conda", "zlib", "1.3.1", fetch.ResolveOptions{
Qualifiers: map[string]string{"subdir": "osx-arm64"},
})
```

The resolver also works without a registered registry for ecosystems with predictable URL patterns:

```go
Expand Down
103 changes: 103 additions & 0 deletions fetch/qualifiers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package fetch

import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"

"github.com/git-pkgs/registries"
_ "github.com/git-pkgs/registries/internal/conda"
)

const anacondaZlib = `{"name": "zlib", "owner": {"login": "conda-forge"}, "versions": ["1.3.1"], "files": [
{"version": "1.3.1", "basename": "linux-64/zlib-1.3.1-hb9d3cd8_2.conda",
"download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/linux-64/zlib-1.3.1-hb9d3cd8_2.conda",
"upload_time": "2024-09-17 07:32:19.214000+00:00", "sha256": "aaa",
"attrs": {"subdir": "linux-64", "arch": "x86_64", "platform": "linux", "build": "hb9d3cd8_2"}},
{"version": "1.3.1", "basename": "osx-arm64/zlib-1.3.1-h8359307_2.conda",
"download_url": "//api.anaconda.org/download/conda-forge/zlib/1.3.1/osx-arm64/zlib-1.3.1-h8359307_2.conda",
"upload_time": "2024-09-17 07:35:01.100000+00:00", "sha256": "bbb",
"attrs": {"subdir": "osx-arm64", "arch": "arm64", "platform": "osx", "build": "h8359307_2"}}
]}`

func TestResolveCondaQualifiers(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/package/conda-forge/zlib" {
http.NotFound(w, r)
return
}
_, _ = w.Write([]byte(anacondaZlib))
}))
defer server.Close()

reg, err := registries.New("conda", server.URL, registries.DefaultClient())
if err != nil {
t.Fatal(err)
}
r := NewResolver()
r.RegisterRegistry(reg)

linux := "zlib-1.3.1-hb9d3cd8_2.conda"
osx := "zlib-1.3.1-h8359307_2.conda"
tests := []struct {
name string
qualifiers map[string]string
want string
}{
{"none", nil, linux},
{"subdir", map[string]string{"subdir": "osx-arm64"}, osx},
{"build", map[string]string{"build": "hb9d3cd8_2"}, linux},
{"arch", map[string]string{"arch": "arm64"}, osx},
{"subdir and build", map[string]string{"subdir": "osx-arm64", "build": "h8359307_2"}, osx},
{"repository_url and channel ignored", map[string]string{"repository_url": "https://example.com", "channel": "bioconda", "subdir": "osx-arm64"}, osx},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
info, err := r.ResolveWithOptions(context.Background(), "conda", "zlib", "1.3.1", ResolveOptions{Qualifiers: tt.qualifiers})
if err != nil {
t.Fatalf("Resolve failed: %v", err)
}
if info.Filename != tt.want {
t.Errorf("Filename = %q, want %q", info.Filename, tt.want)
}
})
}

for _, q := range []map[string]string{
{"subdir": "win-64"},
{"subdir": "osx-arm64", "build": "hb9d3cd8_2"},
} {
_, err := r.ResolveWithOptions(context.Background(), "conda", "zlib", "1.3.1", ResolveOptions{Qualifiers: q})
if !errors.Is(err, ErrNoMatchingArtifact) {
t.Errorf("qualifiers %v: err = %v, want ErrNoMatchingArtifact", q, err)
}
}
}

func TestResolveQualifiersSkipArtifactsWithoutTheField(t *testing.T) {
r := NewResolver()
r.RegisterRegistry(&fakeRegistry{versions: []registries.Version{{
Number: "1.0.0",
Artifacts: []registries.Artifact{
{URL: "https://example.com/pkg-1.0.0-bookworm-amd64.deb", Metadata: map[string]any{"distro": "bookworm", "arch": "amd64"}},
{URL: "https://example.com/pkg-1.0.0-trixie.deb", Metadata: map[string]any{"distro": "trixie"}},
},
}}})

info, err := r.ResolveWithOptions(context.Background(), "fake", "pkg", "1.0.0",
ResolveOptions{Qualifiers: map[string]string{"distro": "trixie", "arch": "amd64"}})
if err != nil {
t.Fatalf("Resolve failed: %v", err)
}
if want := "pkg-1.0.0-trixie.deb"; info.Filename != want {
t.Errorf("Filename = %q, want %q", info.Filename, want)
}

_, err = r.ResolveWithOptions(context.Background(), "fake", "pkg", "1.0.0",
ResolveOptions{Qualifiers: map[string]string{"distro": "sid"}})
if !errors.Is(err, ErrNoMatchingArtifact) {
t.Errorf("err = %v, want ErrNoMatchingArtifact", err)
}
}
25 changes: 25 additions & 0 deletions fetch/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,17 @@ type ArtifactInfo struct {
type ResolveOptions struct {
Filename string
Integrity string
// Qualifiers are PURL qualifiers. Those in selectionQualifiers pick among
// the artifacts a registry lists for a version; an artifact that does not
// record a qualifier is not ruled out by it. repository_url and channel
// choose the registry, which is up to the caller, and are ignored here.
Qualifiers map[string]string
}

// selectionQualifiers are the PURL qualifiers matched against artifact
// metadata of the same name.
var selectionQualifiers = []string{"arch", "distro", "subdir", "build"}

// Resolve returns the download URL and filename for a package artifact.
func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) {
return r.ResolveWithOptions(ctx, ecosystem, name, version, ResolveOptions{})
Expand Down Expand Up @@ -207,6 +216,9 @@ func selectArtifact(candidates []registries.Artifact, options ResolveOptions) (*
if options.Integrity != "" && !integrityMatches(candidate.Integrity, options.Integrity) {
continue
}
if !qualifiersMatch(candidate.Metadata, options.Qualifiers) {
continue
}
if err := checkMetadataURL(candidate.URL); err != nil {
return nil, err
}
Expand All @@ -220,6 +232,19 @@ func selectArtifact(candidates []registries.Artifact, options ResolveOptions) (*
return nil, ErrNoMatchingArtifact
}

func qualifiersMatch(metadata map[string]any, qualifiers map[string]string) bool {
for _, key := range selectionQualifiers {
want := qualifiers[key]
if want == "" {
continue
}
if got, ok := metadata[key].(string); ok && got != "" && got != want {
return false
}
}
return true
}

func matchSingleArtifact(info *ArtifactInfo, options ResolveOptions, err error) (*ArtifactInfo, error) {
if err != nil {
return nil, err
Expand Down
96 changes: 70 additions & 26 deletions internal/conda/conda.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package conda
import (
"context"
"fmt"
"path"
"strings"
"time"

Expand Down Expand Up @@ -74,25 +75,33 @@ type packageResponse struct {
Versions []string `json:"versions"`
LatestVersion string `json:"latest_version"`
Files []fileInfo `json:"files"`
Owner string `json:"owner"`
Owner ownerInfo `json:"owner"`
PublicAccess bool `json:"public_access"`
}

type ownerInfo struct {
Login string `json:"login"`
Name string `json:"name"`
}

type fileInfo struct {
Version string `json:"version"`
Basename string `json:"basename"`
Attrs fileAttrs `json:"attrs"`
UploadTime int64 `json:"upload_time"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
Ndownloads int64 `json:"ndownloads"`
Version string `json:"version"`
Basename string `json:"basename"`
DownloadURL string `json:"download_url"`
Attrs fileAttrs `json:"attrs"`
UploadTime string `json:"upload_time"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
Ndownloads int64 `json:"ndownloads"`
}

type fileAttrs struct {
Depends []string `json:"depends"`
Arch string `json:"arch"`
Platform string `json:"platform"`
Subdir string `json:"subdir"`
Build string `json:"build"`
BuildNumber int `json:"build_number"`
}

Expand Down Expand Up @@ -141,7 +150,7 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package
LatestVersion: resp.LatestVersion,
Metadata: map[string]any{
"channel": channel,
"owner": resp.Owner,
"owner": resp.Owner.Login,
"doc_url": resp.DocURL,
"license_url": resp.LicenseURL,
},
Expand All @@ -167,20 +176,11 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
// Build version info from files
versionMap := make(map[string]*core.Version)
for _, f := range resp.Files {
if _, exists := versionMap[f.Version]; !exists {
var publishedAt time.Time
if f.UploadTime > 0 {
publishedAt = time.Unix(f.UploadTime, 0)
}

var integrity string
if f.SHA256 != "" {
integrity = "sha256-" + f.SHA256
} else if f.MD5 != "" {
integrity = "md5-" + f.MD5
}

versionMap[f.Version] = &core.Version{
integrity := fileIntegrity(f)
ver, exists := versionMap[f.Version]
if !exists {
publishedAt, _ := time.Parse(uploadTimeLayout, f.UploadTime)
ver = &core.Version{
Number: f.Version,
PublishedAt: publishedAt,
Integrity: integrity,
Expand All @@ -189,6 +189,10 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
"downloads": f.Ndownloads,
},
}
versionMap[f.Version] = ver
}
if artifact, ok := fileArtifact(f, integrity); ok {
ver.Artifacts = append(ver.Artifacts, artifact)
}
}

Expand All @@ -205,6 +209,45 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
return versions, nil
}

// uploadTimeLayout is the format of upload_time in anaconda.org file records,
// e.g. "2026-07-17 01:07:03.989000+00:00".
const uploadTimeLayout = "2006-01-02 15:04:05.999999-07:00"

func fileIntegrity(f fileInfo) string {
if f.SHA256 != "" {
return "sha256-" + f.SHA256
}
if f.MD5 != "" {
return "md5-" + f.MD5
}
return ""
}

// fileArtifact describes one build of a version. anaconda.org returns
// scheme-relative download URLs ("//api.anaconda.org/download/...").
func fileArtifact(f fileInfo, integrity string) (core.Artifact, bool) {
downloadURL := f.DownloadURL
if downloadURL == "" {
return core.Artifact{}, false
}
if strings.HasPrefix(downloadURL, "//") {
downloadURL = "https:" + downloadURL
}
return core.Artifact{
URL: downloadURL,
Filename: path.Base(f.Basename),
Integrity: integrity,
Size: f.Size,
Metadata: map[string]any{
"subdir": f.Attrs.Subdir,
"build": f.Attrs.Build,
"build_number": f.Attrs.BuildNumber,
"arch": f.Attrs.Arch,
"platform": f.Attrs.Platform,
},
}, true
}

func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) {
channel, pkgName := parsePackageName(name)
if channel == "" {
Expand Down Expand Up @@ -270,12 +313,13 @@ func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Ma
return nil, err
}

if resp.Owner == "" {
if resp.Owner.Login == "" {
return nil, nil
}

return []core.Maintainer{{
Login: resp.Owner,
Login: resp.Owner.Login,
Name: resp.Owner.Name,
}}, nil
}

Expand Down
Loading
Loading