Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5)
| Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` |
| RubyGems | `pkg:gem/rails@7.1.0` |
| Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` |
| Swift | `pkg:swift/apple/swift-nio@2.65.0?repository_url=https://packages.example.com` |

## Direct Registry Usage

Expand Down Expand Up @@ -192,6 +193,9 @@ import _ "github.com/git-pkgs/registries/all"
| Homebrew | `brew` | https://formulae.brew.sh |
| Deno | `deno` | https://apiland.deno.dev |
| Terraform | `terraform` | https://registry.terraform.io |
| Swift | `swift` | URL required |

Swift package names are a registry scope and name, written `scope/name` (the PURL namespace and name) or `scope.name`. A host-style PURL namespace such as `pkg:swift/github.com/apple/swift-argument-parser` has no defined mapping to a registry scope, so the Swift client rejects it.

## Types

Expand Down
1 change: 1 addition & 0 deletions all/all.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@ import (
_ "github.com/git-pkgs/registries/internal/pub"
_ "github.com/git-pkgs/registries/internal/pypi"
_ "github.com/git-pkgs/registries/internal/rubygems"
_ "github.com/git-pkgs/registries/internal/swift"
_ "github.com/git-pkgs/registries/internal/terraform"
)
51 changes: 35 additions & 16 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,24 @@ func (c *Client) GetJSON(ctx context.Context, url string, v any) error {
return json.Unmarshal(body, v)
}

// GetJSONWithHeaders fetches a URL with extra request headers, which replace
// the defaults of the same name, decodes the JSON response into v and returns
// the response headers.
func (c *Client) GetJSONWithHeaders(ctx context.Context, url string, header http.Header, v any) (http.Header, error) {
body, respHeader, err := c.getBody(ctx, url, header)
if err != nil {
return nil, err
}
return respHeader, json.Unmarshal(body, v)
}

// GetBody fetches a URL and returns the response body.
func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {
body, _, err := c.getBody(ctx, url, nil)
return body, err
}

func (c *Client) getBody(ctx context.Context, url string, header http.Header) ([]byte, http.Header, error) {
var lastErr error

for attempt := 0; attempt <= c.MaxRetries; attempt++ {
Expand All @@ -73,64 +89,67 @@ func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {

select {
case <-ctx.Done():
return nil, ctx.Err()
return nil, nil, ctx.Err()
case <-time.After(delay):
}
}

if c.RateLimiter != nil {
if err := c.RateLimiter.Wait(ctx); err != nil {
return nil, err
return nil, nil, err
}
}

body, err := c.doRequest(ctx, url)
body, respHeader, err := c.doRequest(ctx, url, header)
if err == nil {
return body, nil
return body, respHeader, nil
}

lastErr = err
if errors.Is(err, ErrUpstreamHeld) {
return nil, err
return nil, nil, err
}
var rateErr *RateLimitError
if errors.As(err, &rateErr) {
return nil, err
return nil, nil, err
}

var httpErr *HTTPError
if ok := isHTTPError(err, &httpErr); ok {
if httpErr.StatusCode == http.StatusNotFound {
return nil, err
return nil, nil, err
}
if httpErr.StatusCode == http.StatusTooManyRequests || httpErr.StatusCode >= http.StatusInternalServerError {
continue
}
return nil, err
return nil, nil, err
}
}

return nil, lastErr
return nil, nil, lastErr
}

func (c *Client) doRequest(ctx context.Context, url string) ([]byte, error) {
func (c *Client) doRequest(ctx context.Context, url string, header http.Header) ([]byte, http.Header, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
return nil, nil, err
}

req.Header.Set("User-Agent", c.UserAgent)
req.Header.Set("Accept", "application/json")
for name, values := range header {
req.Header[http.CanonicalHeaderKey(name)] = values
}

resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, err
return nil, nil, err
}
defer func() { _ = resp.Body.Close() }()

body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseSize))
if err != nil {
return nil, err
return nil, nil, err
}

if resp.StatusCode >= http.StatusBadRequest {
Expand All @@ -145,13 +164,13 @@ func (c *Client) doRequest(ctx context.Context, url string) ([]byte, error) {
if delay%time.Second != 0 {
seconds++
}
return nil, &RateLimitError{RetryAfter: int(seconds)}
return nil, nil, &RateLimitError{RetryAfter: int(seconds)}
}
}
return nil, httpErr
return nil, nil, httpErr
}

return body, nil
return body, resp.Header, nil
}

func isHTTPError(err error, target **HTTPError) bool {
Expand Down
31 changes: 31 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,34 @@ func TestResponseWithinLimit(t *testing.T) {
t.Fatalf("got %q, want %q", string(body), payload)
}
}

func TestGetJSONWithHeaders(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Values("Accept"); len(got) != 1 || got[0] != "application/vnd.example+json" {
t.Errorf("Accept = %v, want only the supplied value", got)
}
if r.Header.Get("User-Agent") == "" {
t.Error("default User-Agent was dropped")
}
w.Header().Set("Link", `<https://example.com/?page=2>; rel="next"`)
_, _ = w.Write([]byte(`{"name":"test"}`))
}))
defer srv.Close()

c := DefaultClient()
c.MaxRetries = 0

var result struct {
Name string `json:"name"`
}
header, err := c.GetJSONWithHeaders(context.Background(), srv.URL, http.Header{"accept": {"application/vnd.example+json"}}, &result)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Name != "test" {
t.Errorf("got %+v", result)
}
if header.Get("Link") == "" {
t.Error("response headers not returned")
}
}
28 changes: 28 additions & 0 deletions fetch/swift_resolver_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package fetch

import (
"context"
"testing"

"github.com/git-pkgs/registries"
_ "github.com/git-pkgs/registries/internal/swift"
)

func TestResolveSwift(t *testing.T) {
reg, err := registries.New("swift", "https://packages.example.com", nil)
if err != nil {
t.Fatal(err)
}
r := NewResolver()
r.RegisterRegistry(reg)

for _, name := range []string{"apple/swift-nio", "apple.swift-nio"} {
info, err := r.Resolve(context.Background(), "swift", name, "2.65.0")
if err != nil {
t.Fatalf("Resolve(%q) failed: %v", name, err)
}
if want := "https://packages.example.com/apple/swift-nio/2.65.0.zip"; info.URL != want {
t.Errorf("URL = %q, want %q", info.URL, want)
}
}
}
Loading
Loading