Files
tiny-proxy/internal/github/handler_test.go
MRLokopandClaude Opus 5 1b8f2925bb Add per-upstream cache control
An upstream may now set `cache: false` to be proxied through without
being stored under the vrepo that fetched it. It is meant for chained
setups where the upstream is another vrepo of this proxy: without it
both layers keep their own copy of every artifact.

The decision has to travel with the response — a maven, debian or pacman
vrepo only learns which upstream answered once fallthrough resolves — so
proxycache fetch callbacks now return a Result{Resp, NoStore}. Reads are
unaffected: anything already cached under the vrepo is still served from
storage.

The UI's upstream column now shows each upstream's host and path, since
several vrepos routinely share a host (deb.debian.org/debian vs
/debian-security, chained vrepos of this proxy), and marks the ones that
are not cached.

Also includes an upstream retry transport for 429 responses.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-10 01:49:49 +03:00

524 lines
18 KiB
Go

package github
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"tiny-proxy/internal/config"
"tiny-proxy/internal/proxycache"
"tiny-proxy/internal/storage"
)
// fakeGitHub serves the API under apiPrefix (/api/v3 for GitHub/GHE,
// /api/v1 for Gitea) and asset downloads under
// /<owner>/<repo>/releases/download/<tag>/<asset>, counting requests and
// recording the last Accept header seen.
type fakeGitHub struct {
apiPrefix string // empty defaults to /api/v3
hits atomic.Int32
accept atomic.Value // string
}
func (f *fakeGitHub) api() string {
if f.apiPrefix == "" {
return "/api/v3"
}
return f.apiPrefix
}
const releaseJSON = `{
"tag_name": "v1.2.3",
"name": "First release",
"prerelease": false,
"assets": [
{"name": "app.zip", "size": 12, "browser_download_url": "https://github.com/o/r/releases/download/v1.2.3/app.zip"}
]
}`
func (f *fakeGitHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.hits.Add(1)
f.accept.Store(r.Header.Get("Accept"))
api := f.api()
switch {
case r.URL.Path == api+"/repos/o/r/releases/latest":
fmt.Fprint(w, releaseJSON)
case r.URL.Path == api+"/repos/o/loop/releases/latest":
// A repo whose latest stable tag is literally "latest".
fmt.Fprint(w, `{"tag_name": "latest", "assets": []}`)
case r.URL.Path == api+"/repos/o/r/releases":
fmt.Fprint(w, `[
{"tag_name": "v1.2.3", "prerelease": false},
{"tag_name": "v2.0.0-rc1", "prerelease": true}
]`)
case r.URL.Path == api+"/repos/empty/rel/releases":
fmt.Fprint(w, `[{"tag_name": "v1.0.0", "prerelease": false}]`)
case r.URL.Path == api+"/repos/o/drafty/releases":
// Page 1 is full (100 drafts) to force pagination; page 2 has the
// first real prerelease.
if r.URL.Query().Get("page") == "2" {
fmt.Fprint(w, `[{"tag_name": "v3.0.0-beta", "prerelease": true}]`)
return
}
rels := make([]map[string]any, 0, 100)
for i := 0; i < 100; i++ {
rels = append(rels, map[string]any{
"tag_name": fmt.Sprintf("draft-%d", i), "prerelease": true, "draft": true,
})
}
json.NewEncoder(w).Encode(rels)
case strings.HasPrefix(r.URL.Path, api+"/repos/o/loop/releases/tags/latest"):
fmt.Fprint(w, `{"tag_name": "latest", "assets": []}`)
case strings.HasPrefix(r.URL.Path, api+"/repos/o/r/releases/tags/"):
// Only real tags resolve; anything else (e.g. the "v1.2.3/checksums"
// probe from a .json-asset request) is a 404.
tag := strings.TrimPrefix(r.URL.Path, api+"/repos/o/r/releases/tags/")
if tag == "v1.2.3" || tag == "release/1.0" {
fmt.Fprint(w, releaseJSON)
} else {
http.NotFound(w, r)
}
case strings.HasPrefix(r.URL.Path, "/o/r/releases/download/"):
w.Header().Set("Content-Type", "application/zip")
fmt.Fprint(w, "zip-contents")
case strings.HasPrefix(r.URL.Path, "/o/loop/releases/download/"):
fmt.Fprint(w, "loop-asset")
default:
http.NotFound(w, r)
}
}
func newHandler(t *testing.T, upstream http.Handler, vrepos ...config.VRepo) *Handler {
t.Helper()
h, _ := newHandlerStore(t, upstream, vrepos...)
return h
}
func newHandlerStore(t *testing.T, upstream http.Handler, vrepos ...config.VRepo) (*Handler, storage.Store) {
t.Helper()
srv := httptest.NewServer(upstream)
t.Cleanup(srv.Close)
store, err := storage.NewLocalFS(t.TempDir())
if err != nil {
t.Fatal(err)
}
clients := map[string]*http.Client{}
for i := range vrepos {
if vrepos[i].Upstream.URL == "" {
vrepos[i].Upstream.URL = srv.URL
}
if vrepos[i].Cache.MetadataTTL == 0 {
vrepos[i].Cache.MetadataTTL = time.Minute
}
if vrepos[i].Auth.Mode == "" {
vrepos[i].Auth.Mode = config.AuthPublic
}
clients[vrepos[i].Name] = srv.Client()
}
return NewHandler(vrepos, clients, proxycache.New(store)), store
}
func do(t *testing.T, h *Handler, method, path string) *httptest.ResponseRecorder {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest(method, path, nil)
h.ServeHTTP(rec, req)
return rec
}
func TestLatestRedirect(t *testing.T) {
gh := &fakeGitHub{}
h := newHandler(t, gh, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest")
if rec.Code != http.StatusFound {
t.Fatalf("code = %d, want 302", rec.Code)
}
if loc := rec.Header().Get("Location"); loc != "/github/gh/o/r/v1.2.3" {
t.Errorf("Location = %q", loc)
}
// Alias resolution is cached: a second request must not hit upstream.
do(t, h, http.MethodGet, "/github/gh/o/r/latest")
if got := gh.hits.Load(); got != 1 {
t.Errorf("upstream hits = %d, want 1", got)
}
}
func TestBareTagRedirectsToJSON(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3")
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/github/gh/o/r/v1.2.3.json" {
t.Errorf("code = %d Location = %q", rec.Code, rec.Header().Get("Location"))
}
}
func TestSnapshotPicksPrerelease(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/snapshot")
if rec.Header().Get("Location") != "/github/gh/o/r/v2.0.0-rc1" {
t.Errorf("Location = %q, want snapshot tag", rec.Header().Get("Location"))
}
}
func TestSnapshotNone(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/empty/rel/snapshot")
if rec.Code != http.StatusNotFound {
t.Errorf("code = %d, want 404", rec.Code)
}
}
func TestMetadataRewriteAndCache(t *testing.T) {
gh := &fakeGitHub{}
h := newHandler(t, gh, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest.json")
if rec.Code != http.StatusOK {
t.Fatalf("code = %d", rec.Code)
}
var rel map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &rel); err != nil {
t.Fatal(err)
}
if rel["tag_name"] != "v1.2.3" {
t.Errorf("tag_name = %v", rel["tag_name"])
}
asset := rel["assets"].([]any)[0].(map[string]any)
if got := asset["browser_download_url"]; got != "/github/gh/o/r/v1.2.3/app.zip" {
t.Errorf("browser_download_url = %v", got)
}
// latest.json hit releases/latest + releases/tags/<tag> = 2 upstream calls.
if got := gh.hits.Load(); got != 2 {
t.Fatalf("upstream hits = %d, want 2", got)
}
// The concrete tag URL is already cached: no further upstream calls.
rec = do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3.json")
if rec.Code != http.StatusOK || gh.hits.Load() != 2 {
t.Errorf("tag.json: code = %d hits = %d, want cached", rec.Code, gh.hits.Load())
}
}
func TestAssetAliasRedirectAndDownload(t *testing.T) {
gh := &fakeGitHub{}
h := newHandler(t, gh, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest/app.zip")
if rec.Code != http.StatusFound ||
rec.Header().Get("Location") != "/github/gh/o/r/v1.2.3/app.zip" {
t.Fatalf("code = %d Location = %q", rec.Code, rec.Header().Get("Location"))
}
rec = do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3/app.zip")
if rec.Code != http.StatusOK {
t.Fatalf("download code = %d", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "zip-contents" {
t.Errorf("body = %q", body)
}
hits := gh.hits.Load()
do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3/app.zip")
if gh.hits.Load() != hits {
t.Error("second asset request should be served from cache")
}
}
func TestAllowFilter(t *testing.T) {
v := config.VRepo{Name: "gh", Type: config.VRepoGithub, Allow: []string{"myorg/*", "foo/bar"}}
if !allowed(v.Allow, "myorg", "tool") || !allowed(v.Allow, "foo", "bar") {
t.Error("expected allowed")
}
if allowed(v.Allow, "evil", "repo") || allowed(v.Allow, "myorg", "a/b") {
t.Error("expected denied")
}
if !allowed(nil, "any", "thing") {
t.Error("empty allow list should allow everything")
}
h := newHandler(t, &fakeGitHub{}, v)
if rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest"); rec.Code != http.StatusNotFound {
t.Errorf("denied repo: code = %d, want 404", rec.Code)
}
}
func TestSlashTag(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/release/1.0.json")
if rec.Code != http.StatusOK {
t.Fatalf("code = %d", rec.Code)
}
// Asset form splits at the last slash: tag "release/1.0", asset "app.zip".
rec = do(t, h, http.MethodGet, "/github/gh/o/r/release/1.0/app.zip")
if rec.Code != http.StatusOK {
t.Errorf("asset with slash tag: code = %d", rec.Code)
}
}
func TestNoRedirectVRepo(t *testing.T) {
off := false
v := config.VRepo{Name: "gh", Type: config.VRepoGithub, Redirect: &off}
h := newHandler(t, &fakeGitHub{}, v)
rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest")
if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/json" {
t.Errorf("bare alias: code = %d ct = %q, want metadata in place", rec.Code, rec.Header().Get("Content-Type"))
}
rec = do(t, h, http.MethodGet, "/github/gh/o/r/latest/app.zip")
if rec.Code != http.StatusOK {
t.Errorf("alias asset: code = %d, want 200 in place", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "zip-contents" {
t.Errorf("body = %q", body)
}
}
func TestAuthUsersEnforced(t *testing.T) {
v := config.VRepo{
Name: "gh", Type: config.VRepoGithub,
Auth: config.AuthConfig{Mode: config.AuthUsers, Users: map[string]string{"alice": "pw"}},
}
h := newHandler(t, &fakeGitHub{}, v)
if rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest"); rec.Code != http.StatusUnauthorized {
t.Errorf("no creds: code = %d, want 401", rec.Code)
}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/github/gh/o/r/latest", nil)
req.SetBasicAuth("alice", "pw")
h.ServeHTTP(rec, req)
if rec.Code != http.StatusFound {
t.Errorf("with creds: code = %d, want 302", rec.Code)
}
}
func TestMethodNotAllowed(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
if rec := do(t, h, http.MethodPost, "/github/gh/o/r/latest"); rec.Code != http.StatusMethodNotAllowed {
t.Errorf("code = %d, want 405", rec.Code)
}
}
func TestJSONAssetFallback(t *testing.T) {
gh := &fakeGitHub{}
h := newHandler(t, gh, config.VRepo{Name: "gh", Type: config.VRepoGithub})
// checksums.json is an asset, not a tag: metadata probe 404s, then the
// asset form is served.
rec := do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3/checksums.json")
if rec.Code != http.StatusOK {
t.Fatalf("code = %d", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "zip-contents" {
t.Errorf("body = %q", body)
}
hits := gh.hits.Load() // metadata probe + asset download
// Once cached, the asset is served without any upstream call.
rec = do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3/checksums.json")
if rec.Code != http.StatusOK || gh.hits.Load() != hits {
t.Errorf("cached .json asset: code = %d, hits %d -> %d", rec.Code, hits, gh.hits.Load())
}
}
func TestSlashTagMetadataStillWins(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/release/1.0.json")
if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/json" {
t.Errorf("code = %d ct = %q, want metadata", rec.Code, rec.Header().Get("Content-Type"))
}
}
func TestTagNamedLatestServedInPlace(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
// Must not 302 onto itself.
rec := do(t, h, http.MethodGet, "/github/gh/o/loop/latest")
if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/json" {
t.Errorf("bare latest: code = %d ct = %q", rec.Code, rec.Header().Get("Content-Type"))
}
rec = do(t, h, http.MethodGet, "/github/gh/o/loop/latest.json")
if rec.Code != http.StatusOK {
t.Errorf("latest.json: code = %d", rec.Code)
}
rec = do(t, h, http.MethodGet, "/github/gh/o/loop/latest/blob.bin")
if rec.Code != http.StatusOK {
t.Errorf("latest/blob.bin: code = %d", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "loop-asset" {
t.Errorf("body = %q", body)
}
}
func TestTagNamedRefs(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
// A tag named "refs" must not collide with alias-resolution keys.
rec := do(t, h, http.MethodGet, "/github/gh/o/r/refs/app.zip")
if rec.Code != http.StatusOK {
t.Fatalf("refs/app.zip: code = %d", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "zip-contents" {
t.Errorf("body = %q", body)
}
rec = do(t, h, http.MethodGet, "/github/gh/o/r/latest")
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/github/gh/o/r/v1.2.3" {
t.Errorf("alias after refs asset: code = %d Location = %q", rec.Code, rec.Header().Get("Location"))
}
}
func TestSnapshotPaginationAndDrafts(t *testing.T) {
h := newHandler(t, &fakeGitHub{}, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/drafty/snapshot")
if rec.Code != http.StatusFound ||
rec.Header().Get("Location") != "/github/gh/o/drafty/v3.0.0-beta" {
t.Errorf("code = %d Location = %q, want first non-draft prerelease", rec.Code, rec.Header().Get("Location"))
}
}
func TestMetadataEscapesSpecialChars(t *testing.T) {
gh := &fakeGitHub{}
h := newHandler(t, gh, config.VRepo{Name: "gh", Type: config.VRepoGithub})
rec := do(t, h, http.MethodGet, "/github/gh/o/r/latest.json")
if rec.Code != http.StatusOK {
t.Fatalf("code = %d", rec.Code)
}
if strings.Contains(rec.Body.String(), "github.com/o/r/releases") {
t.Error("metadata still contains upstream download URLs")
}
}
// --- gitea type: same grammar, /gitea/ prefix, /api/v1 API base ----------
func TestGiteaEndToEnd(t *testing.T) {
gt := &fakeGitHub{apiPrefix: "/api/v1"}
h, store := newHandlerStore(t, gt, config.VRepo{Name: "gt", Type: config.VRepoGitea})
// Alias redirect carries the /gitea/ prefix.
rec := do(t, h, http.MethodGet, "/gitea/gt/o/r/latest")
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/gitea/gt/o/r/v1.2.3" {
t.Fatalf("latest: code = %d Location = %q", rec.Code, rec.Header().Get("Location"))
}
// Metadata rewrites asset URLs into the gitea namespace.
rec = do(t, h, http.MethodGet, "/gitea/gt/o/r/latest.json")
if rec.Code != http.StatusOK {
t.Fatalf("latest.json: code = %d", rec.Code)
}
var rel map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &rel); err != nil {
t.Fatal(err)
}
asset := rel["assets"].([]any)[0].(map[string]any)
if got := asset["browser_download_url"]; got != "/gitea/gt/o/r/v1.2.3/app.zip" {
t.Errorf("browser_download_url = %v", got)
}
// Asset download works and is cached under the gitea namespace.
rec = do(t, h, http.MethodGet, "/gitea/gt/o/r/v1.2.3/app.zip")
if rec.Code != http.StatusOK {
t.Fatalf("asset: code = %d", rec.Code)
}
body, _ := io.ReadAll(rec.Result().Body)
if string(body) != "zip-contents" {
t.Errorf("asset body = %q", body)
}
// Snapshot alias resolves via the releases list.
rec = do(t, h, http.MethodGet, "/gitea/gt/o/r/snapshot")
if rec.Header().Get("Location") != "/gitea/gt/o/r/v2.0.0-rc1" {
t.Errorf("snapshot Location = %q", rec.Header().Get("Location"))
}
// API requests carry the gitea Accept header.
if got := gt.accept.Load(); got != "application/json" {
t.Errorf("Accept = %v, want application/json", got)
}
// Cache keys live in gitea/ and gitea-refs/, never github/.
objs, err := store.List(t.Context(), "")
if err != nil {
t.Fatal(err)
}
var metaKey, refsKey, assetKeyFound bool
for _, o := range objs {
if strings.HasPrefix(o.Key, "github/") || strings.HasPrefix(o.Key, "github-refs/") {
t.Errorf("gitea vrepo wrote github-namespace key %q", o.Key)
}
switch o.Key {
case "gitea/gt/o/r/v1.2.3.json":
metaKey = true
case "gitea-refs/gt/o/r/latest":
refsKey = true
case "gitea/gt/o/r/v1.2.3/app.zip":
assetKeyFound = true
}
}
if !metaKey || !refsKey || !assetKeyFound {
t.Errorf("missing cache keys: meta=%v refs=%v asset=%v (keys: %v)", metaKey, refsKey, assetKeyFound, objs)
}
}
func TestGiteaUsesAPIv1(t *testing.T) {
gt := &fakeGitHub{apiPrefix: "/api/v1"}
h := newHandler(t, gt, config.VRepo{Name: "gt", Type: config.VRepoGitea})
// The fake only answers under /api/v1; a /api/v3 request would 404.
rec := do(t, h, http.MethodGet, "/gitea/gt/o/r/latest")
if rec.Code != http.StatusFound {
t.Fatalf("code = %d, want 302 (API base must be <web>/api/v1)", rec.Code)
}
}
func TestCrossTypeNamespaceGuard(t *testing.T) {
gt := &fakeGitHub{apiPrefix: "/api/v1"}
h := newHandler(t, gt, config.VRepo{Name: "gt", Type: config.VRepoGitea})
// A /github/ URL naming a gitea vrepo (and vice versa) must not resolve.
if rec := do(t, h, http.MethodGet, "/github/gt/o/r/latest"); rec.Code != http.StatusNotFound {
t.Errorf("/github/ URL for gitea vrepo: code = %d, want 404", rec.Code)
}
if rec := do(t, h, http.MethodGet, "/gitea/gt/o/r/latest"); rec.Code != http.StatusFound {
t.Errorf("/gitea/ URL for gitea vrepo: code = %d, want 302", rec.Code)
}
if gt.hits.Load() != 1 {
t.Errorf("upstream hits = %d, want 1 (only the well-formed request)", gt.hits.Load())
}
}
// TestUpstreamCacheDisabled: with upstream cache: false nothing is stored —
// release metadata, resolved aliases and assets all go straight through.
func TestUpstreamCacheDisabled(t *testing.T) {
gh := &fakeGitHub{}
no := false
h, store := newHandlerStore(t, gh, config.VRepo{
Name: "gh",
Type: config.VRepoGithub,
Upstream: config.UpstreamConf{Cache: &no},
})
for i := 0; i < 2; i++ {
if rec := do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3.json"); rec.Code != http.StatusOK {
t.Fatalf("metadata request %d: code = %d", i, rec.Code)
}
rec := do(t, h, http.MethodGet, "/github/gh/o/r/v1.2.3/app.zip")
if rec.Code != http.StatusOK || rec.Body.String() != "zip-contents" {
t.Fatalf("asset request %d: code = %d body = %q", i, rec.Code, rec.Body.String())
}
}
if got := gh.hits.Load(); got != 4 {
t.Errorf("upstream hits = %d, want 4 (2 metadata + 2 asset, none cached)", got)
}
for _, key := range []string{"github/gh/o/r/v1.2.3.json", "github/gh/o/r/v1.2.3/app.zip"} {
if _, err := store.Stat(t.Context(), key); err != storage.ErrNotFound {
t.Errorf("%s was stored despite cache: false", key)
}
}
}