MRLokopandClaude Opus 5.5
dbf3cbae27
A home proxy that chains to a remote tiny-proxy and falls back to the real upstreams needs two things it did not have. 502, 503 and 504 now fall through to the next upstream in the maven, debian, pacman and docker walks. They are what an ingress answers when a chained tiny-proxy is down, and what tiny-proxy itself answers (502 upstream error) when its own upstream fails — either way the object is not bad, the path to it is. A 500 is still the upstream's own answer and is returned as-is, as is anything the last upstream says, and a throttle seen earlier still turns a trailing 404 into a 429. Docker vrepos take `upstreams:` like the others. Each entry brings its own client, cache flag and retry budget, and a blob resume goes back to the upstream that served it. An upstream URL whose path starts with /v2/ is a repository root — another tiny-proxy's vrepo — so the image path is appended as-is rather than getting a second /v2/. The library/ prefix is now decided by the upstream's host instead of a substring of its whole URL, so a chained proxy is not handed library/ twice. When nothing answers at all the last transport error is returned, so a single unreachable registry still reads as 502 rather than as a missing image. github and gitea stay single-upstream: they speak a REST API, not a path-for-path mirror. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
422 lines
14 KiB
Go
422 lines
14 KiB
Go
package distro
|
|
|
|
import (
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"tiny-proxy/internal/config"
|
|
"tiny-proxy/internal/proxycache"
|
|
"tiny-proxy/internal/storage"
|
|
)
|
|
|
|
func TestIsMutableDebian(t *testing.T) {
|
|
mutable := []string{
|
|
"dists/bookworm/Release",
|
|
"dists/bookworm/InRelease",
|
|
"dists/bookworm/main/binary-amd64/Packages.gz",
|
|
"dists/bookworm/main/Contents-amd64.gz",
|
|
"dists/bookworm/main/i18n/Translation-en.xz",
|
|
"dists/bookworm/main/source/Sources.xz",
|
|
"Release", // flat repo, no dists/
|
|
"Packages", // flat repo
|
|
"unknown/x", // fail-safe default
|
|
}
|
|
for _, p := range mutable {
|
|
if !isMutableDebian(p) {
|
|
t.Errorf("isMutableDebian(%q) = false, want true", p)
|
|
}
|
|
}
|
|
immutable := []string{
|
|
"dists/bookworm/main/binary-amd64/by-hash/SHA256/0123456789abcdef",
|
|
"pool/main/h/hello/hello_1.0-1_amd64.deb",
|
|
"pool/non-free/f/foo/foo_2.0.dsc",
|
|
"debian/pool/main/h/hello/hello_1.0-1_amd64.deb",
|
|
}
|
|
for _, p := range immutable {
|
|
if isMutableDebian(p) {
|
|
t.Errorf("isMutableDebian(%q) = true, want false", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsMutablePacman(t *testing.T) {
|
|
mutable := []string{
|
|
"core/os/x86_64/core.db",
|
|
"core/os/x86_64/core.db.sig",
|
|
"core/os/x86_64/core.db.tar.zst",
|
|
"extra/os/x86_64/extra.files",
|
|
"extra/os/x86_64/extra.files.tar.gz",
|
|
"multilib/os/x86_64/lastupdate",
|
|
}
|
|
for _, p := range mutable {
|
|
if !isMutablePacman(p) {
|
|
t.Errorf("isMutablePacman(%q) = false, want true", p)
|
|
}
|
|
}
|
|
immutable := []string{
|
|
"core/os/x86_64/linux-6.10.2-1-x86_64.pkg.tar.zst",
|
|
"core/os/x86_64/linux-6.10.2-1-x86_64.pkg.tar.zst.sig",
|
|
"extra/os/x86_64/foo-1.0-1-any.pkg.tar.xz",
|
|
"core/os/x86_64/some.db.bak",
|
|
}
|
|
for _, p := range immutable {
|
|
if isMutablePacman(p) {
|
|
t.Errorf("isMutablePacman(%q) = true, want false", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
// stubUpstream counts requests and serves canned responses.
|
|
type stubUpstream struct {
|
|
srv *httptest.Server
|
|
hits atomic.Int32
|
|
}
|
|
|
|
func newStubUpstream(t *testing.T, h http.HandlerFunc) *stubUpstream {
|
|
t.Helper()
|
|
s := &stubUpstream{}
|
|
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
s.hits.Add(1)
|
|
h(w, r)
|
|
}))
|
|
t.Cleanup(s.srv.Close)
|
|
return s
|
|
}
|
|
|
|
func (s *stubUpstream) conf() config.UpstreamConf { return config.UpstreamConf{URL: s.srv.URL} }
|
|
|
|
func newTestHandler(t *testing.T, spec Spec, vrepo string, ups ...config.UpstreamConf) *Handler {
|
|
t.Helper()
|
|
store, err := storage.NewLocalFS(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repoCfg := config.VRepo{
|
|
Name: vrepo,
|
|
Type: spec.Type,
|
|
Upstreams: ups,
|
|
Auth: config.AuthConfig{Mode: config.AuthPublic},
|
|
}
|
|
clients := map[string][]*http.Client{vrepo: {}}
|
|
for range ups {
|
|
clients[vrepo] = append(clients[vrepo], http.DefaultClient)
|
|
}
|
|
return NewHandler(spec, []config.VRepo{repoCfg}, clients, proxycache.New(store))
|
|
}
|
|
|
|
func get(t *testing.T, h *Handler, urlPath string) (int, string) {
|
|
t.Helper()
|
|
return do(t, h, http.MethodGet, urlPath, "")
|
|
}
|
|
|
|
func do(t *testing.T, h *Handler, method, urlPath, authHeader string) (int, string) {
|
|
t.Helper()
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest(method, urlPath, nil)
|
|
if authHeader != "" {
|
|
req.Header.Set("Authorization", authHeader)
|
|
}
|
|
h.ServeHTTP(rec, req)
|
|
res := rec.Result()
|
|
defer res.Body.Close()
|
|
body, _ := io.ReadAll(res.Body)
|
|
return res.StatusCode, string(body)
|
|
}
|
|
|
|
func TestImmutableCachedForever(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("deb-bytes"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", up.conf())
|
|
|
|
for i := 0; i < 3; i++ {
|
|
code, body := get(t, h, "/debian/deb/pool/main/h/hello/hello_1.0_amd64.deb")
|
|
if code != http.StatusOK || body != "deb-bytes" {
|
|
t.Fatalf("got %d %q, want 200 deb-bytes", code, body)
|
|
}
|
|
}
|
|
if up.hits.Load() != 1 {
|
|
t.Errorf("upstream hits = %d after 3 immutable gets, want 1", up.hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestMutableRefetchedAfterTTL(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("release"))
|
|
})
|
|
store, err := storage.NewLocalFS(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repoCfg := config.VRepo{
|
|
Name: "deb",
|
|
Type: config.VRepoDebian,
|
|
Upstreams: []config.UpstreamConf{up.conf()},
|
|
Auth: config.AuthConfig{Mode: config.AuthPublic},
|
|
Cache: config.CacheConfig{MetadataTTL: 20 * time.Millisecond},
|
|
}
|
|
h := NewHandler(Debian, []config.VRepo{repoCfg},
|
|
map[string][]*http.Client{"deb": {http.DefaultClient}}, proxycache.New(store))
|
|
|
|
if code, _ := get(t, h, "/debian/deb/dists/bookworm/Release"); code != http.StatusOK {
|
|
t.Fatalf("first get = %d, want 200", code)
|
|
}
|
|
if code, _ := get(t, h, "/debian/deb/dists/bookworm/Release"); code != http.StatusOK {
|
|
t.Fatalf("second get = %d, want 200", code)
|
|
}
|
|
if up.hits.Load() != 1 {
|
|
t.Fatalf("upstream hits = %d within TTL, want 1", up.hits.Load())
|
|
}
|
|
time.Sleep(30 * time.Millisecond)
|
|
if code, _ := get(t, h, "/debian/deb/dists/bookworm/Release"); code != http.StatusOK {
|
|
t.Fatalf("get after TTL = %d, want 200", code)
|
|
}
|
|
if up.hits.Load() != 2 {
|
|
t.Errorf("upstream hits = %d after TTL expiry, want 2", up.hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestMultiUpstreamFallback(t *testing.T) {
|
|
notFound := newStubUpstream(t, http.NotFound)
|
|
forbidden := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "forbidden", http.StatusForbidden)
|
|
})
|
|
serving := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("pkg"))
|
|
})
|
|
h := newTestHandler(t, Pacman, "arch", notFound.conf(), forbidden.conf(), serving.conf())
|
|
|
|
code, body := get(t, h, "/pacman/arch/core/os/x86_64/linux-6.10-1-x86_64.pkg.tar.zst")
|
|
if code != http.StatusOK || body != "pkg" {
|
|
t.Fatalf("got %d %q, want 200 pkg", code, body)
|
|
}
|
|
if notFound.hits.Load() != 1 || forbidden.hits.Load() != 1 || serving.hits.Load() != 1 {
|
|
t.Errorf("hits = %d/%d/%d, want 1/1/1",
|
|
notFound.hits.Load(), forbidden.hits.Load(), serving.hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestTransportErrorFallsThrough(t *testing.T) {
|
|
dead := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {})
|
|
dead.srv.Close() // connections now fail
|
|
serving := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("pkg"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", dead.conf(), serving.conf())
|
|
|
|
if code, body := get(t, h, "/debian/deb/pool/main/h/h/h_1.deb"); code != http.StatusOK || body != "pkg" {
|
|
t.Fatalf("got %d %q, want 200 pkg", code, body)
|
|
}
|
|
}
|
|
|
|
func TestAllMissSynthesized404Uncached(t *testing.T) {
|
|
up := newStubUpstream(t, http.NotFound)
|
|
h := newTestHandler(t, Pacman, "arch", up.conf())
|
|
|
|
for i := 0; i < 2; i++ {
|
|
if code, _ := get(t, h, "/pacman/arch/core/os/x86_64/missing.pkg.tar.zst"); code != http.StatusNotFound {
|
|
t.Fatalf("got %d, want 404", code)
|
|
}
|
|
}
|
|
if up.hits.Load() != 2 {
|
|
t.Errorf("upstream hits = %d, want 2 (404s must not be cached)", up.hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestServerErrorDoesNotFallThrough(t *testing.T) {
|
|
broken := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "boom", http.StatusInternalServerError)
|
|
})
|
|
second := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("ok"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", broken.conf(), second.conf())
|
|
|
|
if code, _ := get(t, h, "/debian/deb/dists/bookworm/Release"); code != http.StatusInternalServerError {
|
|
t.Fatalf("got %d, want 500 (non-404 statuses are authoritative)", code)
|
|
}
|
|
if second.hits.Load() != 0 {
|
|
t.Errorf("second upstream consulted %d times after 500, want 0", second.hits.Load())
|
|
}
|
|
}
|
|
|
|
// Gateway errors mean the mirror is down or could not reach its own
|
|
// upstream, so they fall through like a miss.
|
|
func TestFallbackOnGatewayErrors(t *testing.T) {
|
|
for _, status := range []int{http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
|
|
t.Run(strconv.Itoa(status), func(t *testing.T) {
|
|
down := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "down", status)
|
|
})
|
|
serving := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("deb-bytes"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", down.conf(), serving.conf())
|
|
|
|
code, body := get(t, h, "/debian/deb/pool/main/h/hello/hello_1.0_amd64.deb")
|
|
if code != http.StatusOK || body != "deb-bytes" {
|
|
t.Fatalf("got %d %q, want 200 deb-bytes", code, body)
|
|
}
|
|
if down.hits.Load() != 1 || serving.hits.Load() != 1 {
|
|
t.Errorf("hits: down=%d serving=%d, want 1 and 1", down.hits.Load(), serving.hits.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGatewayErrorOnLastMirrorReturnedAsIs(t *testing.T) {
|
|
missing := newStubUpstream(t, http.NotFound)
|
|
down := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "gateway timeout", http.StatusGatewayTimeout)
|
|
})
|
|
h := newTestHandler(t, Pacman, "arch", missing.conf(), down.conf())
|
|
|
|
if code, _ := get(t, h, "/pacman/arch/core/os/x86_64/linux-6.10-1-x86_64.pkg.tar.zst"); code != http.StatusGatewayTimeout {
|
|
t.Fatalf("got %d, want 504 from last mirror", code)
|
|
}
|
|
}
|
|
|
|
func TestAuthEnforced(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("ok"))
|
|
})
|
|
store, err := storage.NewLocalFS(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repoCfg := config.VRepo{
|
|
Name: "deb",
|
|
Type: config.VRepoDebian,
|
|
Upstreams: []config.UpstreamConf{up.conf()},
|
|
Auth: config.AuthConfig{
|
|
Mode: config.AuthUsers,
|
|
Users: map[string]string{"alice": "pw1"},
|
|
},
|
|
}
|
|
h := NewHandler(Debian, []config.VRepo{repoCfg},
|
|
map[string][]*http.Client{"deb": {http.DefaultClient}}, proxycache.New(store))
|
|
|
|
if code, _ := get(t, h, "/debian/deb/dists/bookworm/Release"); code != http.StatusUnauthorized {
|
|
t.Fatalf("no credentials = %d, want 401", code)
|
|
}
|
|
req := httptest.NewRequest(http.MethodGet, "/debian/deb/dists/bookworm/Release", nil)
|
|
req.SetBasicAuth("alice", "pw1")
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("with credentials = %d, want 200", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestPathGuard(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("ok"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", up.conf())
|
|
|
|
bad := []string{
|
|
"/debian/deb/../etc/passwd",
|
|
"/debian/deb/a/./b",
|
|
"/debian/deb/a//b",
|
|
"/debian/deb/a\\b",
|
|
"/debian/deb/",
|
|
}
|
|
for _, p := range bad {
|
|
if code, _ := get(t, h, p); code != http.StatusBadRequest {
|
|
t.Errorf("GET %q = %d, want 400", p, code)
|
|
}
|
|
}
|
|
if up.hits.Load() != 0 {
|
|
t.Errorf("upstream consulted %d times for bad paths, want 0", up.hits.Load())
|
|
}
|
|
}
|
|
|
|
func TestMethodNotAllowed(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {})
|
|
h := newTestHandler(t, Debian, "deb", up.conf())
|
|
if code, _ := do(t, h, http.MethodPost, "/debian/deb/dists/bookworm/Release", ""); code != http.StatusMethodNotAllowed {
|
|
t.Fatalf("POST = %d, want 405", code)
|
|
}
|
|
}
|
|
|
|
func TestHeadFromCache(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("deb-bytes"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", up.conf())
|
|
|
|
if code, _ := get(t, h, "/debian/deb/pool/main/h/h/h_1.deb"); code != http.StatusOK {
|
|
t.Fatalf("get = %d, want 200", code)
|
|
}
|
|
code, body := do(t, h, http.MethodHead, "/debian/deb/pool/main/h/h/h_1.deb", "")
|
|
if code != http.StatusOK || body != "" {
|
|
t.Fatalf("HEAD = %d %q, want 200 with empty body", code, body)
|
|
}
|
|
if up.hits.Load() != 1 {
|
|
t.Errorf("upstream hits = %d, want 1 (HEAD served from cache)", up.hits.Load())
|
|
}
|
|
}
|
|
|
|
// TestUpstreamCacheDisabled: with `cache: false` even an immutable path
|
|
// (pool/**, normally cached forever) is proxied through on every request.
|
|
func TestUpstreamCacheDisabled(t *testing.T) {
|
|
up := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("deb-bytes"))
|
|
})
|
|
no := false
|
|
conf := up.conf()
|
|
conf.Cache = &no
|
|
h := newTestHandler(t, Debian, "deb", conf)
|
|
|
|
for i := 0; i < 3; i++ {
|
|
code, body := get(t, h, "/debian/deb/pool/main/h/hello/hello_1.0_amd64.deb")
|
|
if code != http.StatusOK || body != "deb-bytes" {
|
|
t.Fatalf("request %d: got %d %q, want 200 deb-bytes", i, code, body)
|
|
}
|
|
}
|
|
if got := up.hits.Load(); got != 3 {
|
|
t.Errorf("upstream hits = %d after 3 gets, want 3 (nothing cached)", got)
|
|
}
|
|
}
|
|
|
|
func TestMirrorFallbackOn429(t *testing.T) {
|
|
limited := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "slow down", http.StatusTooManyRequests)
|
|
})
|
|
last := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte("deb-bytes"))
|
|
})
|
|
h := newTestHandler(t, Debian, "deb", limited.conf(), last.conf())
|
|
|
|
code, body := get(t, h, "/debian/deb/pool/main/h/hello/hello_1.0_amd64.deb")
|
|
if code != http.StatusOK || body != "deb-bytes" {
|
|
t.Fatalf("got %d %q, want 200 deb-bytes", code, body)
|
|
}
|
|
}
|
|
|
|
// A 404 after a rate-limited mirror must not be reported as "missing".
|
|
func TestRateLimitedThen404Returns429(t *testing.T) {
|
|
limited := newStubUpstream(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Retry-After", "5")
|
|
http.Error(w, "slow down", http.StatusTooManyRequests)
|
|
})
|
|
missing := newStubUpstream(t, http.NotFound)
|
|
h := newTestHandler(t, Debian, "deb", limited.conf(), missing.conf())
|
|
|
|
rec := httptest.NewRecorder()
|
|
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/debian/deb/pool/main/h/hello/hello_1.0_amd64.deb", nil))
|
|
res := rec.Result()
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusTooManyRequests {
|
|
t.Fatalf("got %d, want 429", res.StatusCode)
|
|
}
|
|
if got := res.Header.Get("Retry-After"); got != "5" {
|
|
t.Errorf("Retry-After = %q, want 5 carried over from mirror", got)
|
|
}
|
|
}
|