diff --git a/README.md b/README.md index 792c76b..ac0d21c 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,18 @@ Add to your `~/.m2/settings.xml`: ``` +The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found. + +For Gradle plugin resolution via the same proxy endpoint: + +```kotlin +pluginManagement { + repositories { + maven(url = "http://localhost:8080/maven/") + } +} +``` + ### Gradle HTTP Build Cache Configure in `settings.gradle(.kts)`: @@ -386,6 +398,7 @@ sudo dnf update ## Configuration The proxy can be configured via: + 1. Command line flags (highest priority) 2. Environment variables 3. Configuration file (YAML or JSON) @@ -958,6 +971,7 @@ The proxy will recreate the database on next start. ## Building from Source Requirements: + - Go 1.25 or later ```bash diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 946d12a..57ecfed 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -72,6 +72,8 @@ // PROXY_DATABASE_URL - PostgreSQL connection URL // PROXY_LOG_LEVEL - Log level // PROXY_LOG_FORMAT - Log format +// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL +// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL // PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads // PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size // PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction @@ -198,6 +200,8 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n") fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n") fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n") diff --git a/config.example.yaml b/config.example.yaml index 4505849..fdab480 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -71,6 +71,12 @@ upstream: # npm registry URL npm: "https://registry.npmjs.org" + # Maven repository URL (used by /maven endpoint) + maven: "https://repo1.maven.org/maven2" + + # Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts) + gradle_plugin_portal: "https://plugins.gradle.org/m2" + # Cargo sparse index URL cargo: "https://index.crates.io" diff --git a/docs/configuration.md b/docs/configuration.md index ac85d54..cf6c101 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -114,6 +114,8 @@ Override default upstream registry URLs: ```yaml upstream: npm: "https://registry.npmjs.org" + maven: "https://repo1.maven.org/maven2" + gradle_plugin_portal: "https://plugins.gradle.org/m2" cargo: "https://index.crates.io" cargo_download: "https://static.crates.io/crates" ``` diff --git a/internal/config/config.go b/internal/config/config.go index 067981d..80ad135 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -210,6 +210,15 @@ type UpstreamConfig struct { // Default: https://registry.npmjs.org NPM string `json:"npm" yaml:"npm"` + // Maven is the upstream Maven repository URL. + // Default: https://repo1.maven.org/maven2 + Maven string `json:"maven" yaml:"maven"` + + // GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL. + // Used to resolve Gradle plugin marker artifacts. + // Default: https://plugins.gradle.org/m2 + GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -287,9 +296,11 @@ func Default() *Config { Format: "text", }, Upstream: UpstreamConfig{ - NPM: "https://registry.npmjs.org", - Cargo: "https://index.crates.io", - CargoDownload: "https://static.crates.io/crates", + NPM: "https://registry.npmjs.org", + Maven: "https://repo1.maven.org/maven2", + GradlePluginPortal: "https://plugins.gradle.org/m2", + Cargo: "https://index.crates.io", + CargoDownload: "https://static.crates.io/crates", }, Gradle: GradleConfig{ BuildCache: GradleBuildCacheConfig{ @@ -383,6 +394,12 @@ func (c *Config) LoadFromEnv() { if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" { c.Log.Format = v } + if v := os.Getenv("PROXY_UPSTREAM_MAVEN"); v != "" { + c.Upstream.Maven = v + } + if v := os.Getenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL"); v != "" { + c.Upstream.GradlePluginPortal = v + } if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" { c.Cooldown.Default = v } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 26a0fc6..38adb84 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -31,6 +31,12 @@ func TestDefault(t *testing.T) { if cfg.Gradle.BuildCache.MaxAge != "168h" { t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h") } + if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" { + t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2") + } + if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" { + t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2") + } } func TestValidate(t *testing.T) { @@ -264,6 +270,8 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_BASE_URL", "https://env.example.com") t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) + t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public") + t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB") t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h") @@ -284,6 +292,12 @@ func TestLoadFromEnv(t *testing.T) { if cfg.Log.Level != testLevelDebug { t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug) } + if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" { + t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public") + } + if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" { + t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2") + } if !cfg.Gradle.BuildCache.ReadOnly { t.Error("Gradle.BuildCache.ReadOnly = false, want true") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 639e976..c9d4918 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -673,7 +673,7 @@ func TestMavenHandler_DownloadCacheHit(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content") - h := NewMavenHandler(proxy, "http://localhost") + h := NewMavenHandler(proxy, "http://localhost", "", "") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -730,7 +730,7 @@ func TestMavenHandler_MetadataProxied(t *testing.T) { func TestMavenHandler_EmptyPathNotFound(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) - h := NewMavenHandler(proxy, "http://localhost") + h := NewMavenHandler(proxy, "http://localhost", "", "") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -748,7 +748,7 @@ func TestMavenHandler_EmptyPathNotFound(t *testing.T) { func TestMavenHandler_ArtifactExtensions(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) - extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"} + extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} for _, ext := range extensions { fetcher.artifact = &fetch.Artifact{ Body: io.NopCloser(strings.NewReader("artifact")), @@ -756,7 +756,7 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) { } fetcher.fetchCalled = false - h := NewMavenHandler(proxy, "http://localhost") + h := NewMavenHandler(proxy, "http://localhost", "", "") upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Errorf("should not proxy artifact file %s to upstream", ext) @@ -789,7 +789,7 @@ func TestMavenHandler_CacheMiss(t *testing.T) { ContentType: "application/java-archive", } - h := NewMavenHandler(proxy, "http://localhost") + h := NewMavenHandler(proxy, "http://localhost", "", "") srv := httptest.NewServer(h.Routes()) defer srv.Close() @@ -809,6 +809,276 @@ func TestMavenHandler_CacheMiss(t *testing.T) { } } +func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) { + tests := []struct { + name string + markerPath string + }{ + { + name: "Spotless", + markerPath: "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom", + }, + { + name: "BenManes", + markerPath: "/com/github/ben-manes/versions/com.github.ben-manes.versions.gradle.plugin/0.54.0/com.github.ben-manes.versions.gradle.plugin-0.54.0.pom", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + primaryUpstream := "https://repo1.maven.org/maven2" + pluginPortalUpstream := "https://plugins.gradle.org/m2" + primaryURL := primaryUpstream + tt.markerPath + + fetcher.fetchErrByURL = map[string]error{ + primaryURL: ErrUpstreamNotFound, + } + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("")), + ContentType: "application/xml", + } + + h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + tt.markerPath) + if err != nil { + t.Fatalf("request failed: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if string(body) != "" { + t.Fatalf("body = %q, want %q", body, "") + } + + wantFallbackURL := pluginPortalUpstream + tt.markerPath + if fetcher.fetchedURL != wantFallbackURL { + t.Fatalf("fallback URL = %q, want %q", fetcher.fetchedURL, wantFallbackURL) + } + + fetcher.fetchCalled = false + resp, err = http.Get(srv.URL + tt.markerPath) + if err != nil { + t.Fatalf("second request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("second status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if fetcher.fetchCalled { + t.Fatal("expected plugin marker POM to be served from cache on second request") + } + }) + } +} + +func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { + paths := map[string]string{ + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", + } + + primaryHits := map[string]int{} + pluginHits := map[string]int{} + + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryHits[r.URL.Path]++ + if _, ok := paths[r.URL.Path]; ok { + http.NotFound(w, r) + return + } + t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path) + })) + defer primary.Close() + + pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pluginHits[r.URL.Path]++ + body, ok := paths[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + defer pluginPortal.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = primary.Client() + + h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for reqPath, wantBody := range paths { + resp, err := http.Get(srv.URL + reqPath) + if err != nil { + t.Fatalf("GET %s failed: %v", reqPath, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK) + } + if string(body) != wantBody { + t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody) + } + + if primaryHits[reqPath] == 0 { + t.Fatalf("GET %s did not hit primary upstream", reqPath) + } + if pluginHits[reqPath] == 0 { + t.Fatalf("GET %s did not hit plugin portal fallback", reqPath) + } + } +} + +func TestMavenHandler_GradlePluginMarkerMetadataFallback_ForwardsConditionalHeadersWithoutCache(t *testing.T) { + const ( + requestPath = "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml" + etagValue = `"marker-etag"` + ) + + primaryHits := 0 + pluginHits := 0 + + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryHits++ + if r.URL.Path != requestPath { + t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path) + } + http.NotFound(w, r) + })) + defer primary.Close() + + pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pluginHits++ + if r.URL.Path != requestPath { + t.Fatalf("unexpected path to plugin portal: %s", r.URL.Path) + } + if got := r.Header.Get("If-None-Match"); got == etagValue { + w.WriteHeader(http.StatusNotModified) + return + } + + w.Header().Set("ETag", etagValue) + _, _ = io.WriteString(w, "") + })) + defer pluginPortal.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = false + + h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+requestPath, nil) + if err != nil { + t.Fatalf("failed to build request: %v", err) + } + req.Header.Set("If-None-Match", etagValue) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotModified { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotModified) + } + if primaryHits != 1 { + t.Fatalf("primary hits = %d, want 1", primaryHits) + } + if pluginHits != 1 { + t.Fatalf("plugin portal hits = %d, want 1", pluginHits) + } +} + +func TestMavenHandler_GradlePluginImplementation_FallbackToPluginPortal(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + primaryUpstream := "https://repo1.maven.org/maven2" + pluginPortalUpstream := "https://plugins.gradle.org/m2" + implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar" + primaryURL := primaryUpstream + implPath + pluginPortalURL := pluginPortalUpstream + implPath + + fetcher.fetchErrByURL = map[string]error{ + primaryURL: ErrUpstreamNotFound, + } + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("plugin impl jar")), + ContentType: "application/java-archive", + } + + h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + implPath) + if err != nil { + t.Fatalf("request failed: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if string(body) != "plugin impl jar" { + t.Fatalf("body = %q, want %q", body, "plugin impl jar") + } + + if fetcher.fetchedURL != pluginPortalURL { + t.Fatalf("implementation artifact should fallback to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL) + } +} + +func TestMavenHandler_GradlePluginImplementation_NotFoundInBothUpstreams(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + + primaryUpstream := "https://repo1.maven.org/maven2" + pluginPortalUpstream := "https://plugins.gradle.org/m2" + implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar" + primaryURL := primaryUpstream + implPath + pluginPortalURL := pluginPortalUpstream + implPath + + fetcher.fetchErrByURL = map[string]error{ + primaryURL: ErrUpstreamNotFound, + pluginPortalURL: ErrUpstreamNotFound, + } + + h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + implPath) + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound) + } + + if fetcher.fetchedURL != pluginPortalURL { + t.Fatalf("expected fallback attempt to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL) + } +} + func TestNuGetHandler_DownloadCacheMiss(t *testing.T) { proxy, _, _, fetcher := setupTestProxy(t) fetcher.artifact = &fetch.Artifact{ diff --git a/internal/handler/gradle.go b/internal/handler/gradle.go index aed98e7..efc37a8 100644 --- a/internal/handler/gradle.go +++ b/internal/handler/gradle.go @@ -7,7 +7,9 @@ import ( "regexp" "strconv" "strings" + "time" + "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/storage" ) @@ -34,33 +36,39 @@ func NewGradleBuildCacheHandler(proxy *Proxy) *GradleBuildCacheHandler { // Routes returns the HTTP handler for Gradle HttpBuildCache requests. func (h *GradleBuildCacheHandler) Routes() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rw := &statusCapturingResponseWriter{ResponseWriter: w, status: http.StatusOK} + defer func() { + metrics.RecordRequest("gradle", rw.status, time.Since(start)) + }() + switch r.Method { case http.MethodGet, http.MethodHead, http.MethodPut: default: - http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + http.Error(rw, "method not allowed", http.StatusMethodNotAllowed) return } key, statusCode := h.parseCacheKey(r.URL.Path) if statusCode != http.StatusOK { if statusCode == http.StatusNotFound { - http.NotFound(w, r) + http.NotFound(rw, r) return } - http.Error(w, "invalid cache key", statusCode) + http.Error(rw, "invalid cache key", statusCode) return } if r.Method == http.MethodPut { if h.proxy.GradleReadOnly { - http.Error(w, "gradle build cache is read-only", http.StatusMethodNotAllowed) + http.Error(rw, "gradle build cache is read-only", http.StatusMethodNotAllowed) return } - h.handlePut(w, r, key) + h.handlePut(rw, r, key) return } - h.handleGetOrHead(w, r, key) + h.handleGetOrHead(rw, r, key) }) } @@ -94,36 +102,51 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http w.Header().Set("Content-Type", gradleBuildCacheContentType) if r.Method == http.MethodHead { + existsStart := time.Now() exists, err := h.proxy.Storage.Exists(r.Context(), storagePath) + metrics.RecordStorageOperation("read", time.Since(existsStart)) if err != nil { + metrics.RecordStorageError("read") h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return } if !exists { + metrics.RecordCacheMiss("gradle") http.NotFound(w, r) return } + metrics.RecordCacheHit("gradle") + sizeStart := time.Now() if size, err := h.proxy.Storage.Size(r.Context(), storagePath); err == nil && size >= 0 { + metrics.RecordStorageOperation("read", time.Since(sizeStart)) w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + } else if err != nil { + metrics.RecordStorageOperation("read", time.Since(sizeStart)) + metrics.RecordStorageError("read") } w.WriteHeader(http.StatusOK) return } + readStart := time.Now() reader, err := h.proxy.Storage.Open(r.Context(), storagePath) + metrics.RecordStorageOperation("read", time.Since(readStart)) if err != nil { if errors.Is(err, storage.ErrNotFound) { + metrics.RecordCacheMiss("gradle") http.NotFound(w, r) return } + metrics.RecordStorageError("read") h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to read cache entry", http.StatusInternalServerError) return } defer func() { _ = reader.Close() }() + metrics.RecordCacheHit("gradle") w.WriteHeader(http.StatusOK) _, _ = io.Copy(w, reader) @@ -138,7 +161,9 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize) + storeStart := time.Now() _, hash, err := h.proxy.Storage.Store(r.Context(), storagePath, r.Body) + metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { var maxBytesErr *http.MaxBytesError if errors.As(err, &maxBytesErr) { @@ -146,6 +171,7 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque return } + metrics.RecordStorageError("write") h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err) http.Error(w, "failed to write cache entry", http.StatusInternalServerError) return @@ -156,3 +182,13 @@ func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Reque w.WriteHeader(http.StatusCreated) } + +type statusCapturingResponseWriter struct { + http.ResponseWriter + status int +} + +func (rw *statusCapturingResponseWriter) WriteHeader(code int) { + rw.status = code + rw.ResponseWriter.WriteHeader(code) +} diff --git a/internal/handler/gradle_test.go b/internal/handler/gradle_test.go index f002a51..b688803 100644 --- a/internal/handler/gradle_test.go +++ b/internal/handler/gradle_test.go @@ -4,8 +4,12 @@ import ( "io" "net/http" "net/http/httptest" + "strconv" "strings" "testing" + + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" ) func TestGradleBuildCacheHandler_PutGetHead(t *testing.T) { @@ -227,3 +231,71 @@ func TestGradleBuildCacheHandler_PutTooLarge(t *testing.T) { t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusRequestEntityTooLarge) } } + +func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewGradleBuildCacheHandler(proxy) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + okBefore := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusOK))) + createdBefore := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusCreated))) + notFoundBefore := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusNotFound))) + hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) + missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) + + key := "metrics-key" + putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload")) + if err != nil { + t.Fatalf("failed to create PUT request: %v", err) + } + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatalf("PUT request failed: %v", err) + } + _ = putResp.Body.Close() + + getResp, err := http.Get(srv.URL + "/" + key) + if err != nil { + t.Fatalf("GET request failed: %v", err) + } + _ = getResp.Body.Close() + + headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil) + if err != nil { + t.Fatalf("failed to create HEAD request: %v", err) + } + headResp, err := http.DefaultClient.Do(headReq) + if err != nil { + t.Fatalf("HEAD request failed: %v", err) + } + _ = headResp.Body.Close() + + missResp, err := http.Get(srv.URL + "/missing-key") + if err != nil { + t.Fatalf("GET miss request failed: %v", err) + } + _ = missResp.Body.Close() + + okAfter := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusOK))) + createdAfter := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusCreated))) + notFoundAfter := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("gradle", strconv.Itoa(http.StatusNotFound))) + hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle")) + missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle")) + + if diff := createdAfter - createdBefore; diff != 1 { + t.Fatalf("created requests delta = %.0f, want 1", diff) + } + if diff := okAfter - okBefore; diff != 2 { + t.Fatalf("ok requests delta = %.0f, want 2", diff) + } + if diff := notFoundAfter - notFoundBefore; diff != 1 { + t.Fatalf("not found requests delta = %.0f, want 1", diff) + } + if diff := hitsAfter - hitsBefore; diff != 2 { + t.Fatalf("cache hits delta = %.0f, want 2", diff) + } + if diff := missesAfter - missesBefore; diff != 1 { + t.Fatalf("cache misses delta = %.0f, want 1", diff) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0ad0776..badd8ab 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -680,9 +680,14 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, return } + p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType) +} + +// writeMetadataCachedResponse writes a cached metadata response and handles +// conditional request headers using metadata cache validators. +func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { cm := p.lookupCachedMeta(ecosystem, cacheKey) - // Honor client conditional request headers if cm.etag != "" { if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag { w.WriteHeader(http.StatusNotModified) @@ -716,12 +721,48 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, // proxyMetadataStream forwards an upstream metadata response by streaming it to the client // without buffering the full body in memory. func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL string, acceptHeaders ...string) { - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + resp, err := p.fetchMetadataStreamResponse(r, upstreamURL, acceptHeaders...) if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + + p.writeMetadataStreamResponse(w, resp) +} + +// proxyMetadataStreamWithFallback streams metadata from the primary upstream and +// retries against fallbackURL only when the primary returns 404 Not Found. +func (p *Proxy) proxyMetadataStreamWithFallback(w http.ResponseWriter, r *http.Request, primaryURL, fallbackURL string, acceptHeaders ...string) { + resp, err := p.fetchMetadataStreamResponse(r, primaryURL, acceptHeaders...) + if err != nil { + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } + if resp.StatusCode != http.StatusNotFound || fallbackURL == "" { + defer func() { _ = resp.Body.Close() }() + p.writeMetadataStreamResponse(w, resp) + return + } + _ = resp.Body.Close() + + fallbackResp, fallbackErr := p.fetchMetadataStreamResponse(r, fallbackURL, acceptHeaders...) + if fallbackErr != nil { + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return + } + defer func() { _ = fallbackResp.Body.Close() }() + + p.writeMetadataStreamResponse(w, fallbackResp) +} + +func (p *Proxy) fetchMetadataStreamResponse(r *http.Request, upstreamURL string, acceptHeaders ...string) (*http.Response, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, err + } + accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] @@ -734,12 +775,10 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst } } - resp, err := p.HTTPClient.Do(req) - if err != nil { - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() + return p.HTTPClient.Do(req) +} + +func (p *Proxy) writeMetadataStreamResponse(w http.ResponseWriter, resp *http.Response) { for _, header := range []string{"Content-Type", "Content-Length", "Last-Modified", "ETag"} { if v := resp.Header.Get(header); v != "" { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 3a1d2ab..bbcab72 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -97,10 +97,11 @@ func (s *mockStorage) Close() error { return nil } // mockFetcher implements fetch.FetcherInterface for testing. type mockFetcher struct { - artifact *fetch.Artifact - fetchErr error - fetchCalled bool - fetchedURL string + artifact *fetch.Artifact + fetchErr error + fetchErrByURL map[string]error + fetchCalled bool + fetchedURL string } func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { @@ -110,6 +111,11 @@ func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, e func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { f.fetchCalled = true f.fetchedURL = url + if f.fetchErrByURL != nil { + if err, ok := f.fetchErrByURL[url]; ok { + return nil, err + } + } if f.fetchErr != nil { return nil, f.fetchErr } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 86664a2..d79a904 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,6 +1,7 @@ package handler import ( + "errors" "fmt" "net/http" "path" @@ -8,23 +9,34 @@ import ( ) const ( - mavenUpstream = "https://repo1.maven.org/maven2" - minMavenParts = 4 // group path segments + artifact + version + filename + mavenCentralUpstream = "https://repo1.maven.org/maven2" + gradlePluginPortalUpstream = "https://plugins.gradle.org/m2" + minMavenParts = 4 // group path segments + artifact + version + filename + gradlePluginMarkerSuffix = ".gradle.plugin" ) // MavenHandler handles Maven repository protocol requests. type MavenHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string + proxy *Proxy + upstreamURL string + pluginPortalUpstreamURL string + proxyURL string } // NewMavenHandler creates a new Maven repository handler. -func NewMavenHandler(proxy *Proxy, proxyURL string) *MavenHandler { +func NewMavenHandler(proxy *Proxy, proxyURL, upstreamURL, pluginPortalUpstreamURL string) *MavenHandler { + if strings.TrimSpace(upstreamURL) == "" { + upstreamURL = mavenCentralUpstream + } + if strings.TrimSpace(pluginPortalUpstreamURL) == "" { + pluginPortalUpstreamURL = gradlePluginPortalUpstream + } + return &MavenHandler{ - proxy: proxy, - upstreamURL: mavenUpstream, - proxyURL: strings.TrimSuffix(proxyURL, "/"), + proxy: proxy, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + pluginPortalUpstreamURL: strings.TrimSuffix(pluginPortalUpstreamURL, "/"), + proxyURL: strings.TrimSuffix(proxyURL, "/"), } } @@ -51,8 +63,7 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { filename := path.Base(urlPath) if h.isMetadataFile(filename) { - cacheKey := strings.ReplaceAll(urlPath, "/", "_") - h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "maven", cacheKey, "*/*") + h.handleMetadata(w, r, urlPath, filename) return } @@ -66,6 +77,42 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { h.proxyUpstream(w, r) } +func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath, filename string) { + cacheKey := strings.ReplaceAll(urlPath, "/", "_") + if !h.shouldFallbackToPluginPortal(urlPath, filename) { + h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "maven", cacheKey, "*/*") + return + } + + upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath) + pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + if !h.proxy.CacheMetadata { + h.proxy.proxyMetadataStreamWithFallback(w, r, upstreamURL, pluginPortalURL, "*/*") + return + } + + body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", + "path", urlPath, "filename", filename) + body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") + } + } + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + h.proxy.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return + } + + h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) +} + // handleDownload serves an artifact file, fetching and caching from upstream if needed. func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) { // Parse Maven path: group/artifact/version/filename @@ -86,6 +133,18 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL) if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath) + h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", + "group", group, "artifact", artifact, "version", version, "filename", filename) + result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL) + } + } + if err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } h.proxy.Logger.Error("failed to get artifact", "error", err) http.Error(w, "failed to fetch artifact", http.StatusBadGateway) return @@ -115,7 +174,7 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file // isArtifactFile returns true if the filename looks like a Maven artifact. func (h *MavenHandler) isArtifactFile(filename string) bool { // Common artifact extensions - extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"} + extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"} for _, ext := range extensions { if strings.HasSuffix(filename, ext) { return true @@ -124,6 +183,67 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { return false } +func (h *MavenHandler) shouldFallbackToPluginPortal(urlPath, filename string) bool { + if !h.isPluginPortalFallbackFile(filename) { + return false + } + + group, artifact, ok := h.extractGroupAndArtifactFromPath(urlPath, filename) + if !ok { + return false + } + + if !strings.HasSuffix(artifact, gradlePluginMarkerSuffix) { + return false + } + + markerPrefix := strings.TrimSuffix(artifact, gradlePluginMarkerSuffix) + return markerPrefix != "" && markerPrefix == group +} + +func (h *MavenHandler) isPluginPortalFallbackFile(filename string) bool { + if filename == "maven-metadata.xml" || strings.HasPrefix(filename, "maven-metadata.xml.") { + return true + } + if strings.HasSuffix(filename, ".pom") || strings.HasSuffix(filename, ".module") { + return true + } + + for _, suffix := range []string{".sha1", ".sha256", ".sha512", ".md5", ".asc"} { + if !strings.HasSuffix(filename, suffix) { + continue + } + + base := strings.TrimSuffix(filename, suffix) + return strings.HasSuffix(base, ".pom") || strings.HasSuffix(base, ".module") + } + + return false +} + +func (h *MavenHandler) extractGroupAndArtifactFromPath(urlPath, filename string) (group, artifact string, ok bool) { + const ( + artifactVersionPathOffset = 3 // .../{artifact}/{version}/{filename} + metadataPathOffset = 2 // .../{artifact}/maven-metadata.xml + ) + + parts := strings.Split(urlPath, "/") + + pathOffset := artifactVersionPathOffset + if filename == "maven-metadata.xml" || strings.HasPrefix(filename, "maven-metadata.xml.") { + pathOffset = metadataPathOffset + } + segmentIdx := len(parts) - pathOffset + + if segmentIdx <= 0 || segmentIdx >= len(parts) { + return "", "", false + } + + group = strings.Join(parts[:segmentIdx], ".") + artifact = parts[segmentIdx] + return group, artifact, group != "" && artifact != "" +} + // isMetadataFile returns true if the filename is Maven metadata. func (h *MavenHandler) isMetadataFile(filename string) bool { return filename == "maven-metadata.xml" || diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index df6917c..38b7bfd 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -52,6 +52,7 @@ func TestMavenIsArtifactFile(t *testing.T) { }{ {"guava-32.1.3-jre.jar", true}, {"guava-32.1.3-jre.pom", true}, + {"guava-32.1.3-jre.module", true}, {"app-1.0.war", true}, {"lib-1.0.aar", true}, {"maven-metadata.xml", false}, @@ -65,3 +66,66 @@ func TestMavenIsArtifactFile(t *testing.T) { } } } + +func TestMavenShouldFallbackToPluginPortal(t *testing.T) { + h := &MavenHandler{} + + tests := []struct { + name string + urlPath string + filename string + want bool + }{ + { + name: "marker pom", + urlPath: "com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom", + filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom", + want: true, + }, + { + name: "marker pom checksum", + urlPath: "com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", + filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1", + want: true, + }, + { + name: "marker metadata", + urlPath: "com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml", + filename: "maven-metadata.xml", + want: true, + }, + { + name: "marker metadata checksum", + urlPath: "com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha256", + filename: "maven-metadata.xml.sha256", + want: true, + }, + { + name: "non marker pom checksum", + urlPath: "com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.pom.sha1", + filename: "guava-32.1.3-jre.pom.sha1", + want: false, + }, + { + name: "jar checksum", + urlPath: "com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.jar.sha1", + filename: "com.diffplug.spotless.gradle.plugin-8.4.0.jar.sha1", + want: false, + }, + { + name: "path too short", + urlPath: "com.diffplug.spotless.gradle.plugin/maven-metadata.xml", + filename: "maven-metadata.xml", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := h.shouldFallbackToPluginPortal(tt.urlPath, tt.filename) + if got != tt.want { + t.Errorf("shouldFallbackToPluginPortal(%q, %q) = %v, want %v", tt.urlPath, tt.filename, got, tt.want) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index ffb357b..cf65651 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -183,7 +183,12 @@ func (s *Server) Start() error { hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL) - mavenHandler := handler.NewMavenHandler(proxy, s.cfg.BaseURL) + mavenHandler := handler.NewMavenHandler( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Maven, + s.cfg.Upstream.GradlePluginPortal, + ) gradleHandler := handler.NewGradleBuildCacheHandler(proxy) nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL) composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL)