diff --git a/.gitignore b/.gitignore index 7c14c5f0..f7ea3371 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ coverage.html .vscode results.json pipeleek.yaml +pipeleek_test_build diff --git a/docs/introduction/configuration.md b/docs/introduction/configuration.md index c0c761d1..75b3bb2f 100644 --- a/docs/introduction/configuration.md +++ b/docs/introduction/configuration.md @@ -196,3 +196,7 @@ pipeleek config gen # Use trace logging to see which keys are loaded pipeleek --log-level=trace gl enum ``` + +## HTTP Client Settings + +See [Using Pipeleek with Proxies](proxying.md) for proxy, TLS, and timeout configuration flags. diff --git a/docs/introduction/proxying.md b/docs/introduction/proxying.md index f3bec8cc..3f4df44a 100644 --- a/docs/introduction/proxying.md +++ b/docs/introduction/proxying.md @@ -32,6 +32,18 @@ SOCKS5 can be used as well. HTTP_PROXY=socks5://127.0.0.1:1080 pipeleek gl scan -u https://gitlab.internal.company.com -t glpat-xxxxx ``` +### Using the `--proxy` Flag + +Alternatively, use the `--proxy` flag to set any proxy from the command line without relying on `HTTP_PROXY`. It accepts both HTTP and SOCKS5 URLs and takes precedence over the environment variable: + +```bash +# HTTP proxy +pipeleek --proxy http://127.0.0.1:8080 gl scan -u https://gitlab.com -t glpat-xxxxx + +# SOCKS5 proxy +pipeleek --proxy socks5://127.0.0.1:1080 gl scan -u https://gitlab.internal.company.com -t glpat-xxxxx +``` + ## Ignoring Proxy Configuration In some environments, `HTTP_PROXY` may be set system-wide but you don't want Pipeleek to use it. Use the `--ignore-proxy` flag to bypass proxy detection: @@ -40,6 +52,35 @@ In some environments, `HTTP_PROXY` may be set system-wide but you don't want Pip HTTP_PROXY=http://127.0.0.1:8080 pipeleek --ignore-proxy gl scan -u https://gitlab.com -t glpat-xxxxx ``` -## TLS/SSL +## TLS Certificate Verification + +By default, Pipeleek skips TLS certificate verification so that self-hosted instances with self-signed certificates work out of the box. Use `--tls-verification` to enforce certificate validation: + +```bash +pipeleek --tls-verification gl scan --token glpat-xxx --url https://gitlab.example.com +``` + +## HTTP Timeout + +Use `--http-timeout` to set a per-request timeout. This is useful when scanning slow or unreliable targets: + +```bash +pipeleek --http-timeout 30s gl scan --token glpat-xxx --url https://gitlab.example.com +``` + +Accepts any Go duration string: `30s`, `2m`, `90s`, etc. The default is no timeout. + +> **Note:** `--http-timeout` applies to platforms that use `GetPipeleekHTTPClient` (GitLab, Gitea, GitHub, Jenkins, CircleCI, NIST, and rule downloads). Bitbucket and Azure DevOps inject only the transport via `GetPipeleekTransport` and are not affected by this flag. + +## Platform Scope + +All proxy and TLS flags share a single HTTP transport injected into every platform client: + +| Flag | Default | Applies to | +|---|---|---| +| `--tls-verification` | `false` | All platforms | +| `--ignore-proxy` | `false` | All platforms | +| `--proxy ` | _(none)_ | All platforms | +| `--http-timeout ` | _(no timeout)_ | GitLab, Gitea, Jenkins, CircleCI, NIST (not Bitbucket/DevOps) | -Pipeleek automatically skips TLS certificate verification (required for self signed certificates). +> **Note:** The GitHub SDK uses a dedicated rate-limit transport (`go-github-ratelimit`) that cannot be replaced. TLS and proxy settings still apply to GitHub via the shared transport layer. diff --git a/go.mod b/go.mod index f3b1fdf7..ab1181d9 100644 --- a/go.mod +++ b/go.mod @@ -209,7 +209,7 @@ require ( github.com/google/go-querystring v1.2.0 // indirect github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/internal/cmd/root.go b/internal/cmd/root.go index 6eb17092..c313a033 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -50,16 +50,22 @@ var ( setGlobalLogLevel(cmd) loadConfigFile(cmd) httpclient.SetIgnoreProxy(IgnoreProxy) + httpclient.SetInsecureSkipVerify(!TLSVerification) + httpclient.SetProxy(Proxy) + httpclient.SetHTTPTimeout(HTTPTimeout) go logging.ShortcutListeners(nil) }, } - JsonLogoutput bool - LogFile string - LogColor bool - LogDebug bool - LogLevel string - IgnoreProxy bool - ConfigFile string + JsonLogoutput bool + LogFile string + LogColor bool + LogDebug bool + LogLevel string + IgnoreProxy bool + ConfigFile string + TLSVerification bool + Proxy string + HTTPTimeout time.Duration // runLogFileHandle holds the file handle when logging to a file is enabled runLogFileHandle *os.File ) @@ -91,6 +97,8 @@ func init() { rootCmd.PersistentFlags().StringVar(&LogLevel, "log-level", "", "Set log level globally (debug, info, warn, error). Example: --log-level=warn") rootCmd.PersistentFlags().BoolVar(&LogColor, "color", true, "Enable colored log output (auto-disabled when using --logfile)") rootCmd.PersistentFlags().BoolVar(&IgnoreProxy, "ignore-proxy", false, "Ignore HTTP_PROXY environment variable") + rootCmd.PersistentFlags().StringVar(&Proxy, "proxy", "", "Proxy URL, e.g. http://127.0.0.1:8080 or socks5://127.0.0.1:1080 (takes precedence over HTTP_PROXY)") + rootCmd.PersistentFlags().DurationVar(&HTTPTimeout, "http-timeout", 0, "HTTP request timeout, e.g. 30s or 2m (default: no timeout)") // Set custom version template to show detailed version info rootCmd.SetVersionTemplate(`{{.Version}} diff --git a/pkg/bitbucket/scan/api.go b/pkg/bitbucket/scan/api.go index c956af58..65097eee 100644 --- a/pkg/bitbucket/scan/api.go +++ b/pkg/bitbucket/scan/api.go @@ -9,6 +9,7 @@ import ( "strconv" "time" + "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/rs/zerolog/log" "resty.dev/v3" @@ -38,7 +39,7 @@ func NewClient(username string, password string, bitBucketCookie string, baseURL } internalBase := parsedBase.Scheme + "://" + internalHost + "/!api" - client := *resty.New().SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)) + client := *httpclient.GetPipeleekHTTPClient("", nil, nil).SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)) if len(bitBucketCookie) > 0 { jar, _ := cookiejar.New(nil) // set cookie on the internal host root so requests to internal endpoints include it diff --git a/pkg/bitbucket/scan/client_test.go b/pkg/bitbucket/scan/client_test.go new file mode 100644 index 00000000..6ac530f3 --- /dev/null +++ b/pkg/bitbucket/scan/client_test.go @@ -0,0 +1,96 @@ +package scan + +import ( + "crypto/tls" + "net/http" + "net/url" + "testing" + + "github.com/CompassSecurity/pipeleek/pkg/httpclient" +) + +func TestNewClient_UsesPipeleekTransport(t *testing.T) { + // Arrange: configure a distinct InsecureSkipVerify value so we can detect it. + httpclient.SetInsecureSkipVerify(false) + t.Cleanup(func() { httpclient.SetInsecureSkipVerify(true) }) + + c := NewClient("user", "pass", "", "https://api.bitbucket.org/2.0") + + tr, ok := c.Client.Transport().(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport as client transport, got %T", c.Client.Transport()) + } + if tr.TLSClientConfig == nil { + t.Fatal("expected TLSClientConfig to be set on transport") + } + if tr.TLSClientConfig.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=false to be reflected from Pipeleek global config") + } +} + +func TestNewClient_DefaultBaseURL(t *testing.T) { + c := NewClient("user", "pass", "", "") + if c.BaseURL != "https://api.bitbucket.org/2.0" { + t.Errorf("unexpected default BaseURL: %s", c.BaseURL) + } +} + +func TestNewClient_InternalBaseURLDerivedFromDefault(t *testing.T) { + c := NewClient("user", "pass", "", "") + if c.InternalBaseURL != "https://bitbucket.org/!api" { + t.Errorf("unexpected InternalBaseURL: %s", c.InternalBaseURL) + } +} + +func TestNewClient_CookieJarSetWhenCookieProvided(t *testing.T) { + c := NewClient("user", "pass", "tok123", "https://api.bitbucket.org/2.0") + if c.Client.CookieJar() == nil { + t.Error("expected cookie jar to be set when cookie is provided") + } +} + +func TestNewClient_NoCookieSetWhenNoCookie(t *testing.T) { + // Resty v3 always creates a default cookie jar. When no BitBucket cookie is + // provided, no cloud.session.token cookie should be present in the jar. + c := NewClient("user", "pass", "", "https://api.bitbucket.org/2.0") + jar := c.Client.CookieJar() + if jar == nil { + // If Resty ever stops setting a default jar this test still passes. + return + } + u, _ := url.Parse("https://bitbucket.org/!api") + for _, ck := range jar.Cookies(u) { + if ck.Name == "cloud.session.token" { + t.Error("cloud.session.token should not be set in jar when no cookie provided") + } + } +} + +func TestNewClient_TLSReflectsGlobalConfig(t *testing.T) { + tests := []struct { + name string + insecureSkipVerify bool + }{ + {"skip=true", true}, + {"skip=false", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + httpclient.SetInsecureSkipVerify(tt.insecureSkipVerify) + t.Cleanup(func() { httpclient.SetInsecureSkipVerify(true) }) + + c := NewClient("u", "p", "", "https://api.bitbucket.org/2.0") + rawTr, ok := c.Client.Transport().(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", c.Client.Transport()) + } + got := rawTr.TLSClientConfig.InsecureSkipVerify + if got != tt.insecureSkipVerify { + t.Errorf("InsecureSkipVerify: want %v, got %v", tt.insecureSkipVerify, got) + } + }) + } +} + +// ensure the helper compiles even when tls package is not directly used in tests above +var _ = (*tls.Config)(nil) diff --git a/pkg/circle/scan/scanner.go b/pkg/circle/scan/scanner.go index 95e660c7..625d5eb7 100644 --- a/pkg/circle/scan/scanner.go +++ b/pkg/circle/scan/scanner.go @@ -12,6 +12,7 @@ import ( "time" "github.com/CompassSecurity/pipeleek/pkg/format" + "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/CompassSecurity/pipeleek/pkg/logging" "github.com/CompassSecurity/pipeleek/pkg/scan/logline" "github.com/CompassSecurity/pipeleek/pkg/scan/result" @@ -567,7 +568,7 @@ func InitializeOptions(input InitializeOptionsInput) (ScanOptions, error) { return ScanOptions{}, err } - httpClient := &http.Client{Timeout: 45 * time.Second} + httpClient := httpclient.GetPipeleekHTTPClient("", nil, nil).Client() apiClient := newCircleAPIClient(baseURL, input.Token, httpClient) if len(projects) == 0 { diff --git a/pkg/devops/scan/api.go b/pkg/devops/scan/api.go index e927cda9..cddb0dcf 100644 --- a/pkg/devops/scan/api.go +++ b/pkg/devops/scan/api.go @@ -5,6 +5,7 @@ import ( "path" "strconv" + "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/rs/zerolog/log" "resty.dev/v3" @@ -22,7 +23,7 @@ func NewClient(username string, password string, baseURL string) AzureDevOpsApiC baseURL = "https://dev.azure.com" } bbClient := AzureDevOpsApiClient{ - Client: *resty.New().SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)), + Client: *httpclient.GetPipeleekHTTPClient("", nil, nil).SetBasicAuth(username, password).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)), BaseURL: baseURL, VsspsURL: "https://app.vssps.visualstudio.com", } diff --git a/pkg/devops/scan/client_test.go b/pkg/devops/scan/client_test.go new file mode 100644 index 00000000..11ca19e1 --- /dev/null +++ b/pkg/devops/scan/client_test.go @@ -0,0 +1,66 @@ +package scan + +import ( + "net/http" + "testing" + + "github.com/CompassSecurity/pipeleek/pkg/httpclient" +) + +func TestAzureDevOpsNewClient_UsesPipeleekTransport(t *testing.T) { + httpclient.SetInsecureSkipVerify(false) + t.Cleanup(func() { httpclient.SetInsecureSkipVerify(true) }) + + c := NewClient("user", "pass", "https://dev.azure.com") + + tr, ok := c.Client.Transport().(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport as client transport, got %T", c.Client.Transport()) + } + if tr.TLSClientConfig == nil { + t.Fatal("expected TLSClientConfig to be set on transport") + } + if tr.TLSClientConfig.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=false to be reflected from Pipeleek global config") + } +} + +func TestAzureDevOpsNewClient_DefaultBaseURL(t *testing.T) { + c := NewClient("user", "pass", "") + if c.BaseURL != "https://dev.azure.com" { + t.Errorf("unexpected default BaseURL: %s", c.BaseURL) + } +} + +func TestAzureDevOpsNewClient_VsspsURLIsFixed(t *testing.T) { + c := NewClient("user", "pass", "https://dev.azure.com") + if c.VsspsURL != "https://app.vssps.visualstudio.com" { + t.Errorf("unexpected VsspsURL: %s", c.VsspsURL) + } +} + +func TestAzureDevOpsNewClient_TLSReflectsGlobalConfig(t *testing.T) { + tests := []struct { + name string + insecureSkipVerify bool + }{ + {"skip=true", true}, + {"skip=false", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + httpclient.SetInsecureSkipVerify(tt.insecureSkipVerify) + t.Cleanup(func() { httpclient.SetInsecureSkipVerify(true) }) + + c := NewClient("u", "p", "") + rawTr, ok := c.Client.Transport().(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", c.Client.Transport()) + } + got := rawTr.TLSClientConfig.InsecureSkipVerify + if got != tt.insecureSkipVerify { + t.Errorf("InsecureSkipVerify: want %v, got %v", tt.insecureSkipVerify, got) + } + }) + } +} diff --git a/pkg/gitea/scan/api.go b/pkg/gitea/scan/api.go index fddbf05d..b4ba850e 100644 --- a/pkg/gitea/scan/api.go +++ b/pkg/gitea/scan/api.go @@ -4,15 +4,14 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/url" "strconv" "time" "code.gitea.io/sdk/gitea" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" + "resty.dev/v3" ) type GiteaScanOptions struct { @@ -32,7 +31,7 @@ type GiteaScanOptions struct { HitTimeout time.Duration Context context.Context Client *gitea.Client - HttpClient *retryablehttp.Client + HttpClient *resty.Client } type AuthTransport struct { @@ -132,32 +131,20 @@ func listWorkflowRuns(client *gitea.Client, repo *gitea.Repository) ([]ActionWor q.Set("limit", strconv.Itoa(limit)) link.RawQuery = q.Encode() - resp, err := scanOptions.HttpClient.Get(link.String()) + resp, err := scanOptions.HttpClient.R().Get(link.String()) if err != nil { return nil, err } - if resp.StatusCode == 404 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } + if resp.StatusCode() == 404 { return allRuns, nil } - if resp.StatusCode != 200 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) } - body, err := io.ReadAll(resp.Body) - if closeErr := resp.Body.Close(); closeErr != nil { - log.Debug().Err(closeErr).Msg("Failed to close response body") - } - if err != nil { - return nil, err - } + body := resp.Bytes() var runsResp ActionWorkflowRunsResponse if err := json.Unmarshal(body, &runsResp); err != nil { @@ -242,32 +229,20 @@ func listWorkflowJobs(client *gitea.Client, repo *gitea.Repository, run ActionWo q.Set("limit", strconv.Itoa(limit)) link.RawQuery = q.Encode() - resp, err := scanOptions.HttpClient.Get(link.String()) + resp, err := scanOptions.HttpClient.R().Get(link.String()) if err != nil { return nil, err } - if resp.StatusCode == 404 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } + if resp.StatusCode() == 404 { return allJobs, nil } - if resp.StatusCode != 200 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) } - body, err := io.ReadAll(resp.Body) - if closeErr := resp.Body.Close(); closeErr != nil { - log.Debug().Err(closeErr).Msg("Failed to close response body") - } - if err != nil { - return nil, err - } + body := resp.Bytes() var jobsResp ActionJobsResponse if err := json.Unmarshal(body, &jobsResp); err != nil { @@ -380,32 +355,20 @@ func listArtifacts(repo *gitea.Repository, run ActionWorkflowRun) ([]ActionArtif q.Set("limit", strconv.Itoa(limit)) link.RawQuery = q.Encode() - resp, err := scanOptions.HttpClient.Get(link.String()) + resp, err := scanOptions.HttpClient.R().Get(link.String()) if err != nil { return nil, err } - if resp.StatusCode == 404 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } + if resp.StatusCode() == 404 { return allArtifacts, nil } - if resp.StatusCode != 200 { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close response body") - } - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) } - body, err := io.ReadAll(resp.Body) - if closeErr := resp.Body.Close(); closeErr != nil { - log.Debug().Err(closeErr).Msg("Failed to close response body") - } - if err != nil { - return nil, err - } + body := resp.Bytes() var artifactsResp ActionArtifactsResponse if err := json.Unmarshal(body, &artifactsResp); err != nil { diff --git a/pkg/gitea/scan/http_utils.go b/pkg/gitea/scan/http_utils.go index bae1a17c..fc9c1d0f 100644 --- a/pkg/gitea/scan/http_utils.go +++ b/pkg/gitea/scan/http_utils.go @@ -1,10 +1,7 @@ package gitea import ( - "bytes" "fmt" - "io" - "net/http" "net/url" "code.gitea.io/sdk/gitea" @@ -22,7 +19,7 @@ func makeHTTPGetRequest(url string) (*httpResponse, error) { return nil, fmt.Errorf("HTTP client is not initialized") } - resp, err := scanOptions.HttpClient.Get(url) + resp, err := scanOptions.HttpClient.R().Get(url) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) } @@ -31,23 +28,10 @@ func makeHTTPGetRequest(url string) (*httpResponse, error) { return nil, fmt.Errorf("HTTP response is nil") } - contentLength := resp.ContentLength - - defer func() { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close HTTP response body") - } - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - return &httpResponse{ - Body: body, - StatusCode: resp.StatusCode, - ContentLength: contentLength, + Body: resp.Bytes(), + StatusCode: resp.StatusCode(), + ContentLength: resp.RawResponse.ContentLength, }, nil } @@ -56,26 +40,10 @@ func makeHTTPPostRequest(urlStr string, body []byte, headers map[string]string) return nil, fmt.Errorf("HTTP client is not initialized") } - client := scanOptions.HttpClient.StandardClient() - if client == nil { - return nil, fmt.Errorf("standard HTTP client is not initialized") - } - - var bodyReader io.Reader - if body != nil { - bodyReader = bytes.NewReader(body) - } - - req, err := http.NewRequest("POST", urlStr, bodyReader) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - for key, value := range headers { - req.Header.Set(key, value) - } - - resp, err := client.Do(req) + resp, err := scanOptions.HttpClient.R(). + SetBody(body). + SetHeaders(headers). + Post(urlStr) if err != nil { return nil, fmt.Errorf("HTTP POST request failed: %w", err) } @@ -84,20 +52,9 @@ func makeHTTPPostRequest(urlStr string, body []byte, headers map[string]string) return nil, fmt.Errorf("HTTP response is nil") } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Debug().Err(err).Msg("Failed to close HTTP POST response body") - } - }() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - return &httpResponse{ - Body: respBody, - StatusCode: resp.StatusCode, + Body: resp.Bytes(), + StatusCode: resp.StatusCode(), }, nil } diff --git a/pkg/gitea/scan/http_utils_test.go b/pkg/gitea/scan/http_utils_test.go index 0f0ee8b1..afb04747 100644 --- a/pkg/gitea/scan/http_utils_test.go +++ b/pkg/gitea/scan/http_utils_test.go @@ -9,10 +9,10 @@ import ( "testing" "code.gitea.io/sdk/gitea" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" + "resty.dev/v3" ) func TestBuildGiteaURL(t *testing.T) { @@ -249,7 +249,7 @@ func TestMakeHTTPGetRequest(t *testing.T) { _, _ = w.Write([]byte("server error")) })) }, - expectError: true, + expectError: false, expectedStatus: http.StatusInternalServerError, expectedBody: "server error", }, @@ -260,9 +260,7 @@ func TestMakeHTTPGetRequest(t *testing.T) { server := tt.setupServer() defer server.Close() - retryClient := retryablehttp.NewClient() - retryClient.RetryMax = 0 - scanOptions.HttpClient = retryClient +scanOptions.HttpClient = resty.New().SetRetryCount(0) resp, err := makeHTTPGetRequest(server.URL) @@ -344,9 +342,7 @@ func TestMakeHTTPPostRequest(t *testing.T) { server := tt.setupServer() defer server.Close() - retryClient := retryablehttp.NewClient() - retryClient.RetryMax = 0 - scanOptions.HttpClient = retryClient +scanOptions.HttpClient = resty.New().SetRetryCount(0) resp, err := makeHTTPPostRequest(server.URL, tt.requestBody, tt.headers) @@ -502,9 +498,8 @@ func setupTestScanOptions() { StartRunID: 0, Context: context.Background(), Client: nil, - HttpClient: retryablehttp.NewClient(), + HttpClient: resty.New().SetRetryCount(0), } - scanOptions.HttpClient.RetryMax = 0 } func TestMain(m *testing.M) { diff --git a/pkg/gitea/scan/scanner.go b/pkg/gitea/scan/scanner.go index b1e6eca1..df77d3e8 100644 --- a/pkg/gitea/scan/scanner.go +++ b/pkg/gitea/scan/scanner.go @@ -12,8 +12,8 @@ import ( "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/CompassSecurity/pipeleek/pkg/scan/runner" pkgscanner "github.com/CompassSecurity/pipeleek/pkg/scanner" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" + "resty.dev/v3" ) // ScanOptions is an alias for GiteaScanOptions for interface consistency with other providers. @@ -263,14 +263,10 @@ func InitializeOptions(token, giteaURL, repository, organization, cookie, maxArt } ctx := context.Background() - client, err := gitea.NewClient(giteaURL, gitea.SetToken(token)) - if err != nil { - return ScanOptions{}, err - } authHeaders := map[string]string{"Authorization": "token " + token} - var httpClient *retryablehttp.Client + var httpClient *resty.Client if cookie != "" { // #nosec G124 - Cookie attributes (Secure/HttpOnly/SameSite) are server-side browser directives; not applicable for client HTTP requests httpClient = httpclient.GetPipeleekHTTPClient( @@ -286,16 +282,18 @@ func InitializeOptions(token, giteaURL, repository, organization, cookie, maxArt authHeaders, ) } else { + // Auth header is injected via defaultHeaders in HeaderRoundTripper; no post-hoc + // transport mutation needed. The Pipeleek transport (TLS, proxy) is preserved. httpClient = httpclient.GetPipeleekHTTPClient("", nil, authHeaders) + } - standardHTTPClient := &http.Client{ - Transport: &AuthTransport{ - Base: http.DefaultTransport, - Token: token, - }, - } - - httpClient.StandardClient().Transport = standardHTTPClient.Transport + // Inject the Pipeleek standard client into the Gitea SDK so it shares the same + // TLS/proxy/SOCKS settings. Auth is handled by gitea.SetToken; no auth headers + // are passed here to avoid duplication. + baseStandardClient := httpclient.GetPipeleekHTTPClient("", nil, nil).Client() + client, err := gitea.NewClient(giteaURL, gitea.SetToken(token), gitea.SetHTTPClient(baseStandardClient)) + if err != nil { + return ScanOptions{}, err } return ScanOptions{ diff --git a/pkg/gitea/scan/scanner_test.go b/pkg/gitea/scan/scanner_test.go new file mode 100644 index 00000000..230218ce --- /dev/null +++ b/pkg/gitea/scan/scanner_test.go @@ -0,0 +1,143 @@ +package gitea + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/CompassSecurity/pipeleek/pkg/httpclient" +) + +// giteaMockServer returns an httptest.Server that satisfies the Gitea SDK's +// version check on NewClient (GET /api/v1/version → {"version":"1.20.0"}). +func giteaMockServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/version" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"version":"1.20.0"}`)) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestInitializeOptions_SDKClientInjected(t *testing.T) { + srv := giteaMockServer(t) + + opts, err := InitializeOptions( + "test-token", srv.URL, + "", "", "", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err != nil { + t.Fatalf("InitializeOptions returned error: %v", err) + } + + if opts.Client == nil { + t.Fatal("expected Gitea SDK client to be non-nil") + } + if opts.HttpClient == nil { + t.Fatal("expected HttpClient to be non-nil") + } +} + +func TestInitializeOptions_AuthHeaderInHttpClient(t *testing.T) { + srv := giteaMockServer(t) + + opts, err := InitializeOptions( + "my-token", srv.URL, + "", "", "", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err != nil { + t.Fatalf("InitializeOptions returned error: %v", err) + } + + // Auth header should be in Resty's client-level headers. + if opts.HttpClient.Header().Get("Authorization") != "token my-token" { + t.Errorf("expected Authorization header 'token my-token', got %q", opts.HttpClient.Header().Get("Authorization")) + } +} + +func TestInitializeOptions_CookiePathSetsJar(t *testing.T) { + srv := giteaMockServer(t) + + opts, err := InitializeOptions( + "my-token", srv.URL, + "", "", "mycookie", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err != nil { + t.Fatalf("InitializeOptions returned error: %v", err) + } + + if opts.HttpClient.CookieJar() == nil { + t.Error("expected cookie jar to be set when cookie is provided") + } +} + +func TestInitializeOptions_NoTransportMutation(t *testing.T) { + // Verify that the transport on the Resty client is Pipeleek's *http.Transport + // (not an AuthTransport or other wrapper — auth is handled via client headers). + srv := giteaMockServer(t) + + opts, err := InitializeOptions( + "tok", srv.URL, + "", "", "", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err != nil { + t.Fatalf("InitializeOptions returned error: %v", err) + } + + _, err = opts.HttpClient.HTTPTransport() + if err != nil { + t.Errorf("expected *http.Transport from Pipeleek client, got error: %v", err) + } +} + +func TestInitializeOptions_TLSReflectsGlobalConfig(t *testing.T) { + httpclient.SetInsecureSkipVerify(false) + t.Cleanup(func() { httpclient.SetInsecureSkipVerify(true) }) + + srv := giteaMockServer(t) + + opts, err := InitializeOptions( + "tok", srv.URL, + "", "", "", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err != nil { + t.Fatalf("InitializeOptions returned error: %v", err) + } + + tr, err := opts.HttpClient.HTTPTransport() + if err != nil { + t.Fatalf("expected *http.Transport, got error: %v", err) + } + if tr.TLSClientConfig == nil || tr.TLSClientConfig.InsecureSkipVerify { + t.Error("expected InsecureSkipVerify=false to be reflected from Pipeleek global config") + } +} + +func TestInitializeOptions_InvalidURL(t *testing.T) { + _, err := InitializeOptions( + "tok", "not-a-url", + "", "", "", "100Mb", + false, false, false, + 0, 0, 4, []string{}, 30*time.Second, + ) + if err == nil { + t.Fatal("expected error for invalid URL, got nil") + } +} diff --git a/pkg/gitea/variables/variables.go b/pkg/gitea/variables/variables.go index 06e5ce9b..1de76b2d 100644 --- a/pkg/gitea/variables/variables.go +++ b/pkg/gitea/variables/variables.go @@ -3,7 +3,6 @@ package variables import ( "encoding/json" "fmt" - "io" "code.gitea.io/sdk/gitea" "github.com/CompassSecurity/pipeleek/pkg/httpclient" @@ -199,21 +198,16 @@ func listRepoActionVariables(ctx *clientContext, owner, repo string, page, pageS authHeaders := map[string]string{"Authorization": "token " + ctx.token} httpClient := httpclient.GetPipeleekHTTPClient("", nil, authHeaders) - resp, err := httpClient.Get(url) + resp, err := httpClient.R().Get(url) if err != nil { return nil, fmt.Errorf("failed to execute request: %w", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != 200 { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode(), string(resp.Bytes())) } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } + body := resp.Bytes() var variables []*gitea.RepoActionVariable if err := json.Unmarshal(body, &variables); err != nil { diff --git a/pkg/github/scan/scanner.go b/pkg/github/scan/scanner.go index 292c0539..b7b23f54 100644 --- a/pkg/github/scan/scanner.go +++ b/pkg/github/scan/scanner.go @@ -2,7 +2,6 @@ package scan import ( "context" - "io" "net/http" "sort" "strings" @@ -19,9 +18,9 @@ import ( "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit" "github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit" "github.com/google/go-github/v69/github" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "resty.dev/v3" ) // ScanOptions contains configuration options for GitHub scanning operations. @@ -43,7 +42,7 @@ type ScanOptions struct { HitTimeout time.Duration Context context.Context Client *github.Client - HttpClient *retryablehttp.Client + HttpClient *resty.Client } type Scanner interface { @@ -68,7 +67,7 @@ func SetupClient(accessToken string, baseURL string) *github.Client { if baseURL == "" { baseURL = "https://api.github.com/" } - rateLimiter := github_ratelimit.New(nil, + rateLimiter := github_ratelimit.New(httpclient.GetPipeleekTransport(), github_primary_ratelimit.WithLimitDetectedCallback(func(ctx *github_primary_ratelimit.CallbackContext) { resetTime := ctx.ResetTime.Add(time.Duration(time.Second * 30)) log.Info().Str("category", string(ctx.Category)).Time("reset", resetTime).Msg("Primary rate limit detected, will resume automatically") @@ -418,19 +417,15 @@ func (s *scanner) downloadWorkflowRunLog(repo *github.Repository, workflowRun *g } func (s *scanner) downloadRunLogZIP(url string) []byte { - res, err := s.options.HttpClient.Get(url) + res, err := s.options.HttpClient.R().Get(url) logLines := make([]byte, 0) if err != nil { return logLines } - if res.StatusCode == 200 { - body, err := io.ReadAll(res.Body) - if err != nil { - log.Err(err).Msg("Failed reading response log body") - return logLines - } + if res.StatusCode() == 200 { + body := res.Bytes() zipResult, err := logline.ExtractLogsFromZip(body) if err != nil { @@ -532,19 +527,15 @@ func (s *scanner) analyzeArtifact(workflowRun *github.WorkflowRun, artifact *git return } - res, err := s.options.HttpClient.Get(url.String()) + res, err := s.options.HttpClient.R().Get(url.String()) if err != nil { log.Err(err).Str("workflow", url.String()).Msg("Failed downloading artifacts zip") return } - if res.StatusCode == 200 { - body, err := io.ReadAll(res.Body) - if err != nil { - log.Err(err).Msg("Failed reading response log body") - return - } + if res.StatusCode() == 200 { + body := res.Bytes() _, err = artifactproc.ProcessZipArtifact(body, artifactproc.ProcessOptions{ MaxGoRoutines: s.options.MaxScanGoRoutines, diff --git a/pkg/gitlab/enum/enum.go b/pkg/gitlab/enum/enum.go index 30f09f27..e55f1f8e 100644 --- a/pkg/gitlab/enum/enum.go +++ b/pkg/gitlab/enum/enum.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/CompassSecurity/pipeleek/pkg/gitlab/util" "github.com/rs/zerolog/log" gitlab "gitlab.com/gitlab-org/api/client-go" @@ -32,7 +33,7 @@ func RunEnum(gitlabUrl, gitlabApiToken string, minAccessLevel int) { log.Debug().Interface("full_user", user).Msg("Full User details") log.Info().Msg("Enumerating Access Token") - client := *resty.New().SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)) + client := *httpclient.GetPipeleekHTTPClient("", nil, nil).SetRedirectPolicy(resty.FlexibleRedirectPolicy(5)) enumCurrentToken(client, gitlabUrl, gitlabApiToken) log.Info().Msg("Enumerating Projects and Groups") diff --git a/pkg/gitlab/jobtoken/exploit.go b/pkg/gitlab/jobtoken/exploit.go index 7c85b711..ca2ef87d 100644 --- a/pkg/gitlab/jobtoken/exploit.go +++ b/pkg/gitlab/jobtoken/exploit.go @@ -73,7 +73,7 @@ func generateBranchName() (string, error) { } func newJobTokenClient(gitlabUrl, jobToken string) (*gitlab.Client, error) { - client := httpclient.GetPipeleekHTTPClient("", nil, nil).StandardClient() + client := httpclient.GetPipeleekHTTPClient("", nil, nil).Client() return gitlab.NewJobClient(jobToken, gitlab.WithBaseURL(gitlabUrl), gitlab.WithHTTPClient(client), diff --git a/pkg/gitlab/scan/queue.go b/pkg/gitlab/scan/queue.go index 9379bbcb..1977d172 100644 --- a/pkg/gitlab/scan/queue.go +++ b/pkg/gitlab/scan/queue.go @@ -246,7 +246,7 @@ func getJobTraceViaWeb(jobWebUrl string, options *ScanOptions) []byte { return nil } - client := httpclient.GetPipeleekHTTPClient(options.GitlabUrl, nil, nil).StandardClient() + client := httpclient.GetPipeleekHTTPClient(options.GitlabUrl, nil, nil).Client() req, err := http.NewRequest(http.MethodGet, rawURL, nil) if err != nil { log.Debug().Err(err).Str("url", rawURL).Msg("Failed building request for web trace") @@ -352,14 +352,13 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, // #nosec G124 - Cookie attributes (Secure/HttpOnly/SameSite) are server-side browser directives; not applicable for client HTTP requests client := httpclient.GetPipeleekHTTPClient(gitlabUrl, []*http.Cookie{{Name: "_gitlab_session", Value: cookieVal}}, nil) - resp, err := client.Get(dotenvUrl) + resp, err := client.R().Get(dotenvUrl) if err != nil { log.Debug().Stack().Err(err).Msg("Failed requesting dotenv artifact") return []byte{} } - defer func() { _ = resp.Body.Close() }() - statCode := resp.StatusCode + statCode := resp.StatusCode() // means no dotenv exists if statCode == 404 { @@ -371,11 +370,7 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, return []byte{} } - body, err := io.ReadAll(resp.Body) - if err != nil { - log.Debug().Err(err).Msg("Failed reading dotenv response body") - return []byte{} - } + body := resp.Bytes() kind, err := filetype.Match(body) if err != nil { diff --git a/pkg/gitlab/shodan/shodan.go b/pkg/gitlab/shodan/shodan.go index 0ef66fbe..7748be8b 100644 --- a/pkg/gitlab/shodan/shodan.go +++ b/pkg/gitlab/shodan/shodan.go @@ -104,17 +104,14 @@ func isRegistrationEnabled(base string) (bool, error) { s := u.String() client := httpclient.GetPipeleekHTTPClient("", nil, nil) - res, err := client.Get(s) + res, err := client.R().Get(s) if err != nil { return false, err } - if res.StatusCode == 200 { - resData, err := io.ReadAll(res.Body) - if err != nil { - return false, err - } + if res.StatusCode() == 200 { + resData := res.Bytes() // sanity check to avoid false positives if strings.Contains(string(resData), "{\"exists\":false}") { @@ -124,7 +121,7 @@ func isRegistrationEnabled(base string) (bool, error) { log.Debug().Msg("Missed sanity check") return false, err } else { - log.Debug().Int("http", res.StatusCode).Msg("Registration username test request") + log.Debug().Int("http", res.StatusCode()).Msg("Registration username test request") return false, nil } } @@ -138,16 +135,13 @@ func checkNrPublicRepos(base string) (int, error) { client := httpclient.GetPipeleekHTTPClient("", nil, nil) u.Path = "/api/v4/projects" s := u.String() - res, err := client.Get(s + "?per_page=100") + res, err := client.R().Get(s + "?per_page=100") if err != nil { return 0, err } - if res.StatusCode == 200 { - resData, err := io.ReadAll(res.Body) - if err != nil { - return 0, err - } + if res.StatusCode() == 200 { + resData := res.Bytes() var val []map[string]interface{} if err := json.Unmarshal(resData, &val); err != nil { return 0, err diff --git a/pkg/gitlab/snippets/scan/scanner.go b/pkg/gitlab/snippets/scan/scanner.go index e333273e..1c4359cd 100644 --- a/pkg/gitlab/snippets/scan/scanner.go +++ b/pkg/gitlab/snippets/scan/scanner.go @@ -2,7 +2,6 @@ package scan import ( "fmt" - "io" "net/http" "net/url" "strings" @@ -141,22 +140,16 @@ func (s *snippetsScanner) fetchFileContent(apiURL string) ([]byte, error) { "PRIVATE-TOKEN": s.options.GitlabToken, }) - resp, err := client.Get(apiURL) + resp, err := client.R().Get(apiURL) if err != nil { return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode()) } - content, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - return content, nil + return resp.Bytes(), nil } func (s *snippetsScanner) scanProjectByPath(git *gitlab.Client, projectPath string) error { diff --git a/pkg/gitlab/users/enum.go b/pkg/gitlab/users/enum.go index d1104a56..ed41d39e 100644 --- a/pkg/gitlab/users/enum.go +++ b/pkg/gitlab/users/enum.go @@ -540,7 +540,7 @@ func collectProjectMembersViaGraphQL(client *http.Client, graphqlURL, projectPat func buildGraphQLClient(git *gitlab.Client) (*http.Client, string) { apiBase := strings.TrimRight(git.BaseURL().String(), "/") graphqlURL := strings.TrimSuffix(apiBase, "/api/v4") + "/api/graphql" - client := httpclient.GetPipeleekHTTPClient(apiBase, nil, nil).StandardClient() + client := httpclient.GetPipeleekHTTPClient(apiBase, nil, nil).Client() return client, graphqlURL } diff --git a/pkg/gitlab/util/util.go b/pkg/gitlab/util/util.go index 1d581842..8b83a9fe 100644 --- a/pkg/gitlab/util/util.go +++ b/pkg/gitlab/util/util.go @@ -3,7 +3,6 @@ package util import ( "errors" "fmt" - "io" "net/http" "net/url" "path" @@ -12,10 +11,10 @@ import ( "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/PuerkitoBio/goquery" - "github.com/hashicorp/go-retryablehttp" "github.com/headzoo/surf" "github.com/rs/zerolog/log" gitlab "gitlab.com/gitlab-org/api/client-go" + "resty.dev/v3" ) // AccessLevelName returns the human-readable name for a GitLab access level value. @@ -98,7 +97,7 @@ func IterateGroupProjects(client *gitlab.Client, groupID interface{}, opts *gitl } func GetGitlabClient(token string, url string) (*gitlab.Client, error) { - return gitlab.NewClient(token, gitlab.WithBaseURL(url), gitlab.WithHTTPClient(httpclient.GetPipeleekHTTPClient("", nil, nil).StandardClient())) + return gitlab.NewClient(token, gitlab.WithBaseURL(url), gitlab.WithHTTPClient(httpclient.GetPipeleekHTTPClient("", nil, nil).Client())) } func CookieSessionValid(gitlabUrl string, cookieVal string) { @@ -106,13 +105,12 @@ func CookieSessionValid(gitlabUrl string, cookieVal string) { // #nosec G124 - Cookie attributes (Secure/HttpOnly/SameSite) are server-side browser directives; not applicable for client HTTP requests client := httpclient.GetPipeleekHTTPClient(gitlabUrl, []*http.Cookie{{Name: "_gitlab_session", Value: cookieVal}}, nil) - resp, err := client.Get(gitlabSessionsUrl) + resp, err := client.R().Get(gitlabSessionsUrl) if err != nil { log.Fatal().Stack().Err(err).Msg("Failed GitLab session test") } - defer func() { _ = resp.Body.Close() }() - statCode := resp.StatusCode + statCode := resp.StatusCode() if statCode != 200 { log.Fatal().Int("http", statCode).Str("testUrl", gitlabSessionsUrl).Msg("Invalid _gitlab_session, not auhthorized to access user sessions page for session validation") @@ -140,8 +138,8 @@ func DetermineVersion(gitlabUrl string, apiToken string) *gitlab.Metadata { } // fetchVersionFromHTML fetches the GitLab version by scraping the /help page HTML. -// Accepts a retryable HTTP client to allow injection for testing. -func fetchVersionFromHTML(gitlabUrl string, client *retryablehttp.Client) *gitlab.Metadata { +// Accepts a Resty client to allow injection for testing. +func fetchVersionFromHTML(gitlabUrl string, client *resty.Client) *gitlab.Metadata { u, err := url.Parse(gitlabUrl) if err != nil { log.Error().Stack().Err(err).Msg("Failed determining GitLab version via Website") @@ -149,18 +147,14 @@ func fetchVersionFromHTML(gitlabUrl string, client *retryablehttp.Client) *gitla } u.Path = path.Join(u.Path, "/help") - response, err := client.Get(u.String()) + response, err := client.R().Get(u.String()) if err != nil { log.Error().Stack().Err(err).Msg("Failed determining GitLab version via Website") return &gitlab.Metadata{Version: "none", Revision: "none", Enterprise: false} } - responseData, err := io.ReadAll(response.Body) - if err != nil { - log.Error().Stack().Err(err).Msg("Failed determining GitLab version via Website") - return &gitlab.Metadata{Version: "none", Revision: "none", Enterprise: false} - } + responseData := response.Bytes() extractLineR := regexp.MustCompile(`instance_version":"\d*.\d*.\d*"`) fullLine := extractLineR.Find(responseData) diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 890eb5ac..caa3da9b 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -1,18 +1,22 @@ // Package httpclient provides a centralized HTTP client configuration for pipeleek. -// It offers a retryable HTTP client with cookie support, custom headers, and proxy configuration. +// It offers a retryable HTTP client with cookie support, custom headers, proxy +// configuration, TLS settings, and SOCKS proxy support. package httpclient import ( - "context" "crypto/tls" + "net" "net/http" "net/http/cookiejar" "net/url" "os" + "sync" "sync/atomic" + "time" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" + "golang.org/x/net/proxy" + "resty.dev/v3" ) // ignoreProxy controls whether the HTTP_PROXY environment variable should be ignored. @@ -26,6 +30,55 @@ func SetIgnoreProxy(ignore bool) { ignoreProxy.Store(ignore) } +// httpClientConfig holds centrally configurable HTTP transport options. +// All fields are safe to read after the mutex is acquired. +type httpClientConfig struct { + insecureSkipVerify bool + proxyURL string + timeout time.Duration +} + +var ( + configMu sync.RWMutex + globalConfig = httpClientConfig{ + // Default true: scanning tools routinely target self-hosted instances + // with self-signed certificates. + insecureSkipVerify: true, + } +) + +// SetInsecureSkipVerify controls TLS certificate verification for all Pipeleek-managed +// HTTP clients. Defaults to true (skip verification) to support self-hosted targets with +// self-signed certificates. Set to false to enforce certificate validation. +func SetInsecureSkipVerify(skip bool) { + configMu.Lock() + defer configMu.Unlock() + globalConfig.insecureSkipVerify = skip +} + +// SetProxy sets a proxy URL for all Pipeleek-managed HTTP clients. Accepts both +// HTTP ("http://host:port") and SOCKS5 ("socks5://host:port") URLs. When non-empty, +// it takes precedence over the HTTP_PROXY environment variable. +func SetProxy(proxyURL string) { + configMu.Lock() + defer configMu.Unlock() + globalConfig.proxyURL = proxyURL +} + +// SetHTTPTimeout sets the per-request timeout applied to all Pipeleek-managed HTTP clients. +// A zero value (the default) means no timeout. +func SetHTTPTimeout(d time.Duration) { + configMu.Lock() + defer configMu.Unlock() + globalConfig.timeout = d +} + +func readGlobalConfig() httpClientConfig { + configMu.RLock() + defer configMu.RUnlock() + return globalConfig +} + // HeaderRoundTripper is an http.RoundTripper that adds default headers to requests. // Headers are only added if they're not already present in the request. type HeaderRoundTripper struct { @@ -51,80 +104,131 @@ func (hrt *HeaderRoundTripper) RoundTrip(req *http.Request) (*http.Response, err return hrt.Next.RoundTrip(req) } -// GetPipeleekHTTPClient creates and configures a retryable HTTP client for pipeleek operations. +// buildTransport constructs an *http.Transport with TLS, proxy, and SOCKS settings +// taken from the provided config snapshot. +func buildTransport(cfg httpClientConfig) *http.Transport { + // #nosec G402 - InsecureSkipVerify is user-configurable; defaults to true so that + // scanning tools can reach self-hosted instances with self-signed certificates. + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: cfg.insecureSkipVerify}, + } + + if cfg.proxyURL != "" { + u, err := url.Parse(cfg.proxyURL) + if err != nil { + log.Fatal().Err(err).Str("proxy", cfg.proxyURL).Msg("Invalid proxy URL") + } + switch u.Scheme { + case "socks5", "socks5h": + // Use the configured timeout for the dialer so that unreachable SOCKS proxies + // do not cause indefinite hangs. Fall back to 30 s when no timeout is set. + dialTimeout := cfg.timeout + if dialTimeout <= 0 { + dialTimeout = 30 * time.Second + } + dialer, err := proxy.FromURL(u, &net.Dialer{Timeout: dialTimeout}) + if err != nil { + log.Fatal().Err(err).Str("proxy", cfg.proxyURL).Msg("Failed creating SOCKS proxy dialer") + } + if cd, ok := dialer.(proxy.ContextDialer); ok { + tr.DialContext = cd.DialContext + } else { + //nolint:staticcheck + tr.Dial = dialer.Dial + } + log.Info().Str("proxy", cfg.proxyURL).Msg("Using SOCKS proxy") + default: + tr.Proxy = http.ProxyURL(u) + log.Info().Str("proxy", cfg.proxyURL).Msg("Using HTTP proxy") + } + return tr + } + + if !ignoreProxy.Load() { + proxyServer, useHttpProxy := os.LookupEnv("HTTP_PROXY") + if useHttpProxy { + proxyUrl, err := url.Parse(proxyServer) + if err != nil { + log.Fatal().Err(err).Str("HTTP_PROXY", proxyServer).Msg("Invalid Proxy URL in HTTP_PROXY environment variable") + } + log.Info().Str("proxy", proxyUrl.String()).Msg("Using HTTP_PROXY") + tr.Proxy = http.ProxyURL(proxyUrl) + } + } + + return tr +} + +// GetPipeleekTransport returns a configured *http.Transport using the current global +// client options (TLS, proxy, SOCKS). Use this to inject Pipeleek's transport settings +// into third-party HTTP client libraries (e.g. Resty, go-github) that manage their own +// request lifecycle but should still share the same network configuration. +func GetPipeleekTransport() *http.Transport { + return buildTransport(readGlobalConfig()) +} + +// GetPipeleekHTTPClient creates and configures a Resty HTTP client for pipeleek operations. // It supports: // - Cookie jar configuration for session management // - Custom default headers -// - Automatic retry logic for 429 and 5xx errors (except 501) +// - Automatic retry logic for 429 and 5xx errors (except 501), and transient network errors // - HTTP proxy support via HTTP_PROXY environment variable (unless SetIgnoreProxy(true) is called) -// - TLS certificate verification bypass (InsecureSkipVerify) +// - Proxy support via SetProxy (HTTP and SOCKS5; takes precedence over HTTP_PROXY) +// - Configurable TLS certificate verification (SetInsecureSkipVerify; defaults to true) +// - Configurable per-request timeout (SetHTTPTimeout; defaults to no timeout) // // Parameters: // - cookieUrl: The URL to associate cookies with (required if cookies are provided) // - cookies: Optional cookies to add to the jar // - defaultHeaders: Optional headers to add to all requests // -// Returns a configured *retryablehttp.Client ready for use. -func GetPipeleekHTTPClient(cookieUrl string, cookies []*http.Cookie, defaultHeaders map[string]string) *retryablehttp.Client { - var jar http.CookieJar +// Returns a configured *resty.Client ready for use. +func GetPipeleekHTTPClient(cookieUrl string, cookies []*http.Cookie, defaultHeaders map[string]string) *resty.Client { + cfg := readGlobalConfig() + + client := resty.New() if len(cookies) > 0 { - var err error - jar, err = cookiejar.New(nil) + jar, err := cookiejar.New(nil) if err != nil { log.Fatal().Err(err).Msg("Failed creating cookie jar") } - urlParsed, err := url.Parse(cookieUrl) if err != nil { log.Fatal().Err(err).Msg("Failed parsing URL for cookie jar") } - jar.SetCookies(urlParsed, cookies) + client.SetCookieJar(jar) + } + + if len(defaultHeaders) > 0 { + client.SetHeaders(defaultHeaders) } - client := retryablehttp.NewClient() - client.Logger = nil - client.HTTPClient.Jar = jar + if cfg.timeout > 0 { + client.SetTimeout(cfg.timeout) + } + + client.SetTransport(buildTransport(cfg)) - client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + client.SetRetryCount(4) + client.SetRetryWaitTime(1 * time.Second) + client.SetRetryMaxWaitTime(30 * time.Second) + client.EnableRetryDefaultConditions() + client.AddRetryHooks(func(r *resty.Response, err error) { if err != nil { log.Error().Err(err).Msg("Retrying HTTP request, error occurred") - return true, nil - } - - if resp == nil { - log.Error().Msg("Retrying HTTP request, no response") - return false, nil + return } - - if resp.StatusCode == 429 || (resp.StatusCode >= 500 && resp.StatusCode != 501) { - url := "" - if resp.Request != nil && resp.Request.URL != nil { - url = resp.Request.URL.String() - } - log.Trace().Str("url", url).Int("statusCode", resp.StatusCode).Msg("Retrying HTTP request") - return true, nil + if r == nil { + return } - - return false, nil - } - - // #nosec G402 - InsecureSkipVerify required for security scanning tool to connect to untrusted targets - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - - if !ignoreProxy.Load() { - proxyServer, useHttpProxy := os.LookupEnv("HTTP_PROXY") - if useHttpProxy { - proxyUrl, err := url.Parse(proxyServer) - if err != nil { - log.Fatal().Err(err).Str("HTTP_PROXY", proxyServer).Msg("Invalid Proxy URL in HTTP_PROXY environment variable") - } - log.Info().Str("proxy", proxyUrl.String()).Msg("Using HTTP_PROXY") - tr.Proxy = http.ProxyURL(proxyUrl) + reqURL := "" + if r.RawResponse != nil && r.RawResponse.Request != nil && r.RawResponse.Request.URL != nil { + reqURL = r.RawResponse.Request.URL.String() } - } + log.Trace().Str("url", reqURL).Int("statusCode", r.StatusCode()).Msg("Retrying HTTP request") + }) - client.HTTPClient.Transport = &HeaderRoundTripper{Headers: defaultHeaders, Next: tr} return client } diff --git a/pkg/httpclient/client_test.go b/pkg/httpclient/client_test.go index 80eb1460..9d7d8ba3 100644 --- a/pkg/httpclient/client_test.go +++ b/pkg/httpclient/client_test.go @@ -5,6 +5,8 @@ import ( "net/http" "net/http/httptest" "testing" + "time" + ) func TestHeaderRoundTripper_RoundTrip(t *testing.T) { @@ -86,10 +88,6 @@ func TestGetPipeleekHTTPClient(t *testing.T) { client := GetPipeleekHTTPClient("", nil, nil) if client == nil { t.Fatal("Expected non-nil client") - return - } - if client.Logger != nil { - t.Error("Expected logger to be nil") } }) @@ -102,14 +100,8 @@ func TestGetPipeleekHTTPClient(t *testing.T) { t.Fatal("Expected non-nil client") return } - - hrt, ok := client.HTTPClient.Transport.(*HeaderRoundTripper) - if !ok { - t.Fatal("Expected HeaderRoundTripper transport") - } - - if hrt.Headers["User-Agent"] != "test-agent" { - t.Errorf("Expected User-Agent header to be 'test-agent', got %q", hrt.Headers["User-Agent"]) + if client.Header().Get("User-Agent") != "test-agent" { + t.Errorf("Expected User-Agent header to be 'test-agent', got %q", client.Header().Get("User-Agent")) } }) @@ -122,42 +114,22 @@ func TestGetPipeleekHTTPClient(t *testing.T) { t.Fatal("Expected non-nil client") return } - if client.HTTPClient.Jar == nil { + if client.CookieJar() == nil { t.Error("Expected cookie jar to be set") } }) - t.Run("check retry function", func(t *testing.T) { + t.Run("check retry conditions", func(t *testing.T) { client := GetPipeleekHTTPClient("", nil, nil) - shouldRetry, _ := client.CheckRetry(nil, &http.Response{StatusCode: 429}, nil) - if !shouldRetry { - t.Error("Expected to retry on 429 status") - } - - shouldRetry, _ = client.CheckRetry(nil, &http.Response{StatusCode: 500}, nil) - if !shouldRetry { - t.Error("Expected to retry on 500 status") + // Resty built-in defaults are enabled instead of a custom condition + if !client.IsRetryDefaultConditions() { + t.Error("Expected Resty default retry conditions to be enabled") } - shouldRetry, _ = client.CheckRetry(nil, &http.Response{StatusCode: 501}, nil) - if shouldRetry { - t.Error("Expected NOT to retry on 501 status") - } - - shouldRetry, _ = client.CheckRetry(nil, &http.Response{StatusCode: 200}, nil) - if shouldRetry { - t.Error("Expected NOT to retry on 200 status") - } - - shouldRetry, _ = client.CheckRetry(nil, nil, nil) - if shouldRetry { - t.Error("Expected NOT to retry with nil response") - } - - shouldRetry, _ = client.CheckRetry(nil, nil, http.ErrServerClosed) - if !shouldRetry { - t.Error("Expected to retry on error") + // No custom retry conditions should be registered + if len(client.RetryConditions()) != 0 { + t.Error("Expected no custom retry conditions — defaults handle 429/5xx/errors") } }) } @@ -181,10 +153,7 @@ func TestSetIgnoreProxy(t *testing.T) { }) t.Run("proxy is ignored when SetIgnoreProxy is true", func(t *testing.T) { - // Set HTTP_PROXY using t.Setenv for automatic cleanup t.Setenv("HTTP_PROXY", "http://127.0.0.1:8888") - - // Set ignoreProxy to true SetIgnoreProxy(true) client := GetPipeleekHTTPClient("", nil, nil) @@ -192,28 +161,18 @@ func TestSetIgnoreProxy(t *testing.T) { t.Fatal("Expected non-nil client") } - // Get the transport - hrt, ok := client.HTTPClient.Transport.(*HeaderRoundTripper) - if !ok { - t.Fatal("Expected HeaderRoundTripper transport") - } - - tr, ok := hrt.Next.(*http.Transport) - if !ok { - t.Fatal("Expected http.Transport as next transport") + tr, err := client.HTTPTransport() + if err != nil { + t.Fatalf("Expected *http.Transport, got error: %v", err) } - // When ignoreProxy is true, Proxy should not be set if tr.Proxy != nil { t.Error("Expected Proxy to be nil when ignoreProxy is true") } }) t.Run("proxy is used when SetIgnoreProxy is false", func(t *testing.T) { - // Set HTTP_PROXY using t.Setenv for automatic cleanup t.Setenv("HTTP_PROXY", "http://127.0.0.1:8888") - - // Set ignoreProxy to false SetIgnoreProxy(false) client := GetPipeleekHTTPClient("", nil, nil) @@ -221,20 +180,156 @@ func TestSetIgnoreProxy(t *testing.T) { t.Fatal("Expected non-nil client") } - // Get the transport - hrt, ok := client.HTTPClient.Transport.(*HeaderRoundTripper) - if !ok { - t.Fatal("Expected HeaderRoundTripper transport") + tr, err := client.HTTPTransport() + if err != nil { + t.Fatalf("Expected *http.Transport, got error: %v", err) + } + + if tr.Proxy == nil { + t.Error("Expected Proxy to be set when ignoreProxy is false and HTTP_PROXY is set") + } + }) +} + +// saveAndRestoreConfig saves the current global config and returns a cleanup function. +func saveAndRestoreConfig(t *testing.T) func() { + t.Helper() + configMu.RLock() + saved := globalConfig + configMu.RUnlock() + return func() { + configMu.Lock() + globalConfig = saved + configMu.Unlock() + } +} + +func TestSetInsecureSkipVerify(t *testing.T) { + restore := saveAndRestoreConfig(t) + defer restore() + + t.Run("default is true (skip verification)", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetInsecureSkipVerify(true) + client := GetPipeleekHTTPClient("", nil, nil) + tr, err := client.HTTPTransport() + if err != nil { + t.Fatalf("Expected *http.Transport, got error: %v", err) + } + if !tr.TLSClientConfig.InsecureSkipVerify { + t.Error("Expected InsecureSkipVerify to be true") + } + }) + + t.Run("can be disabled to enforce TLS verification", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetInsecureSkipVerify(false) + client := GetPipeleekHTTPClient("", nil, nil) + tr, err := client.HTTPTransport() + if err != nil { + t.Fatalf("Expected *http.Transport, got error: %v", err) } + if tr.TLSClientConfig.InsecureSkipVerify { + t.Error("Expected InsecureSkipVerify to be false") + } + }) - tr, ok := hrt.Next.(*http.Transport) - if !ok { - t.Fatal("Expected http.Transport as next transport") + t.Run("GetPipeleekTransport reflects the setting", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetInsecureSkipVerify(false) + tr := GetPipeleekTransport() + if tr.TLSClientConfig.InsecureSkipVerify { + t.Error("Expected GetPipeleekTransport InsecureSkipVerify to be false") } + }) +} + +func TestSetHTTPTimeout(t *testing.T) { + restore := saveAndRestoreConfig(t) + defer restore() + + t.Run("zero timeout means no timeout", func(t *testing.T) { + SetHTTPTimeout(0) + client := GetPipeleekHTTPClient("", nil, nil) + if client.Timeout() != 0 { + t.Errorf("Expected zero timeout, got %v", client.Timeout()) + } + }) + + t.Run("non-zero timeout is set on the client", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + want := 30 * time.Second + SetHTTPTimeout(want) + client := GetPipeleekHTTPClient("", nil, nil) + if client.Timeout() != want { + t.Errorf("Expected timeout %v, got %v", want, client.Timeout()) + } + }) +} - // When ignoreProxy is false and HTTP_PROXY is set, Proxy should be set +func TestSetProxy(t *testing.T) { + restore := saveAndRestoreConfig(t) + defer restore() + + t.Run("valid SOCKS5 proxy sets a dial function on the transport", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetProxy("socks5://127.0.0.1:1080") + tr := GetPipeleekTransport() + if tr.DialContext == nil && tr.Dial == nil { //nolint:staticcheck + t.Error("Expected DialContext or Dial to be set for SOCKS5 proxy") + } + }) + + t.Run("valid HTTP proxy sets Proxy on the transport", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetProxy("http://127.0.0.1:8080") + tr := GetPipeleekTransport() if tr.Proxy == nil { - t.Error("Expected Proxy to be set when ignoreProxy is false and HTTP_PROXY is set") + t.Error("Expected Proxy to be set for HTTP proxy") + } + }) + + t.Run("empty proxy clears the setting", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetProxy("") + // With no proxy, transport should have no dialer and no Proxy func set + tr := GetPipeleekTransport() + if tr.DialContext != nil { + t.Error("Expected DialContext to be nil when no proxy is configured") + } + }) +} + +func TestGetPipeleekTransport(t *testing.T) { + restore := saveAndRestoreConfig(t) + defer restore() + + t.Run("returns non-nil transport", func(t *testing.T) { + tr := GetPipeleekTransport() + if tr == nil { + t.Fatal("Expected non-nil transport") + } + }) + + t.Run("transport TLS config matches InsecureSkipVerify setting", func(t *testing.T) { + restore2 := saveAndRestoreConfig(t) + defer restore2() + SetInsecureSkipVerify(true) + tr := GetPipeleekTransport() + if !tr.TLSClientConfig.InsecureSkipVerify { + t.Error("Expected InsecureSkipVerify true on transport") + } + SetInsecureSkipVerify(false) + tr = GetPipeleekTransport() + if tr.TLSClientConfig.InsecureSkipVerify { + t.Error("Expected InsecureSkipVerify false on transport") } }) } diff --git a/pkg/jenkins/scan/client.go b/pkg/jenkins/scan/client.go index 2616ba12..05b11f24 100644 --- a/pkg/jenkins/scan/client.go +++ b/pkg/jenkins/scan/client.go @@ -7,6 +7,7 @@ import ( "path" "strings" + "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/bndr/gojenkins" ) @@ -25,7 +26,7 @@ type goJenkinsClient struct { func NewClient(serverURL, username, token string) JenkinsClient { base := normalizeBaseURL(serverURL) - jenkins := gojenkins.CreateJenkins(nil, base, username, token) + jenkins := gojenkins.CreateJenkins(httpclient.GetPipeleekHTTPClient("", nil, nil).Client(), base, username, token) return &goJenkinsClient{jenkins: jenkins} } diff --git a/pkg/nist/nist.go b/pkg/nist/nist.go index 494b0a60..cd673ad8 100644 --- a/pkg/nist/nist.go +++ b/pkg/nist/nist.go @@ -4,11 +4,10 @@ package nist import ( "encoding/json" "fmt" - "io" "os" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" + "resty.dev/v3" ) const resultsPerPage = 100 @@ -27,9 +26,9 @@ var PIPELEEK_NIST_BASE_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0" // FetchVulns retrieves all CVE vulnerabilities for a specific CPE name from the NIST NVD API. // It automatically handles pagination if the total results exceed the page size. -// Accepts a retryablehttp client, base URL, and full CPE name to allow dependency injection for testing. +// Accepts a resty client, base URL, and full CPE name to allow dependency injection for testing. // CPE name should be in format: cpe:2.3:a:vendor:product:version:*:*:*:edition:*:*:* -func FetchVulns(client *retryablehttp.Client, cpeName string) (string, error) { +func FetchVulns(client *resty.Client, cpeName string) (string, error) { baseURL := PIPELEEK_NIST_BASE_URL // Allow overriding NIST base URL via environment variable (primarily for testing) @@ -82,23 +81,18 @@ func FetchVulns(client *retryablehttp.Client, cpeName string) (string, error) { return string(jsonData), nil } -func fetchPage(client *retryablehttp.Client, url string) (*nvdResponse, error) { - res, err := client.Get(url) +func fetchPage(client *resty.Client, url string) (*nvdResponse, error) { + res, err := client.R().Get(url) if err != nil { return nil, err } - defer func() { _ = res.Body.Close() }() - if res.StatusCode != 200 { - log.Error().Int("http", res.StatusCode).Str("url", url).Msg("failed fetching vulnerabilities") - return nil, fmt.Errorf("HTTP %d", res.StatusCode) + if res.StatusCode() != 200 { + log.Error().Int("http", res.StatusCode()).Str("url", url).Msg("failed fetching vulnerabilities") + return nil, fmt.Errorf("HTTP %d", res.StatusCode()) } - resData, err := io.ReadAll(res.Body) - if err != nil { - log.Error().Int("http", res.StatusCode).Msg("unable to read HTTP response body") - return nil, err - } + resData := res.Bytes() var nvdResp nvdResponse if err := json.Unmarshal(resData, &nvdResp); err != nil { diff --git a/pkg/nist/nist_test.go b/pkg/nist/nist_test.go index a418c638..aa9c35ca 100644 --- a/pkg/nist/nist_test.go +++ b/pkg/nist/nist_test.go @@ -7,9 +7,9 @@ import ( "net/http/httptest" "testing" - "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "resty.dev/v3" ) // mockNVDServer creates a test HTTP server that simulates the NVD API @@ -69,8 +69,8 @@ func TestFetchVulns_NoPagination(t *testing.T) { t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) // Create a properly configured retryable client - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") require.NoError(t, err) @@ -93,8 +93,8 @@ func TestFetchVulns_WithPagination(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") require.NoError(t, err) @@ -128,8 +128,8 @@ func TestFetchVulns_EmptyResponse(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:99.99.99:*:*:*:*:*:*:*") require.NoError(t, err) @@ -151,9 +151,9 @@ func TestFetchVulns_HTTPError(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() - client.RetryMax = 0 // Disable retries for faster test + client := resty.New() + + client.SetRetryCount(0) // Disable retries for faster test result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") assert.Error(t, err) assert.Equal(t, "{}", result) @@ -170,8 +170,8 @@ func TestFetchVulns_InvalidJSON(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") assert.Error(t, err) assert.Equal(t, "{}", result) @@ -185,8 +185,8 @@ func TestFetchVulns_LargePagination(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") require.NoError(t, err) @@ -215,8 +215,8 @@ func TestFetchVulns_ExactPageBoundary(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") require.NoError(t, err) @@ -236,8 +236,8 @@ func TestFetchVulns_MultiplePagesExactBoundary(t *testing.T) { // Set the environment variable for the test t.Setenv("PIPELEEK_NIST_BASE_URL", server.URL) - client := retryablehttp.NewClient() - client.HTTPClient = server.Client() + client := resty.New() + result, err := FetchVulns(client, "cpe:2.3:a:example:product:1.0.0:*:*:*:*:*:*:*") require.NoError(t, err) diff --git a/pkg/renovate/common.go b/pkg/renovate/common.go index dd8a5cc2..b3d5bdce 100644 --- a/pkg/renovate/common.go +++ b/pkg/renovate/common.go @@ -3,15 +3,14 @@ package renovate import ( "encoding/json" "fmt" - "io" "net/url" "regexp" "strings" "github.com/CompassSecurity/pipeleek/pkg/format" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" "github.com/yosuke-furukawa/json5/encoding/json5" + "resty.dev/v3" ) // DetectCiCdConfig checks if the CI/CD configuration contains Renovate bot references. @@ -114,31 +113,25 @@ func DetectAutodiscoveryFilters(cicdConf, configFileContent string) (bool, strin } // FetchCurrentSelfHostedOptions retrieves the list of self-hosted Renovate configuration options. -// Accepts a retryable HTTP client to allow injection for testing. -func FetchCurrentSelfHostedOptions(cachedOptions []string, client *retryablehttp.Client) []string { +// Accepts a Resty client to allow injection for testing. +func FetchCurrentSelfHostedOptions(cachedOptions []string, client *resty.Client) []string { if len(cachedOptions) > 0 { return cachedOptions } log.Debug().Msg("Fetching current self-hosted configuration from GitHub") - res, err := client.Get("https://raw.githubusercontent.com/renovatebot/renovate/refs/heads/main/docs/usage/self-hosted-configuration.md") + res, err := client.R().Get("https://raw.githubusercontent.com/renovatebot/renovate/refs/heads/main/docs/usage/self-hosted-configuration.md") if err != nil { log.Error().Stack().Err(err).Msg("Failed fetching self-hosted configuration documentation") return []string{} } - defer func() { _ = res.Body.Close() }() - if res.StatusCode != 200 { - log.Error().Int("status", res.StatusCode).Msg("Failed fetching self-hosted configuration documentation") - return []string{} - } - data, err := io.ReadAll(res.Body) - if err != nil { - log.Error().Stack().Err(err).Msg("Failed reading self-hosted configuration documentation") + if res.StatusCode() != 200 { + log.Error().Int("status", res.StatusCode()).Msg("Failed fetching self-hosted configuration documentation") return []string{} } - return ExtractSelfHostedOptions(data) + return ExtractSelfHostedOptions(res.Bytes()) } // ExtractSelfHostedOptions parses self-hosted options from documentation content. @@ -166,8 +159,8 @@ func IsSelfHostedConfig(config string, selfHostedOptions []string) bool { // ExtendRenovateConfig extends a Renovate configuration by sending it to a resolver service. // The config is normalized to valid JSON before sending (removes JSON5 comments/trailing commas). -// Accepts a retryable HTTP client to allow injection for testing. -func ExtendRenovateConfig(renovateConfig string, serviceURL string, projectURL string, client *retryablehttp.Client) string { +// Accepts a Resty client to allow injection for testing. +func ExtendRenovateConfig(renovateConfig string, serviceURL string, projectURL string, client *resty.Client) string { u, err := url.Parse(serviceURL) if err != nil { log.Error().Stack().Err(err).Str("project", projectURL).Msg("Failed to parse renovate config service URL") @@ -177,23 +170,20 @@ func ExtendRenovateConfig(renovateConfig string, serviceURL string, projectURL s normalizedConfig := normalizeRenovateConfig(renovateConfig) - resp, err := client.Post(u.String(), "application/json", strings.NewReader(normalizedConfig)) + resp, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(normalizedConfig). + Post(u.String()) if err != nil { log.Error().Stack().Err(err).Str("project", projectURL).Msg("Failed to extend renovate config") return renovateConfig } - defer func() { _ = resp.Body.Close() }() - - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - log.Error().Stack().Err(err).Str("project", projectURL).Msg("Failed to read response body of renovate config expansion") - return renovateConfig - } + bodyBytes := resp.Bytes() - if resp.StatusCode != 200 { - log.Debug().Int("status", resp.StatusCode).Str("msg", string(bodyBytes)).Str("project", projectURL).Msg("Failed to extend renovate config") + if resp.StatusCode() != 200 { + log.Debug().Int("status", resp.StatusCode()).Str("msg", string(bodyBytes)).Str("project", projectURL).Msg("Failed to extend renovate config") return renovateConfig } @@ -228,8 +218,8 @@ func normalizeRenovateConfig(config string) string { } // ValidateRenovateConfigService checks if the Renovate config resolver service is available. -// Accepts a retryable HTTP client to allow injection for testing. -func ValidateRenovateConfigService(serviceUrl string, client *retryablehttp.Client) error { +// Accepts a Resty client to allow injection for testing. +func ValidateRenovateConfigService(serviceUrl string, client *resty.Client) error { u, err := url.Parse(serviceUrl) if err != nil { log.Error().Stack().Err(err).Msg("Failed to parse renovate config service URL") @@ -237,16 +227,16 @@ func ValidateRenovateConfigService(serviceUrl string, client *retryablehttp.Clie } u = u.JoinPath("health") - resp, err := client.Get(u.String()) + resp, err := client.R().Get(u.String()) if err != nil { log.Error().Stack().Err(err).Msg("Renovate config service healthcheck failed") return err } - if resp.StatusCode != 200 { - log.Error().Int("status", resp.StatusCode).Str("endpoint", u.String()).Msg("Renovate config service healthcheck failed") - return fmt.Errorf("renovate config service healthcheck failed: %d", resp.StatusCode) + if resp.StatusCode() != 200 { + log.Error().Int("status", resp.StatusCode()).Str("endpoint", u.String()).Msg("Renovate config service healthcheck failed") + return fmt.Errorf("renovate config service healthcheck failed: %d", resp.StatusCode()) } return nil diff --git a/pkg/renovate/common_test.go b/pkg/renovate/common_test.go index c41e872b..92a8cf10 100644 --- a/pkg/renovate/common_test.go +++ b/pkg/renovate/common_test.go @@ -419,7 +419,7 @@ func TestFetchCurrentSelfHostedOptions_ParsesResponse(t *testing.T) { defer srv.Close() client := httpclient.GetPipeleekHTTPClient("", nil, nil) - client.HTTPClient.Transport = &redirectTransport{targetURL: srv.URL} + client.SetTransport(&redirectTransport{targetURL: srv.URL}) result := FetchCurrentSelfHostedOptions([]string{}, client) assert.NotEmpty(t, result) @@ -429,13 +429,13 @@ func TestFetchCurrentSelfHostedOptions_ParsesResponse(t *testing.T) { // when the server responds with a non-200 status. func TestFetchCurrentSelfHostedOptions_Non200(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Use 404: the retryablehttp client does not retry 4xx client errors by default + // Use 404: Resty does not retry 4xx client errors by default w.WriteHeader(http.StatusNotFound) })) defer srv.Close() client := httpclient.GetPipeleekHTTPClient("", nil, nil) - client.HTTPClient.Transport = &redirectTransport{targetURL: srv.URL} + client.SetTransport(&redirectTransport{targetURL: srv.URL}) result := FetchCurrentSelfHostedOptions([]string{}, client) assert.Empty(t, result) @@ -466,7 +466,7 @@ func TestExtendRenovateConfig_BadURL(t *testing.T) { // TestExtendRenovateConfig_RequestError verifies that the original config is returned on error. func TestExtendRenovateConfig_RequestError(t *testing.T) { client := httpclient.GetPipeleekHTTPClient("", nil, nil) - client.RetryMax = 0 + client.SetRetryCount(0) orig := `{"extends":["config:base"]}` result := ExtendRenovateConfig(orig, "http://127.0.0.1:0", "https://project.example.com", client) assert.Equal(t, orig, result) @@ -488,7 +488,7 @@ func TestValidateRenovateConfigService_Success(t *testing.T) { // TestValidateRenovateConfigService_Non200 verifies that a non-200 response returns an error. func TestValidateRenovateConfigService_Non200(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Use 404: the retryablehttp client does not retry 4xx client errors by default + // Use 404: Resty does not retry 4xx client errors by default w.WriteHeader(http.StatusNotFound) })) defer srv.Close() @@ -508,7 +508,7 @@ func TestValidateRenovateConfigService_BadURL(t *testing.T) { // TestValidateRenovateConfigService_Unreachable verifies that an unreachable host returns an error. func TestValidateRenovateConfigService_Unreachable(t *testing.T) { client := httpclient.GetPipeleekHTTPClient("", nil, nil) - client.RetryMax = 0 + client.SetRetryCount(0) err := ValidateRenovateConfigService("http://127.0.0.1:0", client) assert.Error(t, err) } diff --git a/pkg/scanner/rules/rules.go b/pkg/scanner/rules/rules.go index b9088e47..e472d181 100644 --- a/pkg/scanner/rules/rules.go +++ b/pkg/scanner/rules/rules.go @@ -2,18 +2,18 @@ package rules import ( "errors" - "io" + "fmt" "os" "slices" "strings" "github.com/CompassSecurity/pipeleek/pkg/httpclient" "github.com/CompassSecurity/pipeleek/pkg/scanner/types" - "github.com/hashicorp/go-retryablehttp" "github.com/rs/zerolog/log" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults" "gopkg.in/yaml.v3" + "resty.dev/v3" ) var ruleFile = "https://raw.githubusercontent.com/mazen160/secrets-patterns-db/master/db/rules-stable.yml" @@ -33,26 +33,18 @@ func DownloadRules() { } } -func downloadFile(url string, filepath string, client *retryablehttp.Client) error { - // #nosec G304 - Creating file for rules download at controlled internal temp path - out, err := os.Create(filepath) +func downloadFile(url string, filepath string, client *resty.Client) error { + resp, err := client.R().Get(url) if err != nil { return err } - defer func() { _ = out.Close() }() - resp, err := client.Get(url) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - - _, err = io.Copy(out, resp.Body) - if err != nil { - return err + if resp.StatusCode() != 200 { + return fmt.Errorf("unexpected status %d downloading rules file", resp.StatusCode()) } - return nil + // #nosec G304 - Writing rules download at controlled internal temp path + return os.WriteFile(filepath, resp.Bytes(), 0600) } func InitRules(confidenceFilter []string) {