PoC for lightweight analysis for OpenAI

pkg/detectors/datadogtoken/datadogtoken.go

@@ -9,7 +9,9 @@ import (
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	lwa "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lightweight_analyze"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
@@ -101,6 +103,7 @@ func (s Scanner) Keywords() []string {
 
 // FromData will find and optionally verify DatadogToken secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	logCtx := logContext.AddLogger(ctx)
 	dataStr := string(data)
 
 	appMatches := appPat.FindAllStringSubmatch(dataStr, -1)
@@ -139,12 +142,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					req.Header.Add("DD-APPLICATION-KEY", resAppMatch)
 					res, err := client.Do(req)
 					if err == nil {
-						defer res.Body.Close()
+						// lightweight analyze: unconditionally preserve the response body
+						resBody := lwa.CopyAndCloseResponseBody(logCtx, s1.ExtraData, res)
+
 						if res.StatusCode >= 200 && res.StatusCode < 300 {
 							s1.Verified = true
 							s1.AnalysisInfo = map[string]string{"api_key": resApiMatch, "app_key": resAppMatch, "endpoint": baseURL}
 							var serviceResponse userServiceResponse
-							if err := json.NewDecoder(res.Body).Decode(&serviceResponse); err == nil {
+							if err := json.Unmarshal(resBody, &serviceResponse); err == nil {
 								// setup emails
 								if len(serviceResponse.Data) > 0 {
 									setUserEmails(serviceResponse.Data, &s1)
@@ -153,6 +158,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 								if len(serviceResponse.Included) > 0 {
 									setOrganizationInfo(serviceResponse.Included, &s1)
 								}
+								// lightweight analyze: annotate standard fields
+								orgName := s1.ExtraData["org_name"]
+								orgURL := s1.ExtraData["org_url"]
+								emails := s1.ExtraData["user_emails"]
+								lwa.AugmentExtraData(s1.ExtraData, lwa.Fields{
+									Name:  &orgName,
+									URL:   &orgURL,
+									Email: &emails,
+								})
 							}
 							// break the loop once we've successfully validated the token against a baseURL
 							break

pkg/detectors/digitaloceanv2/digitaloceanv2.go

@@ -4,14 +4,15 @@
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"strings"
 
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	lwa "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lightweight_analyze"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
@@ -37,6 +38,7 @@
 
 // FromData will find and optionally verify DigitalOceanV2 secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	logCtx := logContext.AddLogger(ctx)
 	dataStr := string(data)
 
 	var uniqueTokens = make(map[string]struct{})
@@ -60,17 +62,19 @@
 			// Check if the token is a refresh token or an access token
 			switch {
 			case strings.HasPrefix(token, "dor_v1_"):
-				verified, verificationErr, newAccessToken := verifyRefreshToken(ctx, client, token)
+				verified, extraData, verificationErr, newAccessToken := verifyRefreshToken(logCtx, client, token)
 				s1.SetVerificationError(verificationErr)
 				s1.Verified = verified
+				s1.ExtraData = extraData
 				if s1.Verified {
 					s1.AnalysisInfo = map[string]string{
 						"key": newAccessToken,
 					}
 				}
 			case strings.HasPrefix(token, "doo_v1_"), strings.HasPrefix(token, "dop_v1_"):
-				verified, verificationErr := verifyAccessToken(ctx, client, token)
+				verified, extraData, verificationErr := verifyAccessToken(logCtx, client, token)
 				s1.Verified = verified
+				s1.ExtraData = extraData
 				s1.SetVerificationError(verificationErr)
 				if s1.Verified {
 					s1.AnalysisInfo = map[string]string{
@@ -90,72 +94,110 @@
 // If the token is valid, it returns the new access token and no error.
 // If the token is invalid/expired, it returns an empty string and no error.
 // If an error is encountered, it returns an empty string along and the error.
-func verifyRefreshToken(ctx context.Context, client *http.Client, token string) (bool, error, string) {
-	// Ref: https://docs.digitalocean.com/reference/api/oauth/
+func verifyRefreshToken(ctx logContext.Context, client *http.Client, token string) (bool, map[string]string, error, string) {
+	// Ref: https://docs.digitalocean.com/reference/api/oauth/#token
 
 	url := "https://cloud.digitalocean.com/v1/oauth/token?grant_type=refresh_token&refresh_token=" + token
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
 	if err != nil {
-		return false, fmt.Errorf("failed to create request: %w", err), ""
+		return false, nil, fmt.Errorf("failed to create request: %w", err), ""
 	}
 
 	res, err := client.Do(req)
 	if err != nil {
-		return false, fmt.Errorf("failed to make request: %w", err), ""
+		return false, nil, fmt.Errorf("failed to make request: %w", err), ""
 	}
 
-	bodyBytes, err := io.ReadAll(res.Body)
-	if err != nil {
-		return false, fmt.Errorf("failed to read response body: %w", err), ""
-	}
-	defer res.Body.Close()
+	extraData := make(map[string]string)
+
+	// lightweight analyze: unconditionally preserve the response body
+	resBody := lwa.CopyAndCloseResponseBody(ctx, extraData, res)
 
 	switch res.StatusCode {
 	case http.StatusOK:
-		var responseMap map[string]interface{}
-		if err := json.Unmarshal(bodyBytes, &responseMap); err != nil {
-			return false, fmt.Errorf("failed to parse response body: %w", err), ""
+		var resData struct {
+			AccessToken *string `json:"access_token"`
+			Info        struct {
+				UUID  *string `json:"uuid"`
+				Name  *string `json:"name"`
+				Email *string `json:"email"`
+			} `json:"info"`
 		}
-		// Extract the access token from the response
-		accessToken, exists := responseMap["access_token"].(string)
-		if !exists {
-			return false, fmt.Errorf("access_token not found in response: %s", string(bodyBytes)), ""
+		if err = json.Unmarshal(resBody, &resData); err != nil {
+			ctx.Logger().Error(err, "failed to parse response")
+			return false, extraData, err, ""
 		}
-		return true, nil, accessToken
+
+		if resData.AccessToken == nil {
+			return false, extraData, fmt.Errorf("access_token not found in response: %s", string(resBody)), ""
+		}
+
+		// lightweight analyze: annotate standard fields
+		lwa.AugmentExtraData(extraData, lwa.Fields{
+			ID:    resData.Info.UUID,
+			Name:  resData.Info.Name,
+			Email: resData.Info.Email,
+		})
+
+		return true, extraData, nil, *resData.AccessToken
 	case http.StatusUnauthorized:
-		return false, nil, ""
+		return false, extraData, nil, ""
 	default:
-		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode), ""
+		return false, extraData, fmt.Errorf("unexpected status code: %d", res.StatusCode), ""
 	}
 }
 
 // verifyAccessToken verifies the access token by making a request to the DigitalOcean API.
 // If the token is valid, it returns true and no error.
 // If the token is invalid, it returns false and no error.
 // If an error is encountered, it returns false along with the error.
-func verifyAccessToken(ctx context.Context, client *http.Client, token string) (bool, error) {
+func verifyAccessToken(ctx logContext.Context, client *http.Client, token string) (bool, map[string]string, error) {
 	// Ref: https://docs.digitalocean.com/reference/api/digitalocean/#tag/Account
 
 	url := "https://api.digitalocean.com/v2/account"
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return false, fmt.Errorf("failed to create request: %w", err)
+		return false, nil, fmt.Errorf("failed to create request: %w", err)
 	}
 
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
 	res, err := client.Do(req)
 	if err != nil {
-		return false, fmt.Errorf("failed to make request: %w", err)
+		return false, nil, fmt.Errorf("failed to make request: %w", err)
 	}
-	defer res.Body.Close()
+
+	extraData := make(map[string]string)
+
+	// lightweight analyze: unconditionally preserve the response body
+	resBody := lwa.CopyAndCloseResponseBody(ctx, extraData, res)
 
 	switch res.StatusCode {
 	case http.StatusOK:
-		return true, nil
+		var resData struct {
+			Account struct {
+				UUID  *string `json:"uuid"`
+				Name  *string `json:"name"`
+				Email *string `json:"email"`
+			} `json:"account"`
+		}
+
+		if err = json.Unmarshal(resBody, &resData); err != nil {
+			ctx.Logger().Error(err, "failed to parse response")
+			return false, extraData, nil
+		}
+
+		// lightweight analyze: annotate standard fields
+		lwa.AugmentExtraData(extraData, lwa.Fields{
+			ID:    resData.Account.UUID,
+			Name:  resData.Account.Name,
+			Email: resData.Account.Email,
+		})
+
+		return true, extraData, nil
 	case http.StatusUnauthorized:
-		return false, nil
+		return false, extraData, nil
 	default:
-		return false, fmt.Errorf("unexpected status code: %d", res.StatusCode)
+		return false, extraData, fmt.Errorf("unexpected status code: %d", res.StatusCode)
 	}
 }
 

pkg/detectors/elevenlabs/v1/elevenlabs.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
+	"maps"
 	"net/http"
 
 	regexp "github.com/wasilibs/go-re2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	lwa "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lightweight_analyze"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
@@ -20,13 +22,6 @@ type Scanner struct {
 
 func (Scanner) Version() int { return 1 }
 
-type UserRes struct {
-	Subscription struct {
-		Tier string `json:"tier"`
-	} `json:"subscription"`
-	Name string `json:"first_name"`
-}
-
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
@@ -44,6 +39,7 @@ func (s Scanner) Keywords() []string {
 
 // FromData will find and optionally verify Elevenlabs secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	logCtx := logContext.AddLogger(ctx)
 	dataStr := string(data)
 
 	uniqueMatches := make(map[string]struct{})
@@ -67,12 +63,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				client = defaultClient
 			}
 
-			isVerified, userResponse, verificationErr := verifyMatch(ctx, client, match)
+			isVerified, extraData, verificationErr := verifyMatch(logCtx, client, match)
 			s1.Verified = isVerified
-			if userResponse != nil {
-				s1.ExtraData["Name"] = userResponse.Name
-				s1.ExtraData["Tier"] = userResponse.Subscription.Tier
-			}
+			maps.Copy(s1.ExtraData, extraData)
 			s1.SetVerificationError(verificationErr, match)
 
 			if s1.Verified {
@@ -88,7 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return
 }
 
-func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, *UserRes, error) {
+func verifyMatch(ctx logContext.Context, client *http.Client, token string) (bool, map[string]string, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.elevenlabs.io/v1/user", nil)
 	if err != nil {
 		return false, nil, err
@@ -99,24 +92,58 @@ func verifyMatch(ctx context.Context, client *http.Client, token string) (bool,
 	if err != nil {
 		return false, nil, err
 	}
-	defer func() {
-		_, _ = io.Copy(io.Discard, res.Body)
-		_ = res.Body.Close()
-	}()
+
+	extraData := make(map[string]string)
+
+	// lightweight analyze: unconditionally preserve the response body
+	resBody := lwa.CopyAndCloseResponseBody(ctx, extraData, res)
 
 	switch res.StatusCode {
 	case http.StatusOK:
 		// If the endpoint returns useful information, we can return it as a map.
-		var userResponse UserRes
-		if err = json.NewDecoder(res.Body).Decode(&userResponse); err != nil {
-			return false, nil, err
+		var userResponse struct {
+			UserID       string `json:"user_id"`
+			Subscription struct {
+				Tier string `json:"tier"`
+			} `json:"subscription"`
+			FirstName string `json:"first_name"`
+		}
+
+		if err = json.Unmarshal(resBody, &userResponse); err != nil {
+			ctx.Logger().Error(err, "failed to parse response")
+			return false, extraData, err
 		}
-		return true, &userResponse, nil
+
+		// lightweight analyze: annotate "standard" fields
+		lwa.AugmentExtraData(extraData, lwa.Fields{
+			ID:   &userResponse.UserID,
+			Name: &userResponse.FirstName,
+			// Could include subscription tier here if wanted
+		})
+
+		extraData["Name"] = userResponse.FirstName
+		extraData["Tier"] = userResponse.Subscription.Tier
+
+		return true, extraData, nil
 	case http.StatusBadRequest, http.StatusUnauthorized:
-		// The secret is determinately not verified (nothing to do)
-		return false, nil, nil
+		// If the response says {"detail":{"status":"missing_permissions","message":"The API key you used is missing the permission user_read to execute this operation."}}
+		// then the key is valid, but we can't add the metadata
+		var errorResponse struct {
+			Detail struct {
+				Status string `json:"status"`
+			} `json:"detail"`
+		}
+
+		if err = json.Unmarshal(resBody, &errorResponse); err != nil {
+			ctx.Logger().Error(err, "failed to parse response")
+			return false, extraData, err
+		}
+		if errorResponse.Detail.Status == "missing_permissions" {
+			return true, extraData, nil
+		}
+		return false, extraData, nil
 	default:
-		return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+		return false, extraData, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 	}
 }