fix: deduplicate extraction logic, remove dead code, fix doc comments

- customParser now delegates to extractWithSelector (eliminates duplicated
  goquery parse+find+html loop)
- image extraction moved out of evaluation loop — runs once on final
  result instead of every iteration
- extract "ai-evaluator" to aiEvaluatorUser constant
- fix incorrect doc comment on callAPI
- remove unused getAuth test helper
- remove redundant cancel() call and restating comments

Author: paskal

extractor/evaluator.go

@@ -45,7 +45,6 @@ type OpenAIEvaluator struct {
 	client       *openai.Client
 }
 
-// getClient returns the OpenAI client, creating it once on first use
 func (e *OpenAIEvaluator) getClient() *openai.Client {
 	e.clientOnce.Do(func() {
 		if e.clientConfig != nil {
@@ -71,7 +70,6 @@ func (e *OpenAIEvaluator) Evaluate(ctx context.Context, reqURL, extractedText, h
 		if !errors.Is(err, errInvalidJSON) {
 			return nil, err
 		}
-		cancel() // release the first context before creating a new one
 
 		// retry once on invalid JSON with a fresh timeout
 		log.Printf("[WARN] invalid JSON from OpenAI for %s, retrying once", reqURL)
@@ -87,7 +85,7 @@ func (e *OpenAIEvaluator) Evaluate(ctx context.Context, reqURL, extractedText, h
 }
 
 // callAPI makes a single API call and parses the response JSON.
-// Returns nil EvalResult (without error) if the response is not valid JSON.
+// returns errInvalidJSON if the response is not valid JSON.
 func (e *OpenAIEvaluator) callAPI(ctx context.Context, client *openai.Client, userPrompt string) (*EvalResult, error) {
 	resp, err := client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
 		Model: e.Model,

extractor/readability.go

@@ -18,7 +18,10 @@ import (
 	"github.com/ukeeper/ukeeper-readability/datastore"
 )
 
-const defaultMaxGPTIter = 3
+const (
+	defaultMaxGPTIter = 3
+	aiEvaluatorUser   = "ai-evaluator"
+)
 
 //go:generate moq -out mocks/rules.go -pkg mocks -skip-ensure -fmt goimports . Rules
 
@@ -173,7 +176,6 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
 	return rb, nil
 }
 
-// maxGPTIter returns MaxGPTIter or the default if not set
 func (f *UReadability) maxGPTIter() int {
 	if f.MaxGPTIter > 0 {
 		return f.MaxGPTIter
@@ -220,29 +222,31 @@ func (f *UReadability) evaluateAndImprove(ctx context.Context, reqURL, htmlBody
 			continue
 		}
 
-		// rebuild the response with new content
+		// rebuild the response with new content (defer link normalisation and image extraction to after the loop)
 		improved := *best
 		improved.Content = f.getText(rawHTML, best.Title)
 		improved.Rich = rawHTML
 		improved.Excerpt = f.getSnippet(improved.Content)
 
-		// normalize links and extract images from the new content
+		best = &improved
+		bestSelector = eval.Selector
+	}
+
+	// post-process the final result: normalise links and extract images once
+	if bestSelector != "" {
 		finalURL, err := url.Parse(best.URL)
 		if err != nil {
 			log.Printf("[WARN] failed to parse URL %q in evaluateAndImprove: %v", best.URL, err)
 			return best
 		}
-		improved.Rich, improved.AllLinks = f.normalizeLinks(improved.Rich, finalURL)
-		darticle, err := goquery.NewDocumentFromReader(strings.NewReader(improved.Rich))
+		best.Rich, best.AllLinks = f.normalizeLinks(best.Rich, finalURL)
+		darticle, err := goquery.NewDocumentFromReader(strings.NewReader(best.Rich))
 		if err == nil {
 			if im, allImages, ok := f.extractPics(darticle.Find("img"), reqURL); ok {
-				improved.Image = im
-				improved.AllImages = allImages
+				best.Image = im
+				best.AllImages = allImages
 			}
 		}
-
-		best = &improved
-		bestSelector = eval.Selector
 	}
 
 	// save rule if we found a better selector
@@ -254,7 +258,7 @@ func (f *UReadability) evaluateAndImprove(ctx context.Context, reqURL, htmlBody
 		}
 		rule.Content = bestSelector
 		rule.Enabled = true
-		rule.User = "ai-evaluator"
+		rule.User = aiEvaluatorUser
 		if _, err := f.Rules.Save(ctx, rule); err != nil {
 			log.Printf("[WARN] failed to save AI-suggested rule for %s: %v", best.Domain, err)
 		} else {
@@ -298,16 +302,10 @@ func (f *UReadability) getContent(ctx context.Context, body, reqURL string, rule
 	// custom rules parser
 	customParser := func(body, reqURL string, rule datastore.Rule) (content, rich string, err error) {
 		log.Printf("[DEBUG] custom extractor for %s", reqURL)
-		dbody, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+		res, err := f.extractWithSelector(body, rule.Content)
 		if err != nil {
 			return "", "", err
 		}
-		var res string
-		dbody.Find(rule.Content).Each(func(_ int, s *goquery.Selection) {
-			if html, err := s.Html(); err == nil {
-				res += html
-			}
-		})
 		if res == "" {
 			return "", "", fmt.Errorf("nothing extracted from %s, rule=%v", reqURL, rule)
 		}

rest/server_test.go

@@ -657,10 +657,6 @@ func TestServer_ContentParsedWrong(t *testing.T) {
 	})
 }
 
-func getAuth(t *testing.T, url string) (response string, statusCode int) {
-	return doAuth(t, "GET", url)
-}
-
 func postAuth(t *testing.T, url string) (response string, statusCode int) {
 	return doAuth(t, "POST", url)
 }