feat: add article summary feature with OpenAI integration

Author: paskal

.golangci.yml

@@ -54,6 +54,8 @@ linters:
           disabled: true
         - name: nested-structs
           disabled: true
+        - name: max-public-structs
+          arguments: [6]
         - name: use-slices-sort
           disabled: true
     gocritic:

README.md

@@ -23,6 +23,18 @@
 | openai-max-iter | OPENAI_MAX_ITER | `3`         | max evaluation iterations per extraction               |
 | dbg          | DEBUG           | `false`        | debug mode                                            |
 
+#### OpenAI integration
+
+| Command line                  | Environment                | Default       | Description                                                      |
+|-------------------------------|----------------------------|---------------|------------------------------------------------------------------|
+| openai.api-key                | OPENAI_API_KEY             | none          | OpenAI API key for summary generation                            |
+| openai.model-type             | OPENAI_MODEL_TYPE          | `gpt-4o-mini` | OpenAI model name (e.g., gpt-4o, gpt-4o-mini)                   |
+| openai.disable-summaries      | OPENAI_DISABLE_SUMMARIES   | `false`       | disable summary generation                                       |
+| openai.summary-prompt         | OPENAI_SUMMARY_PROMPT      | built-in      | custom prompt for summary generation                             |
+| openai.max-content-length     | OPENAI_MAX_CONTENT_LENGTH  | `10000`       | maximum content length to send to OpenAI API (0 for no limit)    |
+| openai.requests-per-minute    | OPENAI_REQUESTS_PER_MINUTE | `10`          | maximum OpenAI API requests per minute (0 for no limit)          |
+| openai.cleanup-interval       | OPENAI_CLEANUP_INTERVAL    | `24h`         | interval for cleaning up expired cached summaries                |
+
 ### Cloudflare Browser Rendering (optional)
 
 When both `--cf-account-id` and `--cf-api-token` are set, the service uses Cloudflare Browser Rendering API to fetch page content instead of direct HTTP. This renders JavaScript and handles bot-protection pages that return empty or "just a moment..." responses to standard HTTP requests.
@@ -44,9 +56,13 @@ When OpenAI is not configured, extraction works exactly as before — no GPT cal
 ### API
 
     GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
+    GET /api/content/v1/parser?token=secret&url=http://aa.com/blah&summary=true - extract content with AI-generated summary
     POST /api/extract {url: http://aa.com/blah}  - extract content
+    GET /api/metrics - summary generation metrics (cache hits, misses, response times)
     POST /api/content-parsed-wrong?url=http://aa.com/blah - force re-extraction with AI evaluation (requires basicAuth)
 
+Summary generation requires a valid token and an OpenAI API key. Summaries are cached in MongoDB with a 1-month expiration. Expired summaries are cleaned up automatically on the configured interval.
+
 ## Development
 
 ### Running tests

datastore/mongo.go

@@ -39,7 +39,8 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error
 
 // Stores contains all DAO instances
 type Stores struct {
-	Rules RulesDAO
+	Rules     RulesDAO
+	Summaries SummariesDAO
 }
 
 // GetStores initialize collections and make indexes
@@ -50,8 +51,15 @@ func (m *MongoServer) GetStores() Stores {
 		{Keys: bson.D{{Key: "domain", Value: 1}, {Key: "match_urls", Value: 1}}},
 	}
 
+	sIndexes := []mongo.IndexModel{
+		{Keys: bson.D{{Key: "created_at", Value: 1}}},
+		{Keys: bson.D{{Key: "model", Value: 1}}},
+		{Keys: bson.D{{Key: "expires_at", Value: 1}}}, // index for cleaning up expired summaries
+	}
+
 	return Stores{
-		Rules: RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Rules:     RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Summaries: SummariesDAO{Collection: m.collection("summaries", sIndexes)},
 	}
 }
 

datastore/summaries.go

@@ -0,0 +1,109 @@
+// Package datastore provides mongo implementation for store to keep and access summaries
+package datastore
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+// Summary contains information about a cached summary
+type Summary struct {
+	ID        string    `bson:"_id"`     // sha256 hash of the content
+	Content   string    `bson:"content"` // original content that was summarized (could be truncated for storage efficiency)
+	Summary   string    `bson:"summary"` // generated summary
+	Model     string    `bson:"model"`   // openAI model used for summarisation
+	CreatedAt time.Time `bson:"created_at"`
+	UpdatedAt time.Time `bson:"updated_at"`
+	ExpiresAt time.Time `bson:"expires_at"` // when this summary expires
+}
+
+// SummariesDAO handles database operations for article summaries
+type SummariesDAO struct {
+	Collection *mongo.Collection
+}
+
+// Get returns summary by content hash
+func (s SummariesDAO) Get(ctx context.Context, content string) (Summary, bool) {
+	contentHash := GenerateContentHash(content)
+	res := s.Collection.FindOne(ctx, bson.M{"_id": contentHash})
+	if res.Err() != nil {
+		if res.Err() == mongo.ErrNoDocuments {
+			return Summary{}, false
+		}
+		log.Printf("[WARN] can't get summary for hash %s: %v", contentHash, res.Err())
+		return Summary{}, false
+	}
+
+	summary := Summary{}
+	if err := res.Decode(&summary); err != nil {
+		log.Printf("[WARN] can't decode summary document for hash %s: %v", contentHash, err)
+		return Summary{}, false
+	}
+
+	return summary, true
+}
+
+// Save creates or updates summary in the database
+func (s SummariesDAO) Save(ctx context.Context, summary Summary) error {
+	if summary.ID == "" {
+		summary.ID = GenerateContentHash(summary.Content)
+	}
+
+	if summary.CreatedAt.IsZero() {
+		summary.CreatedAt = time.Now()
+	}
+	summary.UpdatedAt = time.Now()
+
+	// set default expiration of 1 month if not specified
+	if summary.ExpiresAt.IsZero() {
+		summary.ExpiresAt = time.Now().AddDate(0, 1, 0)
+	}
+
+	opts := options.UpdateOne().SetUpsert(true)
+	_, err := s.Collection.UpdateOne(
+		ctx,
+		bson.M{"_id": summary.ID},
+		bson.M{"$set": summary},
+		opts,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to save summary: %w", err)
+	}
+	return nil
+}
+
+// Delete removes summary from the database
+func (s SummariesDAO) Delete(ctx context.Context, contentHash string) error {
+	_, err := s.Collection.DeleteOne(ctx, bson.M{"_id": contentHash})
+	if err != nil {
+		return fmt.Errorf("failed to delete summary: %w", err)
+	}
+	return nil
+}
+
+// CleanupExpired removes all summaries that have expired
+func (s SummariesDAO) CleanupExpired(ctx context.Context) (int64, error) {
+	now := time.Now()
+	result, err := s.Collection.DeleteMany(
+		ctx,
+		bson.M{"expires_at": bson.M{"$lt": now}},
+	)
+	if err != nil {
+		return 0, fmt.Errorf("failed to cleanup expired summaries: %w", err)
+	}
+	return result.DeletedCount, nil
+}
+
+// GenerateContentHash creates a hash for the content to use as an ID
+func GenerateContentHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}

datastore/summaries_test.go

@@ -0,0 +1,165 @@
+package datastore
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo"
+)
+
+func TestSummariesDAO_SaveAndGet(t *testing.T) {
+	if _, ok := os.LookupEnv("ENABLE_MONGO_TESTS"); !ok {
+		t.Skip("ENABLE_MONGO_TESTS env variable is not set")
+	}
+
+	mdb, err := New("mongodb://localhost:27017", "test_ureadability", 0)
+	require.NoError(t, err)
+
+	// create a unique collection for this test to avoid conflicts
+	collection := mdb.client.Database(mdb.dbName).Collection("summaries_test")
+	defer func() {
+		_ = collection.Drop(context.Background())
+	}()
+
+	// create an index on the expiresAt field
+	_, err = collection.Indexes().CreateOne(context.Background(),
+		mongo.IndexModel{
+			Keys: bson.D{{Key: "expires_at", Value: 1}},
+		})
+	require.NoError(t, err)
+
+	dao := SummariesDAO{Collection: collection}
+
+	content := "This is a test article content. It should generate a unique hash."
+	summary := Summary{
+		Content:   content,
+		Summary:   "This is a test summary of the article.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: time.Now(),
+	}
+
+	// test saving a summary
+	err = dao.Save(context.Background(), summary)
+	require.NoError(t, err)
+
+	// test getting the summary
+	foundSummary, found := dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, summary.Summary, foundSummary.Summary)
+	assert.Equal(t, summary.Model, foundSummary.Model)
+	assert.NotEmpty(t, foundSummary.ID)
+
+	// test getting a non-existent summary
+	_, found = dao.Get(context.Background(), "non-existent content")
+	assert.False(t, found)
+
+	// test updating an existing summary
+	updatedSummary := Summary{
+		ID:        foundSummary.ID,
+		Content:   content,
+		Summary:   "This is an updated summary.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: foundSummary.CreatedAt,
+	}
+
+	err = dao.Save(context.Background(), updatedSummary)
+	require.NoError(t, err)
+
+	foundSummary, found = dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, "This is an updated summary.", foundSummary.Summary)
+	assert.Equal(t, updatedSummary.CreatedAt, foundSummary.CreatedAt)
+	assert.NotEqual(t, updatedSummary.UpdatedAt, foundSummary.UpdatedAt) // UpdatedAt should be set by the DAO
+
+	// test deleting a summary
+	err = dao.Delete(context.Background(), foundSummary.ID)
+	require.NoError(t, err)
+
+	_, found = dao.Get(context.Background(), content)
+	assert.False(t, found)
+}
+
+func TestGenerateContentHash(t *testing.T) {
+	content1 := "This is a test content."
+	content2 := "This is a different test content."
+
+	hash1 := GenerateContentHash(content1)
+	hash2 := GenerateContentHash(content2)
+
+	assert.NotEqual(t, hash1, hash2)
+	assert.Equal(t, hash1, GenerateContentHash(content1)) // same content should produce same hash
+	assert.Equal(t, 64, len(hash1))                       // SHA-256 produces 64 character hex string
+}
+
+func TestSummariesDAO_CleanupExpired(t *testing.T) {
+	if _, ok := os.LookupEnv("ENABLE_MONGO_TESTS"); !ok {
+		t.Skip("ENABLE_MONGO_TESTS env variable is not set")
+	}
+
+	mdb, err := New("mongodb://localhost:27017", "test_ureadability", 0)
+	require.NoError(t, err)
+
+	// create a unique collection for this test to avoid conflicts
+	collection := mdb.client.Database(mdb.dbName).Collection("summaries_expired_test")
+	defer func() {
+		_ = collection.Drop(context.Background())
+	}()
+
+	// create an index on the expiresAt field
+	_, err = collection.Indexes().CreateOne(context.Background(),
+		mongo.IndexModel{
+			Keys: bson.D{{Key: "expires_at", Value: 1}},
+		})
+	require.NoError(t, err)
+
+	dao := SummariesDAO{Collection: collection}
+	ctx := context.Background()
+
+	// add expired summary
+	expiredSummary := Summary{
+		Content:   "This is an expired summary",
+		Summary:   "Expired content",
+		Model:     "gpt-4o-mini",
+		CreatedAt: time.Now().Add(-48 * time.Hour),
+		UpdatedAt: time.Now().Add(-48 * time.Hour),
+		ExpiresAt: time.Now().Add(-24 * time.Hour), // expired 24 hours ago
+	}
+	err = dao.Save(ctx, expiredSummary)
+	require.NoError(t, err)
+
+	// add valid summary
+	validSummary := Summary{
+		Content:   "This is a valid summary",
+		Summary:   "Valid content",
+		Model:     "gpt-4o-mini",
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		ExpiresAt: time.Now().Add(24 * time.Hour), // expires in 24 hours
+	}
+	err = dao.Save(ctx, validSummary)
+	require.NoError(t, err)
+
+	// verify both summaries exist
+	_, foundExpired := dao.Get(ctx, expiredSummary.Content)
+	assert.True(t, foundExpired, "Expected to find expired summary before cleanup")
+
+	_, foundValid := dao.Get(ctx, validSummary.Content)
+	assert.True(t, foundValid, "Expected to find valid summary before cleanup")
+
+	// run cleanup
+	count, err := dao.CleanupExpired(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), count, "Expected to clean up exactly one record")
+
+	// verify expired summary is gone but valid remains
+	_, foundExpired = dao.Get(ctx, expiredSummary.Content)
+	assert.False(t, foundExpired, "Expected expired summary to be deleted")
+
+	_, foundValid = dao.Get(ctx, validSummary.Content)
+	assert.True(t, foundValid, "Expected valid summary to still exist")
+}