diff --git a/.gitignore b/.gitignore index 929d39f..5776821 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,4 @@ pnpm-lock.yaml .expo # Data -books.jsonl \ No newline at end of file +books.jsonl diff --git a/README.md b/README.md index 65a0116..9b7db35 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This is a monorepo containing multiple standalone projects. Each project lives i code-samples/ ├── typesense-angular-search-bar/ # Angular + Typesense search implementation ├── typesense-astro-search/ # Astro + Typesense search implementation +├── typesense-django-full-text-search/ # Python (Django) + Typesense backend implementation ├── typesense-gin-full-text-search/ # Go (Gin) + Typesense backend implementation ├── typesense-kotlin/ # Kotlin (Android) + Typesense search implementation ├── typesense-langchain-ai-search/ # Python + LangChain + Typesense AI search implementation @@ -34,6 +35,7 @@ code-samples/ | ---------------------------------------------------------------------------- | ------------- | --------------------------------------------------------------- | | [typesense-angular-search-bar](./typesense-angular-search-bar) | Angular | A modern search bar with instant search capabilities | | [typesense-astro-search](./typesense-astro-search) | Astro | A modern search bar with instant search capabilities | +| [typesense-django-full-text-search](./typesense-django-full-text-search) | Python (Django) | Backend API with full-text search using Typesense | | [typesense-gin-full-text-search](./typesense-gin-full-text-search) | Go (Gin) | Backend API with full-text search using Typesense | | [typesense-kotlin](./typesense-kotlin) | Kotlin (Android) | A native Android search bar with instant search capabilities | | [typesense-langchain-ai-search](./typesense-langchain-ai-search) | LangChain (Python) | Conversational AI-powered movie search and recommendation app | diff --git a/typesense-django-full-text-search/.env.example b/typesense-django-full-text-search/.env.example new file mode 100644 index 0000000..85b3899 --- /dev/null +++ b/typesense-django-full-text-search/.env.example @@ -0,0 +1,15 @@ +PORT=8000 + +DB_HOST=localhost +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=typesense_books +DB_PORT=5432 + +TYPESENSE_HOST=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_API_KEY=xyz + +# Django +DJANGO_SECRET_KEY=change-me-in-production diff --git a/typesense-django-full-text-search/.gitignore b/typesense-django-full-text-search/.gitignore new file mode 100644 index 0000000..bb80788 --- /dev/null +++ b/typesense-django-full-text-search/.gitignore @@ -0,0 +1,31 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +ENV/ + +# Django +*.log +db.sqlite3 +db.sqlite3-journal +staticfiles/ +media/ + +# Environment +.env + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + diff --git a/typesense-django-full-text-search/README.md b/typesense-django-full-text-search/README.md new file mode 100644 index 0000000..762bbdc --- /dev/null +++ b/typesense-django-full-text-search/README.md @@ -0,0 +1,281 @@ +# Django Full-Text Search with Typesense + +A production-ready RESTful search API built with Python, Django, PostgreSQL, and Typesense. Features full-text search, CRUD operations, real-time async indexing, soft deletes, paginated sync, and background workers. + +## Tech Stack + +- Python 3.10+ +- Django 5.0+ +- PostgreSQL (`psycopg2-binary`) +- Typesense (`typesense-python`) +- APScheduler (background periodic sync) +- Docker + +## Prerequisites + +- Python 3.10+ installed +- Docker (for Typesense and PostgreSQL) +- Basic knowledge of Python, Django, REST APIs, and SQL + +## Quick Start + +### 1. Clone the repository + +```bash +git clone https://github.com/typesense/code-samples.git +cd typesense-django-full-text-search +``` + +### 2. Set up virtual environment & install dependencies + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +### 3. Start Typesense and PostgreSQL + +Run Typesense and PostgreSQL using Docker: + +```bash +# Create local data directory for Typesense +mkdir -p typesense-data + +# Start Typesense +docker run -d \ + -p 8108:8108 \ + -v "$(pwd)"/typesense-data:/data \ + typesense/typesense:27.1 \ + --data-dir /data \ + --api-key=xyz \ + --enable-cors + +# Start PostgreSQL +docker run -d \ + -p 5432:5432 \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=typesense_books \ + postgres:15 +``` + +### 4. Set up environment variables + +Copy `.env.example` to `.env`: + +```bash +cp .env.example .env +``` + +The `.env` file should look like this: + +```env +PORT=8000 + +# PostgreSQL Configuration +DB_HOST=localhost +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=typesense_books +DB_PORT=5432 + +# Typesense Configuration +TYPESENSE_HOST=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_API_KEY=xyz + +# Django Configuration +DJANGO_SECRET_KEY=change-me-in-production +``` + +### 5. Run Database Migrations + +```bash +python manage.py migrate +``` + +### 6. Project Structure + +```text +typesense-django-full-text-search/ +├── books/ +│ ├── search/ +│ │ ├── __init__.py # Exports search client and sync functions +│ │ ├── client.py # Typesense Python client initialization +│ │ ├── collections.py # Typesense collection schema definition +│ │ ├── sync.py # Paginated full & incremental sync logic +│ │ └── worker.py # APScheduler background periodic sync worker +│ ├── tests/ +│ │ ├── test_models.py # Tests for soft-delete & model managers +│ │ ├── test_sync.py # Tests for document mapping & sync logic +│ │ └── test_views.py # Tests for API endpoints & search proxy +│ ├── apps.py # Starts background sync worker on Django startup +│ ├── models.py # Book Django model with ActiveBookManager +│ ├── urls.py # App routing +│ └── views.py # CRUD & search proxy API views +├── typesensedjango/ +│ ├── settings.py # Django project settings +│ └── urls.py # Root URL configuration +├── .env.example +├── manage.py +└── requirements.txt +``` + +### 7. Start the Development Server + +```bash +python manage.py runserver 8000 +``` + +The server starts at `http://localhost:8000`. Upon startup, Django automatically initializes the Typesense collection schema and launches the background periodic sync thread. + +### 8. API Endpoints + +#### Search + +```bash +GET /search/?q= +``` + +Example: + +```bash +curl "http://localhost:8000/search/?q=harry" +``` + +Response: + +```json +{ + "query": "harry", + "found": 1, + "results": [...], + "facet_counts": [] +} +``` + +#### CRUD Operations + +**List books (Paginated):** + +```bash +GET /books/?page=1&limit=10 +``` + +**Create a book:** + +```bash +POST /books/ +Content-Type: application/json + +{ + "title": "The Hobbit", + "authors": ["J.R.R. Tolkien"], + "publication_year": 1937, + "average_rating": 4.8, + "image_url": "https://example.com/hobbit.jpg", + "ratings_count": 1250000 +} +``` + +**Get a book:** + +```bash +GET /books/:id/ +``` + +**Update a book:** + +```bash +PUT /books/:id/ +Content-Type: application/json + +{ + "title": "The Hobbit: There and Back Again", + "average_rating": 4.9 +} +``` + +**Delete a book (soft delete):** + +```bash +DELETE /books/:id/ +``` + +Returns `204 No Content` and soft-deletes the row in PostgreSQL while deleting the document from Typesense. + +#### Sync Operations + +**Trigger manual sync:** + +```bash +POST /sync/ +``` + +Response: + +```json +{ + "message": "Sync completed", + "syncedAt": "2026-07-23T16:30:00+00:00" +} +``` + +**Check sync status:** + +```bash +GET /sync/status/ +``` + +Response: + +```json +{ + "lastSyncTime": "2026-07-23T16:30:00+00:00", + "syncWorkerRunning": true, + "syncJobActive": false +} +``` + +### 9. How It Works + +#### Architecture + +```plaintext +User Request + ↓ +Django REST Views (CRUD) + ↓ +PostgreSQL (Source of Truth) + ↓ +Real-Time / Periodic Sync → Typesense (Search Index) + ↑ +Background Worker (APScheduler thread running every 60s) +``` + +#### Sync Strategies + +1. **Real-time Sync**: On every POST/PUT/DELETE API request, the change hits PostgreSQL first, and is then immediately mirrored to Typesense. +2. **Periodic Background Sync**: Runs every 60 seconds via `APScheduler` in a thread. Uses `updated_at > last_sync_time` to catch any database changes that happened outside the API or during server downtime. +3. **Startup Sync**: On boot, if Typesense is empty, a full sync runs. If Typesense has documents, an incremental sync runs starting from epoch so missing records are backfilled. +4. **Manual Sync**: `POST /sync/` triggers an on-demand full sync. + +#### Memory-Safe Keyset Pagination + +Both full sync and incremental sync fetch records from PostgreSQL using keyset pagination (`id__gt=last_id`) in batches of 1,000 to prevent memory allocation spikes when syncing large tables. + +### 10. Running Tests + +Run the complete test suite using Django's test runner: + +```bash +python manage.py test books.tests +``` + +```text +Ran 19 tests in 0.15s + +OK +``` diff --git a/typesense-django-full-text-search/books/__init__.py b/typesense-django-full-text-search/books/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/typesense-django-full-text-search/books/admin.py b/typesense-django-full-text-search/books/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/typesense-django-full-text-search/books/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/typesense-django-full-text-search/books/apps.py b/typesense-django-full-text-search/books/apps.py new file mode 100644 index 0000000..bf4f9dc --- /dev/null +++ b/typesense-django-full-text-search/books/apps.py @@ -0,0 +1,41 @@ +import sys +import logging +from django.apps import AppConfig + +logger = logging.getLogger(__name__) + +class BooksConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'books' + + def ready(self): + # Avoid running during management commands (like migrate, makemigrations) + if 'manage.py' in sys.argv and 'runserver' not in sys.argv: + return + + from .search import ( + initialize_typesense, + determine_and_run_startup_sync, + start_background_sync_worker + ) + import threading + + def _startup_sequence(): + try: + logger.info('Initializing Typesense...') + initialize_typesense() + + logger.info('Running startup sync...') + try: + determine_and_run_startup_sync() + except Exception as sync_err: + logger.warning('Startup sync skipped (database might not be migrated yet): %s', sync_err) + + start_background_sync_worker() + except Exception as e: + logger.error('Failed to start background sync worker: %s', e) + + import os + if os.environ.get('RUN_MAIN') == 'true' or not sys.argv[0].endswith('manage.py'): + thread = threading.Thread(target=_startup_sequence, daemon=True) + thread.start() diff --git a/typesense-django-full-text-search/books/migrations/0001_initial.py b/typesense-django-full-text-search/books/migrations/0001_initial.py new file mode 100644 index 0000000..78c0da0 --- /dev/null +++ b/typesense-django-full-text-search/books/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 6.0.6 on 2026-07-02 12:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Book', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('authors', models.JSONField(default=list)), + ('publication_year', models.IntegerField(blank=True, null=True)), + ('average_rating', models.DecimalField(blank=True, decimal_places=2, max_digits=3, null=True)), + ('image_url', models.CharField(blank=True, max_length=255, null=True)), + ('ratings_count', models.IntegerField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'books', + 'ordering': ['id'], + }, + ), + ] diff --git a/typesense-django-full-text-search/books/migrations/__init__.py b/typesense-django-full-text-search/books/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/typesense-django-full-text-search/books/models.py b/typesense-django-full-text-search/books/models.py new file mode 100644 index 0000000..c66744b --- /dev/null +++ b/typesense-django-full-text-search/books/models.py @@ -0,0 +1,40 @@ +from django.db import models +from django.utils import timezone + +class ActiveBookManager(models.Manager): + def get_queryset(self): + return super().get_queryset().filter(deleted_at__isnull=True) + +class TolerantJSONField(models.JSONField): + def from_db_value(self, value, expression, connection): + if isinstance(value, (list, dict)): + return value + return super().from_db_value(value, expression, connection) + +class Book(models.Model): + title = models.CharField(max_length=255) + authors = TolerantJSONField(default=list) + publication_year = models.IntegerField(null=True, blank=True) + average_rating = models.DecimalField(max_digits=3, decimal_places=2, null=True, blank=True) + image_url = models.CharField(max_length=255, null=True, blank=True) + ratings_count = models.IntegerField(null=True, blank=True) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + deleted_at = models.DateTimeField(null=True, blank=True) + + # Use ActiveBookManager by default to hide soft-deleted records + objects = ActiveBookManager() + # Provide access to all objects, including soft-deleted ones + all_objects = models.Manager() + + class Meta: + db_table = 'books' + ordering = ['id'] + + def delete(self, using=None, keep_parents=False): + self.deleted_at = timezone.now() + self.save() + + def __str__(self): + return self.title diff --git a/typesense-django-full-text-search/books/search/__init__.py b/typesense-django-full-text-search/books/search/__init__.py new file mode 100644 index 0000000..b15af01 --- /dev/null +++ b/typesense-django-full-text-search/books/search/__init__.py @@ -0,0 +1,4 @@ +from .client import typesense_client +from .collections import BOOKS_COLLECTION_NAME, initialize_typesense +from .sync import run_full_sync, run_incremental_sync, determine_and_run_startup_sync, get_last_sync_time +from .worker import start_background_sync_worker, get_sync_status diff --git a/typesense-django-full-text-search/books/search/client.py b/typesense-django-full-text-search/books/search/client.py new file mode 100644 index 0000000..9d78457 --- /dev/null +++ b/typesense-django-full-text-search/books/search/client.py @@ -0,0 +1,15 @@ +import os +import typesense +from dotenv import load_dotenv + +load_dotenv() + +typesense_client = typesense.Client({ + 'nodes': [{ + 'host': os.environ.get('TYPESENSE_HOST', 'localhost'), + 'port': os.environ.get('TYPESENSE_PORT', '8108'), + 'protocol': os.environ.get('TYPESENSE_PROTOCOL', 'http') + }], + 'api_key': os.environ.get('TYPESENSE_API_KEY', 'xyz'), + 'connection_timeout_seconds': 5 +}) diff --git a/typesense-django-full-text-search/books/search/collections.py b/typesense-django-full-text-search/books/search/collections.py new file mode 100644 index 0000000..ee649d2 --- /dev/null +++ b/typesense-django-full-text-search/books/search/collections.py @@ -0,0 +1,33 @@ +import logging +from .client import typesense_client + +logger = logging.getLogger(__name__) + +BOOKS_COLLECTION_NAME = 'books' + +books_collection_schema = { + 'name': BOOKS_COLLECTION_NAME, + 'fields': [ + {'name': 'id', 'type': 'string'}, + {'name': 'title', 'type': 'string'}, + {'name': 'authors', 'type': 'string[]', 'facet': True}, + {'name': 'publication_year', 'type': 'int32', 'facet': True, 'optional': True}, + {'name': 'average_rating', 'type': 'float', 'facet': True, 'optional': True}, + {'name': 'image_url', 'type': 'string', 'optional': True}, + {'name': 'ratings_count', 'type': 'int32', 'optional': True}, + ] +} + +def initialize_typesense(): + try: + collections = typesense_client.collections.retrieve() + collection_exists = any(c['name'] == BOOKS_COLLECTION_NAME for c in collections) + + if not collection_exists: + logger.info('Creating collection %s...', BOOKS_COLLECTION_NAME) + typesense_client.collections.create(books_collection_schema) + logger.info('Collection %s created successfully.', BOOKS_COLLECTION_NAME) + else: + logger.info('Collection %s already exists.', BOOKS_COLLECTION_NAME) + except Exception as e: + logger.error('Error initializing Typesense collection: %s', e) diff --git a/typesense-django-full-text-search/books/search/sync.py b/typesense-django-full-text-search/books/search/sync.py new file mode 100644 index 0000000..9bc6079 --- /dev/null +++ b/typesense-django-full-text-search/books/search/sync.py @@ -0,0 +1,128 @@ +import logging +import datetime +import threading +from django.utils import timezone +from .client import typesense_client +from .collections import BOOKS_COLLECTION_NAME + +logger = logging.getLogger(__name__) + +# Thread-safe sync state +_sync_lock = threading.Lock() +_last_sync_time = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + +BATCH_SIZE = 1000 + +def get_last_sync_time(): + with _sync_lock: + return _last_sync_time + +def _set_last_sync_time(value): + global _last_sync_time + with _sync_lock: + _last_sync_time = value + +def _map_book_to_document(book): + return { + 'id': str(book.id), + 'title': book.title, + 'authors': book.authors if isinstance(book.authors, list) else [book.authors], + 'publication_year': book.publication_year or 0, + 'average_rating': float(book.average_rating) if book.average_rating else 0.0, + 'image_url': book.image_url or '', + 'ratings_count': book.ratings_count or 0, + } + +def run_full_sync(): + logger.info('Running full sync...') + from books.models import Book + + last_id = 0 + total_processed = 0 + + while True: + try: + books = list(Book.objects.filter(id__gt=last_id).order_by('id')[:BATCH_SIZE]) + except Exception as err: + logger.error('Database error during full sync fetching: %s', err) + break + + if not books: + break + + last_id = books[-1].id + documents = [_map_book_to_document(b) for b in books] + + try: + typesense_client.collections[BOOKS_COLLECTION_NAME].documents.import_(documents, {'action': 'upsert'}) + total_processed += len(documents) + logger.info('Full sync: Processed %d books.', total_processed) + except Exception as err: + logger.error('Error importing documents during full sync: %s', err) + break + + _set_last_sync_time(timezone.now()) + logger.info('Full sync completed.') + +def run_incremental_sync(): + sync_started_at = timezone.now() + current_last_sync = get_last_sync_time() + + logger.info('Running incremental sync since %s...', current_last_sync.isoformat()) + from books.models import Book + + # 1. Upsert newly created or updated books — paginated + last_id = 0 + total_upserted = 0 + + while True: + batch = list( + Book.objects.filter(updated_at__gt=current_last_sync, id__gt=last_id) + .order_by('id')[:BATCH_SIZE] + ) + if not batch: + break + + last_id = batch[-1].id + documents = [_map_book_to_document(b) for b in batch] + + try: + typesense_client.collections[BOOKS_COLLECTION_NAME].documents.import_(documents, {'action': 'upsert'}) + total_upserted += len(documents) + except Exception as err: + logger.error('Error upserting documents in incremental sync: %s', err) + + if total_upserted: + logger.info('Incremental sync: Upserted %d books.', total_upserted) + + # 2. Delete soft-deleted books + deleted_books = Book.all_objects.filter(deleted_at__gt=current_last_sync) + + if deleted_books.exists(): + for book in deleted_books: + try: + typesense_client.collections[BOOKS_COLLECTION_NAME].documents[str(book.id)].delete() + logger.info('Incremental sync: Deleted book %d from Typesense.', book.id) + except Exception as err: + if not (hasattr(err, 'status_code') and err.status_code == 404): + logger.error('Error deleting book %d from Typesense: %s', book.id, err) + + _set_last_sync_time(sync_started_at) + logger.info('Incremental sync completed.') + +def determine_and_run_startup_sync(): + from books.models import Book + + try: + search_stats = typesense_client.collections[BOOKS_COLLECTION_NAME].retrieve() + doc_count = search_stats.get('num_documents', 0) + + if doc_count == 0: + run_full_sync() + else: + # Set last_sync_time to epoch so incremental sync picks up + # any records that may have been missed while Typesense was down. + _set_last_sync_time(datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) + run_incremental_sync() + except Exception as error: + logger.error('Error during startup sync: %s', error) diff --git a/typesense-django-full-text-search/books/search/worker.py b/typesense-django-full-text-search/books/search/worker.py new file mode 100644 index 0000000..2964a14 --- /dev/null +++ b/typesense-django-full-text-search/books/search/worker.py @@ -0,0 +1,38 @@ +import logging +import threading +from apscheduler.schedulers.background import BackgroundScheduler +from .sync import run_incremental_sync + +logger = logging.getLogger(__name__) + +_sync_job_lock = threading.Lock() +_scheduler = None + +def _sync_job(): + acquired = _sync_job_lock.acquire(blocking=False) + if not acquired: + logger.info('Sync already running, skipping this iteration.') + return + + try: + run_incremental_sync() + except Exception as error: + logger.error('Error in background sync worker: %s', error) + finally: + _sync_job_lock.release() + +def start_background_sync_worker(): + global _scheduler + if _scheduler is not None: + return + + logger.info('Starting background periodic sync worker (every 60s)...') + _scheduler = BackgroundScheduler() + _scheduler.add_job(_sync_job, 'interval', seconds=60) + _scheduler.start() + +def get_sync_status(): + return { + 'syncWorkerRunning': _scheduler is not None and _scheduler.running, + 'syncJobActive': _sync_job_lock.locked() + } diff --git a/typesense-django-full-text-search/books/tests/__init__.py b/typesense-django-full-text-search/books/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/typesense-django-full-text-search/books/tests/test_models.py b/typesense-django-full-text-search/books/tests/test_models.py new file mode 100644 index 0000000..8d19932 --- /dev/null +++ b/typesense-django-full-text-search/books/tests/test_models.py @@ -0,0 +1,35 @@ +from django.test import TestCase +from django.utils import timezone +from books.models import Book + + +class BookSoftDeleteTest(TestCase): + def test_delete_sets_deleted_at(self): + book = Book.objects.create(title='Test Book', authors=['Author']) + self.assertIsNone(book.deleted_at) + book.delete() + book.refresh_from_db() + self.assertIsNotNone(book.deleted_at) + + def test_default_manager_excludes_deleted(self): + book = Book.objects.create(title='Test Book', authors=['Author']) + book.delete() + self.assertEqual(Book.objects.count(), 0) + + def test_all_objects_includes_deleted(self): + book = Book.objects.create(title='Test Book', authors=['Author']) + book.delete() + self.assertEqual(Book.all_objects.count(), 1) + + +class BookFieldTest(TestCase): + def test_authors_stored_as_list(self): + book = Book.objects.create(title='Test', authors=['A', 'B']) + book.refresh_from_db() + self.assertEqual(book.authors, ['A', 'B']) + + def test_optional_fields_default_to_none(self): + book = Book.objects.create(title='Test', authors=[]) + self.assertIsNone(book.publication_year) + self.assertIsNone(book.average_rating) + self.assertIsNone(book.ratings_count) diff --git a/typesense-django-full-text-search/books/tests/test_sync.py b/typesense-django-full-text-search/books/tests/test_sync.py new file mode 100644 index 0000000..680b643 --- /dev/null +++ b/typesense-django-full-text-search/books/tests/test_sync.py @@ -0,0 +1,81 @@ +import datetime +from unittest.mock import patch, MagicMock +from django.test import TestCase +from django.utils import timezone +from books.models import Book +from books.search.sync import ( + _map_book_to_document, + run_full_sync, + run_incremental_sync, + get_last_sync_time, + _set_last_sync_time, + BATCH_SIZE, +) + + +class MapBookToDocumentTest(TestCase): + def test_maps_all_fields(self): + book = Book.objects.create( + title='Test', + authors=['A', 'B'], + publication_year=2020, + average_rating=4.5, + image_url='http://example.com/img.jpg', + ratings_count=100, + ) + doc = _map_book_to_document(book) + self.assertEqual(doc['id'], str(book.id)) + self.assertEqual(doc['title'], 'Test') + self.assertEqual(doc['authors'], ['A', 'B']) + self.assertEqual(doc['publication_year'], 2020) + self.assertAlmostEqual(doc['average_rating'], 4.5) + self.assertEqual(doc['image_url'], 'http://example.com/img.jpg') + self.assertEqual(doc['ratings_count'], 100) + + def test_handles_none_fields(self): + book = Book.objects.create(title='Minimal', authors=[]) + doc = _map_book_to_document(book) + self.assertEqual(doc['publication_year'], 0) + self.assertEqual(doc['average_rating'], 0.0) + self.assertEqual(doc['image_url'], '') + self.assertEqual(doc['ratings_count'], 0) + + +@patch('books.search.sync.typesense_client') +class IncrementalSyncPaginationTest(TestCase): + def test_incremental_sync_paginates(self, mock_client): + """Verify incremental sync doesn't load all records at once.""" + _set_last_sync_time(datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) + + # Create books + for i in range(5): + Book.objects.create(title=f'Book {i}', authors=['A']) + + mock_import = MagicMock() + mock_client.collections.__getitem__.return_value.documents.import_ = mock_import + + run_incremental_sync() + + # All 5 should be upserted (in one batch since < BATCH_SIZE) + mock_import.assert_called_once() + docs = mock_import.call_args[0][0] + self.assertEqual(len(docs), 5) + + +@patch('books.search.sync.typesense_client') +class FullSyncTest(TestCase): + def test_full_sync_processes_all_active_books(self, mock_client): + Book.objects.create(title='Active', authors=['A']) + deleted = Book.objects.create(title='Deleted', authors=['B']) + deleted.delete() + + mock_import = MagicMock() + mock_client.collections.__getitem__.return_value.documents.import_ = mock_import + + run_full_sync() + + # Only active book should be synced + mock_import.assert_called_once() + docs = mock_import.call_args[0][0] + self.assertEqual(len(docs), 1) + self.assertEqual(docs[0]['title'], 'Active') diff --git a/typesense-django-full-text-search/books/tests/test_views.py b/typesense-django-full-text-search/books/tests/test_views.py new file mode 100644 index 0000000..95e67e0 --- /dev/null +++ b/typesense-django-full-text-search/books/tests/test_views.py @@ -0,0 +1,100 @@ +import json +from unittest.mock import patch, MagicMock +from django.test import TestCase, Client +from books.models import Book + + +@patch('books.views.sync_book_to_typesense') +@patch('books.views.delete_book_from_typesense') +class BookViewTest(TestCase): + def setUp(self): + self.client = Client() + self.book = Book.objects.create( + title='Existing Book', + authors=['Author One'], + publication_year=2020, + ) + + def test_list_books(self, mock_delete, mock_sync): + response = self.client.get('/books/') + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data['total'], 1) + self.assertEqual(len(data['data']), 1) + + def test_list_books_invalid_page(self, mock_delete, mock_sync): + response = self.client.get('/books/?page=abc') + self.assertEqual(response.status_code, 400) + + def test_list_books_limit_clamped(self, mock_delete, mock_sync): + response = self.client.get('/books/?limit=999') + data = response.json() + self.assertEqual(data['limit'], 100) + + def test_list_books_negative_page(self, mock_delete, mock_sync): + response = self.client.get('/books/?page=-5') + data = response.json() + self.assertEqual(data['page'], 1) + + def test_create_book(self, mock_delete, mock_sync): + response = self.client.post( + '/books/', + data=json.dumps({'title': 'New Book', 'authors': ['A']}), + content_type='application/json', + ) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()['title'], 'New Book') + mock_sync.assert_called_once() + + def test_create_book_rejects_protected_fields(self, mock_delete, mock_sync): + response = self.client.post( + '/books/', + data=json.dumps({ + 'title': 'Hacked', + 'authors': ['X'], + 'id': 9999, + 'deleted_at': '2020-01-01T00:00:00Z', + }), + content_type='application/json', + ) + self.assertEqual(response.status_code, 201) + book = Book.objects.get(title='Hacked') + self.assertNotEqual(book.id, 9999) + self.assertIsNone(book.deleted_at) + + def test_update_book_allowlist(self, mock_delete, mock_sync): + response = self.client.put( + f'/books/{self.book.id}/', + data=json.dumps({'title': 'Updated', 'deleted_at': '2020-01-01T00:00:00Z'}), + content_type='application/json', + ) + self.assertEqual(response.status_code, 200) + self.book.refresh_from_db() + self.assertEqual(self.book.title, 'Updated') + self.assertIsNone(self.book.deleted_at) + + def test_delete_returns_204_no_body(self, mock_delete, mock_sync): + response = self.client.delete(f'/books/{self.book.id}/') + self.assertEqual(response.status_code, 204) + self.assertEqual(response.content, b'') + mock_delete.assert_called_once_with(self.book.id) + + def test_get_nonexistent_book_returns_404(self, mock_delete, mock_sync): + response = self.client.get('/books/99999/') + self.assertEqual(response.status_code, 404) + + +@patch('books.views.typesense_client') +class SearchViewTest(TestCase): + def test_search_returns_results(self, mock_client): + mock_search = MagicMock(return_value={ + 'found': 1, + 'hits': [{'document': {'title': 'Test'}}], + 'facet_counts': [], + }) + mock_client.collections.__getitem__.return_value.documents.search = mock_search + + response = self.client.get('/search/?q=test') + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data['found'], 1) diff --git a/typesense-django-full-text-search/books/urls.py b/typesense-django-full-text-search/books/urls.py new file mode 100644 index 0000000..ff5830a --- /dev/null +++ b/typesense-django-full-text-search/books/urls.py @@ -0,0 +1,10 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('books/', views.books_list_create, name='books_list_create'), + path('books//', views.books_detail, name='books_detail'), + path('search/', views.search, name='search'), + path('sync/', views.manual_sync, name='manual_sync'), + path('sync/status/', views.sync_status, name='sync_status'), +] diff --git a/typesense-django-full-text-search/books/views.py b/typesense-django-full-text-search/books/views.py new file mode 100644 index 0000000..59f0e2c --- /dev/null +++ b/typesense-django-full-text-search/books/views.py @@ -0,0 +1,140 @@ +import json +import logging +from django.http import JsonResponse, HttpResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from .models import Book +from .search import typesense_client, BOOKS_COLLECTION_NAME, run_full_sync, get_sync_status +from .search.sync import _map_book_to_document, get_last_sync_time + +logger = logging.getLogger(__name__) + +# Allowlist of fields that can be set via API +MUTABLE_FIELDS = {'title', 'authors', 'publication_year', 'average_rating', 'image_url', 'ratings_count'} + +def _book_to_response(book): + return { + 'id': book.id, + 'title': book.title, + 'authors': book.authors, + 'publication_year': book.publication_year, + 'average_rating': book.average_rating, + 'image_url': book.image_url, + 'ratings_count': book.ratings_count, + } + +def sync_book_to_typesense(book): + try: + document = _map_book_to_document(book) + typesense_client.collections[BOOKS_COLLECTION_NAME].documents.upsert(document) + except Exception as err: + logger.error('Failed to sync book %d to Typesense: %s', book.id, err) + +def delete_book_from_typesense(book_id): + try: + typesense_client.collections[BOOKS_COLLECTION_NAME].documents[str(book_id)].delete() + except Exception as err: + logger.error('Failed to delete book %d from Typesense: %s', book_id, err) + +@csrf_exempt +@require_http_methods(["GET", "POST"]) +def books_list_create(request): + if request.method == 'GET': + try: + page = max(int(request.GET.get('page', 1)), 1) + limit = min(max(int(request.GET.get('limit', 10)), 1), 100) + except (ValueError, TypeError): + return JsonResponse({'error': 'page and limit must be integers'}, status=400) + + offset = (page - 1) * limit + + queryset = Book.objects.all().order_by('id') + total = queryset.count() + books = list(queryset[offset:offset+limit].values()) + + return JsonResponse({ + 'total': total, + 'page': page, + 'limit': limit, + 'data': books + }) + + elif request.method == 'POST': + try: + data = json.loads(request.body) + filtered = {k: v for k, v in data.items() if k in MUTABLE_FIELDS} + book = Book.objects.create(**filtered) + sync_book_to_typesense(book) + return JsonResponse(_book_to_response(book), status=201) + except Exception as e: + return JsonResponse({'error': str(e)}, status=400) + +@csrf_exempt +@require_http_methods(["GET", "PUT", "DELETE"]) +def books_detail(request, pk): + try: + book = Book.objects.get(pk=pk) + except Book.DoesNotExist: + return JsonResponse({'error': 'Book not found'}, status=404) + + if request.method == 'GET': + return JsonResponse(_book_to_response(book)) + + elif request.method == 'PUT': + try: + data = json.loads(request.body) + for key, value in data.items(): + if key in MUTABLE_FIELDS: + setattr(book, key, value) + book.save() + sync_book_to_typesense(book) + return JsonResponse(_book_to_response(book)) + except Exception as e: + return JsonResponse({'error': str(e)}, status=400) + + elif request.method == 'DELETE': + try: + book_id = book.id + book.delete() + delete_book_from_typesense(book_id) + return HttpResponse(status=204) + except Exception as e: + return JsonResponse({'error': str(e)}, status=500) + +@require_http_methods(["GET"]) +def search(request): + query = request.GET.get('q', '') + try: + search_results = typesense_client.collections[BOOKS_COLLECTION_NAME].documents.search({ + 'q': query, + 'query_by': 'title,authors', + }) + return JsonResponse({ + 'query': query, + 'found': search_results.get('found', 0), + 'results': search_results.get('hits', []), + 'facet_counts': search_results.get('facet_counts', []), + }) + except Exception as error: + return JsonResponse({'error': str(error)}, status=500) + +@csrf_exempt +@require_http_methods(["POST"]) +def manual_sync(request): + try: + run_full_sync() + return JsonResponse({ + 'message': 'Sync completed', + 'syncedAt': get_last_sync_time().isoformat() + }) + except Exception as error: + return JsonResponse({'error': str(error)}, status=500) + +@require_http_methods(["GET"]) +def sync_status(request): + status = get_sync_status() + return JsonResponse({ + 'lastSyncTime': get_last_sync_time().isoformat(), + 'syncWorkerRunning': status['syncWorkerRunning'], + 'syncJobActive': status['syncJobActive'] + }) diff --git a/typesense-django-full-text-search/manage.py b/typesense-django-full-text-search/manage.py new file mode 100755 index 0000000..fa3945e --- /dev/null +++ b/typesense-django-full-text-search/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'typesensedjango.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/typesense-django-full-text-search/requirements.txt b/typesense-django-full-text-search/requirements.txt new file mode 100644 index 0000000..c566377 --- /dev/null +++ b/typesense-django-full-text-search/requirements.txt @@ -0,0 +1,5 @@ +django>=5.0,<7.0 +typesense>=0.21.0 +psycopg2-binary>=2.9 +python-dotenv>=1.0 +apscheduler>=3.10,<4.0 diff --git a/typesense-django-full-text-search/typesensedjango/__init__.py b/typesense-django-full-text-search/typesensedjango/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/typesense-django-full-text-search/typesensedjango/asgi.py b/typesense-django-full-text-search/typesensedjango/asgi.py new file mode 100644 index 0000000..5e446a8 --- /dev/null +++ b/typesense-django-full-text-search/typesensedjango/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for typesensedjango project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/6.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'typesensedjango.settings') + +application = get_asgi_application() diff --git a/typesense-django-full-text-search/typesensedjango/settings.py b/typesense-django-full-text-search/typesensedjango/settings.py new file mode 100644 index 0000000..efb77af --- /dev/null +++ b/typesense-django-full-text-search/typesensedjango/settings.py @@ -0,0 +1,127 @@ +""" +Django settings for typesensedjango project. + +Generated by 'django-admin startproject' using Django 6.0.6. + +For more information on this file, see +https://docs.djangoproject.com/en/6.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/6.0/ref/settings/ +""" + +from pathlib import Path +import os +from dotenv import load_dotenv + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Load environment variables from .env file +load_dotenv(BASE_DIR / '.env') + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/6.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-*x$rqyx0w#d1kz*=t@vhn87-%f#bmml=9sc4a8i2pbo6q+!!&f') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'books', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'typesensedjango.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'typesensedjango.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/6.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': os.getenv('DB_NAME', 'typesense_books'), + 'USER': os.getenv('DB_USER', 'postgres'), + 'PASSWORD': os.getenv('DB_PASSWORD', 'password'), + 'HOST': os.getenv('DB_HOST', 'localhost'), + 'PORT': os.getenv('DB_PORT', '5432'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/6.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/6.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/6.0/howto/static-files/ + +STATIC_URL = 'static/' diff --git a/typesense-django-full-text-search/typesensedjango/urls.py b/typesense-django-full-text-search/typesensedjango/urls.py new file mode 100644 index 0000000..d1f6882 --- /dev/null +++ b/typesense-django-full-text-search/typesensedjango/urls.py @@ -0,0 +1,23 @@ +""" +URL configuration for typesensedjango project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/6.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', include('books.urls')), +] diff --git a/typesense-django-full-text-search/typesensedjango/wsgi.py b/typesense-django-full-text-search/typesensedjango/wsgi.py new file mode 100644 index 0000000..d03953b --- /dev/null +++ b/typesense-django-full-text-search/typesensedjango/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for typesensedjango project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/6.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'typesensedjango.settings') + +application = get_wsgi_application()