diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5246f89f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,71 @@ +# Git +.git +.gitignore +.gitattributes + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +.venv +*.egg-info/ +dist/ +build/ +*.egg + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Docker (don't copy docker files into the image) +Dockerfile +docker-compose*.yml +.dockerignore + +# Database lock files +*.db +*.sqlite +*.sqlite3 +.db_initialized + +# Logs +logs/ +*.log + +# Test files +.coverage +htmlcov/ +.pytest_cache/ +.tox/ + +# CI/CD +.github/ +.travis.yml +.circleci/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# Secrets and credentials (prevent accidental inclusion) +*.pem +*.key +service-account.json +gcp-key.json +secret_key +secret_csrf +config.py +.env diff --git a/.gitattributes b/.gitattributes index 0b222972..1e62db73 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ -*.bat text eol=crlf \ No newline at end of file +*.bat text eol=crlf +*.sh text eol=lf \ No newline at end of file diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 00000000..61eb3575 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,82 @@ +# Running the Sample Platform with Docker + +A two-container development stack: MySQL 8 and the Flask application served by +Gunicorn. It gives contributors a working platform without installing MySQL, +Python, or the native libraries the app depends on. + +## Prerequisites + +- Docker Engine 24+ with the Compose plugin (`docker compose`). + +## Quick start (empty database) + +```sh +cp env.example .env # then edit the passwords +docker compose up --build +``` + +On first start the database initialises, the app waits for it, builds the +schema, loads the bundled fixture set (categories, samples, regression tests) +and serves on . Later starts only apply migrations +newer than the database, and never re-seed. + +Create an administrator so you can sign in: + +```sh +docker compose exec backend \ + python install/init_db.py "$SQLALCHEMY_DATABASE_URI" admin admin@example.com admin +``` + +## Starting from a database dump + +To develop against real data, load a `mysqldump` on the database's **first** +start by dropping it into the init directory. Create `docker-compose.override.yml`: + +```yaml +services: + db: + volumes: + - /absolute/path/to/dump.sql:/docker-entrypoint-initdb.d/01-dump.sql:ro +``` + +Then `docker compose up --build`. MySQL imports the dump before the app starts; +the app applies any migrations newer than the dump on top of it. The import +only runs while the `db_data` volume is empty — `docker compose down -v` first +to reload a different dump. + +## Live reload + +The image is self-contained (code is copied in, not mounted). For an +edit-refresh loop, mount the package you are working on and enable Gunicorn's +reloader in the override file: + +```yaml +services: + backend: + environment: + GUNICORN_RELOAD: "1" + volumes: + - ./mod_sample:/app/mod_sample + - ./templates:/app/templates +``` + +## Common commands + +| Task | Command | +|---|---| +| Start | `docker compose up --build` | +| Stop | `docker compose down` | +| Reset the database | `docker compose down -v` | +| App logs | `docker compose logs -f backend` | +| A shell in the app | `docker compose exec backend bash` | +| A MySQL shell | `docker compose exec db mysql -u root -p` | + +## Notes + +- The database port is not published to the host. Reach MySQL through + `docker compose exec`, or add a `ports` mapping in an override if you need a + local client. +- The image generates throwaway secret keys and GCP credentials at build time + so the app can boot offline. They are not suitable for production. +- Storage falls back to the local `/repository` volume; no GCS bucket is + required for development. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..77fe8d74 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,72 @@ +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + FLASK_APP=run.py + +# git for the build commit run.py reads, libmagic for upload sniffing, +# mediainfo for sample metadata. No compiler: everything has a wheel. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git libmagic1 mediainfo \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Dependencies first so application edits don't invalidate this layer. +# Pinned, and wheels only, so nothing runs a setup script during the build. +# cryptography is used by the credential generator further down. +COPY requirements.txt . +RUN pip install --only-binary :all: --upgrade \ + pip==26.2 setuptools==83.0.0 wheel==0.47.0 \ + && pip install --only-binary :all: \ + cryptography==50.0.0 gunicorn==26.0.0 \ + && pip install --only-binary :all: -r requirements.txt + +# Listed by name instead of "COPY . ." so a secret_key, config.py or +# service-account.json left in a working tree can't end up in the image. +COPY run.py config_parser.py database.py decorators.py exceptions.py \ + log_configuration.py mailer.py utility.py ./ +COPY mod_api/ ./mod_api/ +COPY mod_auth/ ./mod_auth/ +COPY mod_ci/ ./mod_ci/ +COPY mod_customized/ ./mod_customized/ +COPY mod_health/ ./mod_health/ +COPY mod_home/ ./mod_home/ +COPY mod_regression/ ./mod_regression/ +COPY mod_sample/ ./mod_sample/ +COPY mod_test/ ./mod_test/ +COPY mod_upload/ ./mod_upload/ +COPY install/ ./install/ +COPY migrations/ ./migrations/ +COPY static/ ./static/ +COPY templates/ ./templates/ + +# The app reads its settings from config.py; this variant takes them from +# the environment. +COPY config.docker.py config.py + +# Done once here instead of on every start: unprivileged user, secret keys, +# throwaway GCP credentials, the repository tree (named volumes inherit this +# layout on first mount) and a git repo so run.py can resolve a commit. +RUN useradd --create-home --uid 1001 appuser \ + && python install/generate_dev_credentials.py \ + && head -c 32 /dev/urandom > secret_key \ + && head -c 32 /dev/urandom > secret_csrf \ + && mkdir -p logs \ + /repository/ci-tests /repository/unsafe-ccextractor /repository/TempFiles \ + /repository/LogFiles /repository/TestResults /repository/TestFiles/media \ + /repository/QueuedFiles /repository/TestData/ci-linux \ + /repository/TestData/ci-windows /repository/vm_data \ + && git init -q . \ + && git -c user.email=dev@local -c user.name=docker add -A \ + && git -c user.email=dev@local -c user.name=docker commit -qm "container image" \ + && chown -R appuser:appuser /app /repository + +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN sed -i 's/\r$//' /usr/local/bin/docker-entrypoint.sh \ + && chmod +x /usr/local/bin/docker-entrypoint.sh + +USER appuser +EXPOSE 5000 +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/config.docker.py b/config.docker.py new file mode 100644 index 00000000..0feb840c --- /dev/null +++ b/config.docker.py @@ -0,0 +1,60 @@ +"""Container configuration for the Sample Platform. + +Copied to ``config.py`` inside the image at build time. Every value is read +from the environment (see ``env.example``) with local-development defaults, +so the repository never carries a real secret. Override every placeholder +before pointing this at anything but a throwaway database. +""" +import os + + +def _int(name: str, default: int) -> int: + try: + return int(os.environ[name]) + except (KeyError, ValueError): + return default + + +APPLICATION_ROOT = None +CSRF_ENABLED = True + +DATABASE_URI = os.environ.get( + "SQLALCHEMY_DATABASE_URI", + "mysql+pymysql://sample_platform:sample_platform@db/sample_platform?charset=utf8mb4", +) +SERVER_NAME = os.environ.get("SERVER_NAME", "localhost:5000") +SESSION_COOKIE_PATH = "/" + +INSTALL_FOLDER = os.environ.get("INSTALL_FOLDER", "/app") +SAMPLE_REPOSITORY = os.environ.get("SAMPLE_REPOSITORY", "/repository") + +HMAC_KEY = os.environ.get("HMAC_KEY", "dev-hmac-key") +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") +GITHUB_OWNER = os.environ.get("GITHUB_OWNER", "CCExtractor") +GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "ccextractor") +GITHUB_CI_KEY = os.environ.get("GITHUB_CI_KEY", "") +GITHUB_CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID", "") +GITHUB_CLIENT_KEY = os.environ.get("GITHUB_CLIENT_KEY", "") + +EMAIL_DOMAIN = os.environ.get("EMAIL_DOMAIN", "") +EMAIL_API_KEY = os.environ.get("EMAIL_API_KEY", "") + +FTP_PORT = _int("FTP_PORT", 21) +MAX_CONTENT_LENGTH = 512 * 1024 * 1024 +MIN_PWD_LEN = 10 +MAX_PWD_LEN = 500 + +# GCP / Cloud Storage. The build generates a throwaway service account so the +# storage client can initialise offline; file serving falls back to local disk. +SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] +SERVICE_ACCOUNT_FILE = os.environ.get("SERVICE_ACCOUNT_FILE", "service-account.json") +ZONE = "us-west4-b" +PROJECT_NAME = "ccextractor-sampleplatform" +MACHINE_TYPE = f"zones/{ZONE}/machineTypes/n1-standard-1" +WINDOWS_INSTANCE_PROJECT_NAME = "windows-cloud" +WINDOWS_INSTANCE_FAMILY_NAME = "windows-2019" +LINUX_INSTANCE_PROJECT_NAME = "ubuntu-os-cloud" +LINUX_INSTANCE_FAMILY_NAME = "ubuntu-minimal-2404-lts-amd64" +GCP_INSTANCE_MAX_RUNTIME = 120 +GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", "sample-platform-dev") +GCS_SIGNED_URL_EXPIRY_LIMIT = 720 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..27d2a05b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +services: + db: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD in .env} + MYSQL_DATABASE: ${MYSQL_DATABASE:-sample_platform} + MYSQL_USER: ${MYSQL_USER:-sample_platform} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD in .env} + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - db_data:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}"] + interval: 5s + timeout: 5s + retries: 20 + networks: + - sample_platform + + backend: + build: . + depends_on: + db: + condition: service_healthy + environment: + SQLALCHEMY_DATABASE_URI: mysql+pymysql://${MYSQL_USER:-sample_platform}:${MYSQL_PASSWORD}@db/${MYSQL_DATABASE:-sample_platform}?charset=utf8mb4 + DB_HOST: db + DB_PORT: "3306" + DB_USER: ${MYSQL_USER:-sample_platform} + DB_PASSWORD: ${MYSQL_PASSWORD} + SERVER_NAME: ${SERVER_NAME:-localhost:5000} + HMAC_KEY: ${HMAC_KEY:-dev-hmac-key} + GCS_BUCKET_NAME: ${GCS_BUCKET_NAME:-sample-platform-dev} + GITHUB_OWNER: ${GITHUB_OWNER:-CCExtractor} + GITHUB_REPOSITORY: ${GITHUB_REPOSITORY:-ccextractor} + DEBUG: ${DEBUG:-True} + ports: + - "${APP_PORT:-5000}:5000" + volumes: + - repository_data:/repository + networks: + - sample_platform + restart: unless-stopped + +networks: + sample_platform: + +volumes: + db_data: + repository_data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 00000000..3fb0160b --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +# /app is a git repo created at build time; run.py reads a build commit from +# it. safe.directory guards against ownership mismatches under a bind mount. +git config --global --add safe.directory /app 2>/dev/null || true + +python install/wait_for_db.py + +# Creates the schema on an empty database, applies pending migrations on an +# existing one. See install/init_schema.py for why it is not just an upgrade. +python install/init_schema.py + +exec gunicorn \ + --workers "${GUNICORN_WORKERS:-3}" \ + --bind 0.0.0.0:5000 \ + --timeout 120 \ + --access-logfile - \ + --error-logfile - \ + ${GUNICORN_RELOAD:+--reload} \ + run:app diff --git a/env.example b/env.example new file mode 100644 index 00000000..1a2433a4 --- /dev/null +++ b/env.example @@ -0,0 +1,19 @@ +# Sample Platform — Docker environment. Copy to .env and edit before use. + +# MySQL. The root password stays with the database container and is never +# passed to the application, which connects only as the user below. +MYSQL_ROOT_PASSWORD=change-me-root +MYSQL_DATABASE=sample_platform +MYSQL_USER=sample_platform +MYSQL_PASSWORD=change-me-app + +# Host port for the web app (the container always listens on 5000). +APP_PORT=5000 + +# Application settings. +SERVER_NAME=localhost:5000 +HMAC_KEY=change-me +GCS_BUCKET_NAME=sample-platform-dev +GITHUB_OWNER=CCExtractor +GITHUB_REPOSITORY=ccextractor +DEBUG=True diff --git a/install/generate_dev_credentials.py b/install/generate_dev_credentials.py new file mode 100644 index 00000000..b9a24540 --- /dev/null +++ b/install/generate_dev_credentials.py @@ -0,0 +1,44 @@ +"""Write a throwaway Google service-account file for local development. + +run.py builds a storage client at import time, which needs a syntactically +valid service-account JSON even when there is no GCP project behind it. The +key generated here authenticates nothing. +""" +import json +from pathlib import Path + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +# Matches SERVICE_ACCOUNT_FILE in config.docker.py. Not a command line +# argument: nothing needs to pick the location, and accepting one would only +# be a way to write outside the application directory. +TARGET = Path(__file__).resolve().parent.parent / 'service-account.json' + + +def main() -> None: + """Write the credentials file unless one is already present.""" + if TARGET.exists(): + return + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ).decode() + + TARGET.write_text(json.dumps({ + "type": "service_account", + "project_id": "sample-platform-dev", + "private_key_id": "dev", + "private_key": private_key, + "client_email": "dev@sample-platform-dev.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/install/init_schema.py b/install/init_schema.py new file mode 100644 index 00000000..0754d8f2 --- /dev/null +++ b/install/init_schema.py @@ -0,0 +1,66 @@ +"""Bring the database schema up to date, creating it on the first start. + +The migration chain cannot be replayed against an empty database: early +revisions drop foreign keys by the names MySQL generates automatically +(``regression_test_ibfk_1`` and friends), and those exist only when the +schema was first built from the models rather than from the migrations. A +fresh database is therefore created from the models and stamped at the +current head, which is what the application does anyway on its first +request. A database that already carries an ``alembic_version`` table -- an +existing environment, or one restored from a dump -- only gets the pending +migrations applied. + +Creating the tables is deliberately skipped in that second case: doing both +would let ``create_all`` add a table that a pending migration still expects +to create itself, which then fails. + +A fresh database is also seeded, because several pages read rows they assume +are always present -- the home page dereferences the ``last_commit`` entry +without a null check -- and an empty schema alone therefore serves a 500. +""" +import subprocess +import sys +from os import path + +from flask_migrate import stamp, upgrade +from sqlalchemy import create_engine, inspect + +# Need to append server root path to ensure we can import the necessary files. +ROOT = path.dirname(path.dirname(path.abspath(__file__))) +sys.path.append(ROOT) + + +def main() -> int: + """Create or migrate the schema, depending on what is already there.""" + from database import create_session + from run import app, config + + uri = config['DATABASE_URI'] + engine = create_engine(uri) + try: + established = inspect(engine).has_table('alembic_version') + finally: + engine.dispose() + + with app.app_context(): + if established: + print('existing database: applying pending migrations', flush=True) + upgrade() + else: + print('fresh database: creating schema from the models', flush=True) + create_session(uri) + stamp() + + if not established: + # Run out of process: sample_db seeds on import and reads the URI from + # argv, so it cannot simply be called. + print('seeding development data', flush=True) + subprocess.check_call( + [sys.executable, path.join(ROOT, 'install', 'sample_db.py'), uri]) + + print('schema is up to date', flush=True) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/install/wait_for_db.py b/install/wait_for_db.py new file mode 100644 index 00000000..634b1471 --- /dev/null +++ b/install/wait_for_db.py @@ -0,0 +1,39 @@ +"""Block until the database accepts connections, then exit. + +The Docker entrypoint runs this before starting the app so migrations and +the web server never race the database container. Connection details come +from the environment; see ``env.example``. +""" +import os +import sys +import time + +import pymysql + + +def main() -> int: + host = os.environ.get("DB_HOST", "db") + port = int(os.environ.get("DB_PORT", "3306")) + user = os.environ.get("DB_USER", "sample_platform") + password = os.environ.get("DB_PASSWORD", "sample_platform") + timeout = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + pymysql.connect( + host=host, port=port, user=user, password=password, + connect_timeout=3, + ).close() + print(f"database reachable at {host}:{port}", flush=True) + return 0 + except pymysql.Error as exc: + print(f"waiting for database at {host}:{port} ({exc})", flush=True) + time.sleep(2) + + print(f"gave up waiting for database after {timeout}s", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utility.py b/utility.py index 98eeec53..ecc8df7e 100644 --- a/utility.py +++ b/utility.py @@ -10,7 +10,8 @@ import requests import werkzeug -from flask import abort, g, redirect, request +from flask import abort, g, redirect, request, send_file +from google.auth.exceptions import GoogleAuthError ROOT_DIR = path.dirname(path.abspath(__file__)) @@ -19,6 +20,11 @@ def serve_file_download(file_name, file_folder, file_sub_folder='') -> werkzeug. """ Serve file download by redirecting using Signed Download URLs. + Falls back to the copy in the sample repository when the service account + does not authenticate, which is the case in the Docker development stack: + there the credentials are generated locally and ``/repository`` is a plain + volume rather than a gcsfuse mount backed by a bucket. + :param file_name: name of the file :type file_name: str :param file_folder: name of the folder @@ -31,15 +37,25 @@ def serve_file_download(file_name, file_folder, file_sub_folder='') -> werkzeug. from run import config, storage_client_bucket file_path = path.join(file_folder, file_sub_folder, file_name) - blob = storage_client_bucket.blob(file_path) - blob.content_disposition = f'attachment; filename="{file_name}"' - blob.patch() - url = blob.generate_signed_url( - version="v4", - expiration=timedelta(minutes=config.get('GCS_SIGNED_URL_EXPIRY_LIMIT', '')), - method="GET", - ) - return redirect(url) + + try: + blob = storage_client_bucket.blob(file_path) + blob.content_disposition = f'attachment; filename="{file_name}"' + blob.patch() + url = blob.generate_signed_url( + version="v4", + expiration=timedelta(minutes=config.get('GCS_SIGNED_URL_EXPIRY_LIMIT', '')), + method="GET", + ) + return redirect(url) + except GoogleAuthError: + # Only raised when the token exchange itself fails, so a working + # service account never lands here and a Cloud Storage outage still + # surfaces in production instead of being served a stale local copy. + local_path = path.join(config.get('SAMPLE_REPOSITORY', ''), file_path) + if path.isfile(local_path): + return send_file(local_path, as_attachment=True, download_name=file_name) + return abort(404) def request_from_github(abort_code: int = 418) -> Callable: