From 117384c0ab6464640790e3189958ce76b2021d20 Mon Sep 17 00:00:00 2001 From: Leonardo Ayala Date: Wed, 17 Jun 2026 15:05:45 +0200 Subject: [PATCH 1/4] Creates initial skeleton of project, including basic GH actions, smoke tests, pre-commit hooks and packaging configuration. --- .github/release_drafter.yaml | 34 +++++++ .github/workflows/pre-commit.yaml | 14 +++ .github/workflows/release.yaml | 32 +++++++ .github/workflows/release_drafter.yaml | 17 ++++ .pre-commit-config.yaml | 120 +++++++++++++++++++++++ .python-version | 1 + .secrets.baseline | 127 +++++++++++++++++++++++++ README.md | 25 +++++ pyproject.toml | 100 +++++++++++++++++++ src/dstrack/__init__.py | 5 + tests/smoke_test.py | 45 +++++++++ uv.lock | 8 ++ 12 files changed, 528 insertions(+) create mode 100644 .github/release_drafter.yaml create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/release_drafter.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version create mode 100644 .secrets.baseline create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 src/dstrack/__init__.py create mode 100644 tests/smoke_test.py create mode 100644 uv.lock diff --git a/.github/release_drafter.yaml b/.github/release_drafter.yaml new file mode 100644 index 0000000..57f39d3 --- /dev/null +++ b/.github/release_drafter.yaml @@ -0,0 +1,34 @@ +name-template: "v$RESOLVED_VERSION 🌈" +tag-template: "v$RESOLVED_VERSION" +categories: + - title: "🚀 Features" + semver-increment: minor + when: + labels: + - "feature" + - "enhancement" + - title: "🐛 Bug Fixes" + when: + labels: + - "fix" + - "bugfix" + - "bug" + - title: "🧰 Maintenance" + when: + label: "chore" + - type: "pre-exclude" + when: + label: "skip-changelog" + - type: "version-resolver" + semver-increment: "major" + when: + label: "major" + - type: "version-resolver" + semver-increment: "patch" +change-template: "- $TITLE @$AUTHOR (#$NUMBER)" +change-title-escapes: '\<*_&' +tag-prefix: "v" +template: | + ## Changes + + $CHANGES diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..2b11178 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..f247399 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,32 @@ +name: "Publish release to PyPI" + +on: + push: + tags: + # Publish on any tag starting with a `v`, e.g., v0.1.0 + - v* + +jobs: + run: + runs-on: ubuntu-latest + environment: + name: pypi + permissions: + id-token: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - name: Install Python 3.13 + run: uv python install 3.13 + - name: Build + run: uv build + # Check that basic features work and we didn't miss to include crucial files + - name: Smoke test (wheel) + run: uv run --isolated --no-project --with dist/*.whl tests/smoke_test.py + - name: Smoke test (source distribution) + run: uv run --isolated --no-project --with dist/*.tar.gz tests/smoke_test.py + - name: Publish + run: uv publish diff --git a/.github/workflows/release_drafter.yaml b/.github/workflows/release_drafter.yaml new file mode 100644 index 0000000..290d3ee --- /dev/null +++ b/.github/workflows/release_drafter.yaml @@ -0,0 +1,17 @@ +name: Release Drafter + +on: + workflow_dispatch: + +# Permissions for default token (secrets.GITHUB_TOKEN) +permissions: + contents: write + pull-requests: read + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v7 + with: + config-name: release-drafter.yml # the default, loads '.github/release-drafter.yml' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cc3c4e8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,120 @@ +# .pre-commit-config.yaml +# +# Setup: +# pip install pre-commit # or: uv tool install pre-commit +# pre-commit install # installs the git hook +# pre-commit install --hook-type commit-msg # only if you enable commitizen below +# pre-commit run --all-files # run once across the whole repo +# +# Maintenance: +# pre-commit autoupdate # bump all `rev:` pins to latest +# +# Docs: https://pre-commit.com | Hook index: https://pre-commit.com/hooks.html + +minimum_pre_commit_version: "3.5.0" + +default_language_version: + python: python3.11 + +# Keep commits fast: only run on the commit stage by default. +default_stages: [pre-commit] + +# Don't run hooks on generated / vendored / data paths. +exclude: | + (?x)^( + \.git/.*| + \.venv/.*| + build/.*| + dist/.*| + docs/_build/.*| + .*\.egg-info/.*| + data/.*| + datasets/.*| + .*\.ipynb_checkpoints/.* + )$ + +repos: + # --------------------------------------------------------------------------- + # Core hygiene: cheap, fast, catch-everything checks. + # --------------------------------------------------------------------------- + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: check-yaml + - id: check-toml + - id: check-json + - id: check-merge-conflict + - id: check-case-conflict + - id: check-symlinks + - id: debug-statements # blocks stray breakpoint()/pdb + - id: detect-private-key # blocks committed private keys + - id: check-added-large-files + args: [--maxkb=1024] + + # --------------------------------------------------------------------------- + # Ruff: linter + formatter (replaces flake8, isort, black, pyupgrade, etc.). + # Lint (with --fix) MUST run before the formatter. + # --------------------------------------------------------------------------- + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.17 + hooks: + - id: ruff-check + args: [--fix] + types_or: [python, pyi, jupyter] + - id: ruff-format + types_or: [python, pyi, jupyter] + + # --------------------------------------------------------------------------- + # Static type checking (run on the package source only, to stay fast). + # --------------------------------------------------------------------------- + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v2.1.0 + hooks: + - id: mypy + files: ^src/ + args: [--ignore-missing-imports, --install-types, --non-interactive] + + # --------------------------------------------------------------------------- + # Spelling: catches typos in code, docstrings, and docs. + # --------------------------------------------------------------------------- + - repo: https://github.com/codespell-project/codespell + rev: v2.4.2 + hooks: + - id: codespell + additional_dependencies: [tomli] + # Configure ignores in pyproject.toml under [tool.codespell]. + + # --------------------------------------------------------------------------- + # Notebooks: strip outputs before commit (clean diffs, no data leakage). + # Essential for ML repos that carry .ipynb files. + # --------------------------------------------------------------------------- + - repo: https://github.com/kynan/nbstripout + rev: 0.9.1 + hooks: + - id: nbstripout + + # --------------------------------------------------------------------------- + # Secret scanning (baseline-driven). Generate the baseline once with: + # detect-secrets scan > .secrets.baseline + # --------------------------------------------------------------------------- + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: [--baseline, .secrets.baseline] + + + # --------------------------------------------------------------------------- + # uv lock file update to make sure it is properly synced even if the + # dependencies have been updated. + # --------------------------------------------------------------------------- + - repo: https://github.com/astral-sh/uv-pre-commit + # uv version. + rev: 0.11.21 + hooks: + - id: uv-lock diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..e538df4 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,127 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2026-06-17T12:55:45Z" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..b82aff9 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# dstrack + +A Python package for versioning and monitoring dataset changes throughout the machine learning lifecycle. + +## Overview + +`dstrack` helps data scientists and ML engineers track how datasets evolve over time, catching schema drift, distribution shifts, and unexpected mutations before they silently break pipelines or degrade model performance. + +## Installation + +```bash +pip install dstrack +``` + +## Quickstart + +```bash +dstrack +``` + +## Features + +- Dataset versioning across ML pipeline stages +- Change detection and drift monitoring +- Lightweight CLI interface diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f261f68 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[project] +name = "dstrack" +version = "0.1.0" +description = "A Python package for versioning and monitoring dataset changes throughout the machine learning lifecycle." +readme = "README.md" +license = { file = "LICENSE" } +authors = [ + { name = "Leonardo Ayala" } +] +keywords = ["dataset", "versioning", "monitoring", "machine learning", "mlops", "data science"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", +] +requires-python = ">=3.11" +dependencies = [] + +[project.urls] +GitHub = "https://github.com/leoyala/dstrack" + +[project.scripts] +dstrack = "dstrack:main" + +[build-system] +requires = ["uv_build>=0.11.21,<0.12.0"] +build-backend = "uv_build" + +# ----------------------------------------------------------------------------- +# Ruff: linter + formatter +# ----------------------------------------------------------------------------- +[tool.ruff] +line-length = 88 +src = ["src", "tests"] + +[tool.ruff.lint] +# A pragmatic, high-signal rule set. +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort (import sorting) + "B", # flake8-bugbear (likely bugs) + "UP", # pyupgrade (modernize syntax) + "SIM", # flake8-simplify + "C4", # flake8-comprehensions + "PERF", # perflint (performance anti-patterns) + "RUF", # ruff-specific rules +] +ignore = [ + "E501", # line length is handled by the formatter +] + +[tool.ruff.lint.per-file-ignores] +# Tests can use assert, magic values, etc. +"tests/**" = ["B", "S101"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.format] +docstring-code-format = true +quote-style = "double" + +# ----------------------------------------------------------------------------- +# mypy: static type checking +# ----------------------------------------------------------------------------- +[tool.mypy] +files = ["src"] +python_version = "3.11" +strict = true +show_error_codes = true +warn_unreachable = true +warn_redundant_casts = true +warn_unused_ignores = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + +# Silence missing stubs for third-party libs that don't ship types. +[[tool.mypy.overrides]] +module = [ + "numpy.*", + "pandas.*", + "sklearn.*", +] +ignore_missing_imports = true + +# ----------------------------------------------------------------------------- +# codespell: spell-checking for code, docstrings, and docs +# ----------------------------------------------------------------------------- +[tool.codespell] +ignore-words-list = "nd,hist,ans" +skip = "*.lock,*.csv,*.parquet,./data,./datasets" diff --git a/src/dstrack/__init__.py b/src/dstrack/__init__.py new file mode 100644 index 0000000..22aa852 --- /dev/null +++ b/src/dstrack/__init__.py @@ -0,0 +1,5 @@ +__version__ = "0.1.0" + + +def main() -> None: + print("Hello from dstrack!") diff --git a/tests/smoke_test.py b/tests/smoke_test.py new file mode 100644 index 0000000..b569764 --- /dev/null +++ b/tests/smoke_test.py @@ -0,0 +1,45 @@ +"""Smoke test, run against the built wheel/sdist, not the source tree. + +Usage (matches the release workflow): + uv run --isolated --no-project --with dist/*.whl tests/smoke_test.py +""" + +import sys + + +def test_import() -> None: + import dstrack # noqa: F401 + + +def test_version() -> None: + import dstrack + + assert isinstance(dstrack.__version__, str) and dstrack.__version__, ( + "__version__ must be a non-empty string" + ) + + +def test_cli_entry_point() -> None: + # Verifies the entry point function is importable and callable (the script + # wiring in pyproject.toml points here). + from dstrack import main + + main() + + +if __name__ == "__main__": + tests = [test_import, test_version, test_cli_entry_point] + failed = [] + for t in tests: + try: + t() + print(f" ok {t.__name__}") + except Exception as exc: + print(f"FAIL {t.__name__}: {exc}") + failed.append(t.__name__) + + if failed: + print(f"\n{len(failed)} test(s) failed.") + sys.exit(1) + + print(f"\nAll {len(tests)} smoke tests passed.") diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..296bf46 --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "dstrack" +version = "0.1.0" +source = { editable = "." } From 818ca74682df1e10ba5521c1c50ebf52fc369d25 Mon Sep 17 00:00:00 2001 From: Leonardo Ayala Date: Wed, 17 Jun 2026 21:04:33 +0200 Subject: [PATCH 2/4] Fixes pre-commit gh action and adds PR template. --- .github/PULL_REQUEST_TEMPLATE.md | 16 ++++++++++++++++ .github/workflows/pre-commit.yaml | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..c14e834 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Summary + + + +## Changes + + + +- + +## Test plan + + + +- [ ] `uv run pytest` passes +- [ ] Manual test (describe what you tried): diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 2b11178..2bf13f5 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -9,6 +9,8 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.11' - uses: pre-commit/action@v3.0.1 From 7d2a2ff9f1614497d35fea21131fdbe849d99e1d Mon Sep 17 00:00:00 2001 From: Leonardo Ayala Date: Wed, 17 Jun 2026 21:21:02 +0200 Subject: [PATCH 3/4] removes pre-commit gh action and replaces it with pre-commit.ci configuration. --- .github/workflows/pre-commit.yaml | 16 ---------------- .pre-commit-config.yaml | 13 +++++++++++++ 2 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 .github/workflows/pre-commit.yaml diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml deleted file mode 100644 index 2bf13f5..0000000 --- a/.github/workflows/pre-commit.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - - uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc3c4e8..e843708 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,19 @@ exclude: | .*\.ipynb_checkpoints/.* )$ +# Configure pre-commit.ci to autofix pre-commit errors in PRs +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false + repos: # --------------------------------------------------------------------------- # Core hygiene: cheap, fast, catch-everything checks. From 0a8170749107ea030c2acf82d3fc6b8d94aefe9e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Jun 2026 19:24:18 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 83972fa..d99a8b0 100644 --- a/.gitignore +++ b/.gitignore @@ -195,9 +195,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder # .vscode/ # Temporary file for partial code execution