pyproject.toml

[build-system]
requires = ["poetry-core>=1.9.0", "poetry-dynamic-versioning>=1.4.0"]
build-backend = "poetry_dynamic_versioning.backend"


[project]
name = "deepnote-toolkit"
dynamic = ["version"]
description = "Essential Python toolkit for Deepnote environments"
authors = [
    {name = "Deepnote Engineering", email = "developers@deepnote.com"}
]
maintainers = [
    {name = "Deepnote Engineering", email = "developers@deepnote.com"}
]
readme = "README.md"
license = {text = "Apache-2.0"}
requires-python = ">=3.10.0,<3.14"
keywords = [
    "deepnote",
    "jupyter",
    "notebooks",
    "data-science",
    "analytics",
    "sql",
    "visualization",
    "streamlit"
]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: Scientific/Engineering",
    "Topic :: Scientific/Engineering :: Information Analysis",
    "Topic :: Scientific/Engineering :: Visualization",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Operating System :: OS Independent",
    "Environment :: Console",
    "Framework :: Jupyter"
]
dependencies = [
    # Core data libraries
    "pandas>=1.2.5,<2.2; python_version < '3.12'",
    "pandas>=2.1.0,<2.3; python_version >= '3.12'",
    "pandas>=2.2.3,<2.4; python_version >= '3.13'",
    "numpy>=1.23,<2; python_version == '3.10'",
    "numpy>=1.24,<2; python_version == '3.11'",
    "numpy>=1.26.0,<2; python_version == '3.12'",
    "numpy>=2.1.0,<3; python_version >= '3.13'",

    # SQL and database dependencies
    "duckdb>=1.1.0,<2.0.0; python_version < '3.12'",
    "duckdb>=1.1.0,<2.0.0; python_version >= '3.12'",
    "duckdb>=1.4.1,<2.0.0; python_version >= '3.13'",
    "duckdb-extensions>=1.1.0,<2.0.0", # bake in as dependency to not pull extensions from the internet on every notebook start
    "duckdb-extension-spatial>=1.1.0,<2.0.0",
    "duckdb-extension-excel>=1.1.0,<2.0.0",
    "google-cloud-bigquery-storage>=2.33.1,<3",

    "pyarrow>=10.0.0,<=17.0.0; python_version == '3.10' and sys_platform != 'darwin'",
    "pyarrow>=16.0.0,<18.0.0; python_version == '3.10' and sys_platform == 'darwin'",
    "pyarrow>=13.0.0,<=17.0.0; python_version == '3.11' and sys_platform != 'darwin'",
    "pyarrow>=16.0.0,<=18.0.0; python_version == '3.11' and sys_platform == 'darwin'",
    "pyarrow>=14.0.1; python_version >= '3.12'",
    "pyarrow>=18.0.0,<22.0; python_version >= '3.13'",

    "db-dtypes>=1.2.0,<1.4",

    # pymssql: macOS uses versions with ARM64 wheels (2.3.5+)
    "pymssql>=2.3.5,<2.4; sys_platform == 'darwin'",
    "pymssql>=2.2.9,<2.4; python_version < '3.12' and sys_platform != 'darwin'",
    "pymssql>=2.3.0,<2.4; python_version >= '3.12' and sys_platform != 'darwin'",
    "pymssql>=2.3.8,<2.4; python_version >= '3.13'",

    "sshtunnel==0.4.0",
    "psycopg2-binary>=2.9.4,<2.10; python_version < '3.13'",
    "psycopg2-binary>=2.9.10,<3; python_version>='3.13'",

    # SQL templating
    "jinja2>=3.1.6,<4",
    "sqlparse>=0.4.4,<0.6",
    "pymysql>=1.1.1,<1.2",
    "pymongo>=4.6.3,<4.10; python_version < '3.13'",
    "pymongo>=4.9.0,<5; python_version>='3.13'",
    "dnspython>=2.6.1,<2.7; python_version < '3.13'",
    "dnspython>=2.7.0,<3.0; python_version>='3.13'",
    "paramiko>=3.4.0,<3.6",
    "sqlalchemy<2; python_version <= '3.11'",
    "sqlalchemy>=2.0.31,<3; python_version >= '3.12'",
    "snowflake-sqlalchemy>=1.6.0,<1.9",
    "snowflake-connector-python>=3.13.1",
    "snowflake-connector-python>=3.16.0,<5; python_version>='3.13'",
    "sqlalchemy-spanner>=1.6.2,<2",
    "clickhouse-sqlalchemy>0,<0.4",
    "sqlalchemy-bigquery>1.6.1,<1.13; python_version < '3.13'",
    "sqlalchemy-bigquery>=1.15.0,<1.20; python_version>='3.13'",
    "zstd>1.5.5.1; python_version == '3.10'",
    "zstd>1.5.7.0; python_version >= '3.11'",
    "deepnote-sqlalchemy-redshift==0.8.17",
    "trino[sqlalchemy]>=0.327.0,<=0.330.0",
    "trino[sqlalchemy]>=0.329.0,<=0.336.0; python_version>='3.13'",
    "pyathena[sqlalchemy]==3.9.0",
    "pyathena[sqlalchemy]>=3.9.0,<4; python_version>='3.13'",
    "sqlalchemy-dremio>=3.0.3,<3.1",
    "dill>=0.3.8,<=0.4; python_version <= '3.12'",
    "dill>=0.3.9,<=0.4; python_version >= '3.13'",
    "databricks-sqlalchemy>=2.0.0,<3.0.0; python_version >= '3.12'",
    "databricks-sqlalchemy>=1.0.5,<2.0.0; python_version < '3.12'", # SQLAlchemy 1.x compatibility

    # AlloyDB
    "pg8000==1.31.5",
    "nbformat>=5.10.4,<6",
    "typing-extensions>=4.12.2,<5.0.0",
    "wrapt>=1.12.0",

    # Jupyter & IPython dependencies
    "ipykernel>=6.30.1,<7",
    "jupyter-core>=5.8.1,<6",

    # Visualization
    # NOTE: 1.7.0 is latest version of vl-convert which ships with Vega 5. In webapp we can't update to Vega 6 due to ESM-only issues
    # so we keep vl-convert version pinned as well
    "vl-convert-python==1.7.0",
    "deepnote-vegafusion>=2.1.0,<3",
    "matplotlib-inline>=0.1.7,<0.2.0; python_version <= '3.10'",  # 0.2.0 is not compatible with matplotlib 3.6.3 on Python <=3.10
    "matplotlib-inline>=0.2.1,<0.3.0; python_version >= '3.11'",

    # Security constraint updates for transitive dependencies
    "urllib3>=2.6.3,<3",
    "cryptography>=46.0.5,<47",
    "protobuf>=5.29.6,<6",
    "requests>=2.32.4,<3",
    "tornado>=6.5.5,<7",
    "filelock>=3.20.3,<4",
    "pyjwt>=2.12.0,<3",
    "pynacl>=1.6.2,<2",  # https://github.com/deepnote/deepnote-toolkit/security/dependabot/8
    "pyopenssl>=26.0.0,<27",  # https://github.com/deepnote/deepnote-toolkit/security/dependabot/21
    "pillow>=12.2.0,<13",  # https://github.com/deepnote/deepnote-toolkit/security/dependabot/34 https://github.com/deepnote/deepnote-toolkit/security/dependabot/17

    # Config dependencies - they need to be declared both in main and server extras, keep them in sync
    "pydantic>=1.10.0,<3",
    "pydantic>=2.8.0,<3; python_version>='3.13'",
    "pyyaml>=6,<7",
    "tomli>=2,<3; python_version < '3.11'",
    "toml>=0.10.0,<1.0.0",

    # Compatibility constraints for transitive dependencies
    "executing>=2.1.0,<3; python_version>='3.13'",  # Python 3.13 AST compatibility for IPython tracebacks
    "grpcio>=1.66.2; python_version >= '3.13'",
    "greenlet>=3.1.0; python_version >= '3.13'"
]

[project.urls]
Homepage = "https://deepnote.com"
Documentation = "https://docs.deepnote.com"
Repository = "https://github.com/deepnote/deepnote-toolkit"
Issues = "https://github.com/deepnote/deepnote-toolkit/issues"

[project.optional-dependencies]
server = [
    "jupyter-server==2.17.0",
    "jupyter-server-terminals==0.5.3",
    "jupyter-resource-usage>=1.1.0,<2.0.0",
    "streamlit>=1.40.0,<2.0.0",
    "deepnote-python-lsp-server==1.13.1rc2",

    # Config dependencies - they need to be declared both in main and server extras, keep them in sync
    "pydantic>=1.10.0,<3",
    "pydantic>=2.8.0,<3; python_version>='3.13'",
    "pyyaml>=6,<7",
    "tomli>=2,<3; python_version < '3.11'",
    "toml>=0.10.0,<1.0.0"
]

[project.scripts]
generate-constraints = "dockerfiles.builder.constraintsgen:main"
deepnote-toolkit = "deepnote_toolkit.cli.main:main"

[project.entry-points."sqlalchemy.dialects"]
# Backward-compatible dialect alias for Databricks.
# The integration generates URLs with 'databricks+connector://' scheme (for the old
# sqlalchemy-databricks package), but we now use databricks-sqlalchemy which only
# registers the 'databricks' dialect. This alias ensures the old URL format still works.
"databricks.connector" = "databricks.sqlalchemy:DatabricksDialect"

[dependency-groups]
dev = [
    "pylint>=3.3.0,<4.0.0",
    "flake8>=7.1.0,<8.0.0",
    "black>=26.3.1,<27.0.0",
    "isort>=5.13.2,<6.0.0",
    "pytest>=8.3.1,<9.0.0",
    "pytest-cov>=6.0.0,<7.0.0",
    "coverage[toml]>=7.10.0,<8.0.0",
    "mypy>=1.13.0,<2.0.0",
    "pre-commit>=3.6.0,<4.0.0",
    "responses>=0.25.7,<0.26.0",
    "parameterized>=0.9.0,<0.10.0",
    "geopandas>=1.1.2,<2; python_version < '3.12'",
    "geopandas>=1.1.2,<2; python_version >= '3.12'",
    # We do not distribute PySpark for all users because it's huge dependency (300mb) and not all
    # users need it. So in code we get pyspark from global module cache (since user already imported it at
    # that point). But we also have it as dev dependency to provide better autocompletion and for tests
    "pyspark>=3.5.5,<4.0.0; python_version < '3.13'",
    "pyspark>=4.0.0,<5.0.0; python_version >= '3.13'",
    "nox>=2025.11.12,<2026.0.0",
    "virtualenv>=20.36.1",
    "poetry-dynamic-versioning>=1.4.0,<2.0.0",
    "twine>=6.1.0,<7.0.0",
    "codespell>=2.3.0,<3.0.0",
    "pytest-subtests>=0.15.0,<0.16.0",
    "python-dotenv>=1.2.1,<2.0.0",
    "polars[rtcompat] (>=1.39.3,<2.0.0)"
]
license-check = [
    # Dependencies needed for license checking that aren't in main production dependencies
    "geopandas>=1.1.2,<2; python_version < '3.12'",
    "geopandas>=1.1.2,<2; python_version >= '3.12'",
    "pyspark>=3.5.5,<4.0.0; python_version < '3.13'",
    "pyspark>=4.0.0,<5.0.0; python_version >= '3.13'"
]

[tool.poetry]
requires-poetry = ">=2.2"
packages = [
    {include = "deepnote_toolkit"},
    {include = "deepnote_core"},
    {include = "installer"}
]
version = "0.1.0"

[tool.poetry.requires-plugins]
poetry-dynamic-versioning = {version = ">=1.0.0,<2.0.0", extras = ["plugin"]}

[tool.poetry.build]
generate-setup-file = false

[tool.poetry-dynamic-versioning]
enable = true
vcs = "git"
pattern = "default-unprefixed"
format-jinja = """
    {%- if distance == 0 -%}
        {{ serialize_pep440(base, stage, revision) }}
    {%- elif branch == 'main' -%}
        {{ serialize_pep440(base, post=distance) }}
    {%- else -%}
        {{ serialize_pep440(base, dev=distance, metadata=[commit[:7]]) }}
    {%- endif -%}
"""

[tool.poetry-dynamic-versioning.substitution]
files = ["deepnote_toolkit/_version.py"]
patterns = [
    "(^__version__\\s*(?::.*?)?=\\s*['\"])[^'\"]*(['\"])"
]


[tool.isort]
profile = "black"
line_length = 88

[tool.poe.tasks]
lint = "poetry run flake8 ."
format = "poetry run black ."
setup-hooks = "poetry run pre-commit install"

# I would go in future with more strict mypy config, but now we have lot of cleaning to do, link for docutmention https://mypy.readthedocs.io/en/stable/config_file.html
[tool.mypy]
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false
#disallow_untyped_defs = true -> after cleaning up code change to this
disallow_incomplete_defs = false
check_untyped_defs = true
disallow_untyped_decorators = false
warn_redundant_casts = true
warn_no_return = true
warn_unreachable = true
# Disable failing rules
disable_error_code = [
    "import-untyped",
    "var-annotated",
    "union-attr",
    "misc",
    "valid-type",
    "call-arg",
    "call-overload",
    "assignment",
    "arg-type",
    "no-any-return",
    "return-value",
    "operator",
    "unreachable",
    "attr-defined",
    "index"
]
exclude = [
    "tests/",
    ".nox/",
    ".pytest_cache/",
    "(^|[\\/])__pycache__[\\/]",
    "(^|[\\/])deepnote_core[\\/]resources[\\/]"
]

[tool.coverage.run]
branch = true
source = ["deepnote_toolkit", "installer", "deepnote_core"]
omit = [
    "*/tests/*",
    "*/__pycache__/*",
    "*/venv/*",
    "*/.nox/*"
]
parallel = true
concurrency = ["multiprocessing"]
data_file = ".coverage"
relative_files = true

[tool.coverage.paths]
deepnote_toolkit = [
    "deepnote_toolkit",
    "/deepnote-toolkit/deepnote_toolkit",
    "*/deepnote_toolkit",
    "*/site-packages/deepnote_toolkit"
]
installer = [
    "installer",
    "/deepnote-toolkit/installer",
    "*/installer",
    "*/site-packages/installer"
]
deepnote_core = [
    "deepnote_core",
    "/deepnote-toolkit/deepnote_core",
    "*/deepnote_core",
    "*/site-packages/deepnote_core"
]

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "if TYPE_CHECKING:",
    "raise NotImplementedError",
    "@(abc\\.)?abstractmethod",
    "pass",
    "except ImportError:",
    "class .*\\bProtocol\\):",
    "@overload"
]
exclude_also = [
    "if self.debug:",
    "if settings.DEBUG",
    "raise AssertionError",
    "return NotImplemented"
]
precision = 2
show_missing = true
skip_covered = true
fail_under = 55

[tool.coverage.html]
directory = "coverage/htmlcov"

[tool.coverage.xml]
output = "coverage/coverage.xml"

[tool.coverage.json]
output = "coverage/coverage.json"
pretty_print = true

[tool.pytest.ini_options]
addopts = """
    -v
    --strict-markers
    --tb=short
"""
testpaths = ["tests"]

[tool.codespell]
skip = "node_modules,package-lock.json,dist,build,coverage,.vscode,.git,*.min.js,*.min.css,*.map,__pycache__,.pytest_cache,.nox,.venv,*.egg-info,poetry.lock,.mypy_cache"
ignore-words-list = "autoindent,chrisartmann,frontends,hostnames,ipythondir,krishnan,ldflags,pkcs,pythonunbuffered,plotly,pydantic,re-use,sie,sigpipe,streamlit,suspendisse,synchronise,truecolor,adipiscing,amet,andrej,argstring,asyncpg,autoawait,autocall,autocompletions,automagic,autorestart,autorestore,autouse,behaviour,caplog,capsys,cdouble,collab,colnames,commodo,configurables,consectetur,creds,csingle,csvfile,ctes,datetimetz,dedup,deepnote,delenv,descr,devel,dictschema,dnenv,dntk,dockerfiles,docrepr,docstrings,drivername,elit,elpy,ename,endmacro,endset,evalue,excinfo,followedby,forcelist,freetds,fspath,getdbt,getsizeof,getuid,gitkey,highlevel,hotreload,htmlcov,importstring,importstrings,inclause,interactiveshell,interruptible,ioloop,iopub,ipkernel,ipynb,ipympl,isin,isort,issubdtype,jinjasql,jpserver,jsonable,jsonutil,junitxml,jupyter,jupyterlab,keepalive,kernelmanager,kernelspec,kernelspecs,kokso,largefilemanager,levelname,levelno,localstack,logappend,logstart,lokso,lowres,mathjax,metus,mimebundle,monorepo,monkeypatch,mountpoint,msecs,mydb,mypy,myschema,myval,nans,naturalearth,nbagg,nbextensions,nbserver,nbytes,ndarray,nginys,nisl,normalised,nox,nulla,numpy,oldpassword,olduser,packb,pathlib,performantly,pkey,pnpm,poethepoet,poetry,posargs,pprinter,prepresent,procs,proxified,purelib,pycache,pyformat,pyglet,pygmentize,pylab,pylsp,pymssql,pyproject,pytest,pyver,qmark,randn,readlinelike,redef,scatterchart,scatterplot,screwdrivercd,selectbox,serialise,sessionmanager,setwd,signum,sparksql,sphinxify,sqlsafe,sripathi,timesheet,traited,ufunc,unpackb,usecols,usermeta,userpod,vegafusion,vegalite,venv,xheaders,xmode,yearmonth,zombied"