From 636e88b79201c42ec5c403fe335b37e79321f31a Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 22 Aug 2026 00:01:11 +0000 Subject: [PATCH 1/2] Migrate Python type checking from basedpyright to ty Replace basedpyright with Astral's ty type checker for the Python bindings: - Swap the shared basedpyright dev dependency for ty and replace the [tool.basedpyright] sections with [tool.ty] configuration. First-party source roots let ty resolve vortex and vortex_cuda from the in-tree sources and stubs, and the vendored obstore stubs stay excluded. - Update CI workflows, check.sh, and the Python agent guidance to run 'ty check' instead of basedpyright. - Drop all now-dead 'pyright: ignore' comments and file-level pyright directives. Most suppressed rules have no ty counterpart; where ty still diagnoses the same stub or API mismatch, carry the suppression over as a targeted 'ty: ignore' comment. - Annotate _import_cudf_modules with ModuleType instead of object so attribute access on the dynamically imported modules type-checks without suppressions. Signed-off-by: "Robert" --- .github/workflows/ci.yml | 6 +- .github/workflows/cuda.yaml | 4 +- pyproject.toml | 17 ++- uv.lock | 125 +++++++++--------- .../python/vortex_cuda/__init__.py | 4 +- vortex-python-cuda/test/test_cuda.py | 1 - vortex-python-cuda/test/test_native_bridge.py | 5 +- vortex-python/AGENTS.md | 2 +- vortex-python/benchmark/conftest.py | 8 +- vortex-python/benchmark/test_aggregation.py | 8 +- vortex-python/benchmark/test_filter.py | 2 +- vortex-python/benchmark/test_scalar_at.py | 2 +- vortex-python/benchmark/test_scan.py | 2 +- vortex-python/benchmark/test_serialization.py | 4 +- vortex-python/check.sh | 2 +- vortex-python/clickbench.py | 44 +++--- vortex-python/pyproject.toml | 8 +- vortex-python/python/vortex/__init__.py | 16 +-- vortex-python/python/vortex/_lib/expr.pyi | 4 +- vortex-python/python/vortex/_lib/scalar.pyi | 2 +- .../python/vortex/_lib/store/__init__.pyi | 2 +- vortex-python/python/vortex/arrays.py | 26 ++-- .../python/vortex/arrow/expression.py | 20 +-- vortex-python/python/vortex/cli.py | 2 +- vortex-python/python/vortex/dataset.py | 25 ++-- vortex-python/python/vortex/datasets.py | 33 ++--- vortex-python/python/vortex/expr.py | 2 +- vortex-python/python/vortex/file.py | 20 +-- vortex-python/python/vortex/io.py | 2 +- vortex-python/python/vortex/polars_.py | 64 ++++----- vortex-python/python/vortex/ray/datasource.py | 10 +- vortex-python/python/vortex/registry.py | 2 +- vortex-python/python/vortex/scan.py | 6 +- vortex-python/python/vortex/serde.py | 2 +- vortex-python/python/vortex/store/__init__.py | 10 +- vortex-python/python/vortex/store/_aws.py | 17 +-- vortex-python/python/vortex/store/_azure.py | 15 ++- vortex-python/python/vortex/store/_gcs.py | 15 ++- vortex-python/python/vortex/store/_http.py | 8 +- vortex-python/python/vortex/store/_local.py | 4 +- vortex-python/python/vortex/store/_memory.py | 2 +- vortex-python/python/vortex/substrait.py | 8 +- vortex-python/python/vortex/type_aliases.py | 6 +- vortex-python/test/bench.py | 10 +- vortex-python/test/test_compress.py | 8 +- vortex-python/test/test_dataset.py | 20 +-- vortex-python/test/test_datasource.py | 18 +-- vortex-python/test/test_duckdb.py | 2 +- vortex-python/test/test_expr.py | 4 +- vortex-python/test/test_expression.py | 14 +- vortex-python/test/test_file.py | 26 ++-- vortex-python/test/test_fixed_size_list.py | 12 +- vortex-python/test/test_hf_datasets.py | 32 ++--- vortex-python/test/test_polars_.py | 10 +- vortex-python/test/test_pyarray.py | 38 +++--- vortex-python/test/test_scan.py | 10 +- 56 files changed, 384 insertions(+), 387 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5e642417c3..da4464e4700 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,13 +85,13 @@ jobs: run: uvx ruff format --check . - name: Python Lint - Ruff run: uvx ruff check . - # PyRight needs the project for type information, so use uv run - - name: Python Lint - PyRight + # ty needs the project for type information, so use uv run + - name: Python Lint - ty env: MATURIN_PEP517_ARGS: "--profile ci" run: | uv sync --all-packages - uv run basedpyright vortex-python + uv run ty check vortex-python python-test: name: "Python (test)" diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 87bb4887790..86d59d61bd5 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -134,9 +134,9 @@ jobs: env: MATURIN_PEP517_ARGS: "--profile ci" run: | - # --all-packages installs the shared dev tooling (basedpyright) from the + # --all-packages installs the shared dev tooling (ty) from the # root `dev` group; --extra cuda adds the vortex-data-cuda extension. - uv run --all-packages --extra cuda basedpyright vortex-python vortex-python-cuda + uv run --all-packages --extra cuda ty check vortex-python vortex-python-cuda uv run --all-packages --extra cuda pytest --benchmark-disable vortex-python/test/test_cuda.py vortex-python-cuda/test cuda-test-sanitizer: diff --git a/pyproject.toml b/pyproject.toml index 7fcdfbfa93a..54efc05efef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ packages = ["dummy"] # Required for workspace project # Shared dev tooling. Member-specific dev deps live in each member's pyproject.toml. # `uv sync --all-packages` picks up dev groups from all workspace members. dev = [ - "basedpyright>=1.31", + "ty>=0.0.73", "ipython>=8.26.0", "pip>=23.3.2", "pytest>=7.4.0", @@ -73,5 +73,16 @@ log_cli = true log_cli_level = "INFO" xfail_strict = true -[tool.basedpyright] -exclude = ["vortex-python/python/vortex/_lib/store/**.pyi"] +[tool.ty.environment] +# First-party source roots so `vortex` and `vortex_cuda` resolve without an +# editable install of the Rust extensions. +root = [ + "vortex-python/python", + "vortex-python", + "vortex-python-cuda/python", + "vortex-python-cuda", +] + +[tool.ty.src] +# Vendored obstore stubs are not checked. +exclude = ["vortex-python/python/vortex/_lib/store/**/*.pyi"] diff --git a/uv.lock b/uv.lock index f8c2aa8699e..2b78f8e889b 100644 --- a/uv.lock +++ b/uv.lock @@ -215,18 +215,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] -[[package]] -name = "basedpyright" -version = "1.31.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nodejs-wheel-binaries" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/ba/ed69e8df732a09c8ca469f592c8e08707fe29149735b834c276d94d4a3da/basedpyright-1.31.7.tar.gz", hash = "sha256:394f334c742a19bcc5905b2455c9f5858182866b7679a6f057a70b44b049bceb", size = 22710948, upload-time = "2025-10-11T05:12:48.3Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/90/ce01ad2d0afdc1b82b8b5aaba27e60d2e138e39d887e71c35c55d8f1bfcd/basedpyright-1.31.7-py3-none-any.whl", hash = "sha256:7c54beb7828c9ed0028630aaa6904f395c27e5a9f5a313aa9e91fc1d11170831", size = 11817571, upload-time = "2025-10-11T05:12:45.432Z" }, -] - [[package]] name = "bench-orchestrator" version = "0.1.0" @@ -1166,22 +1154,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/ac/686789b9145413f1a61878c407210e41bfdb097976864e0913078b24098c/myst_parser-5.0.0-py3-none-any.whl", hash = "sha256:ab31e516024918296e169139072b81592336f2fef55b8986aa31c9f04b5f7211", size = 84533, upload-time = "2026-01-15T09:08:16.788Z" }, ] -[[package]] -name = "nodejs-wheel-binaries" -version = "22.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/54/02f58c8119e2f1984e2572cc77a7b469dbaf4f8d171ad376e305749ef48e/nodejs_wheel_binaries-22.20.0.tar.gz", hash = "sha256:a62d47c9fd9c32191dff65bbe60261504f26992a0a19fe8b4d523256a84bd351", size = 8058, upload-time = "2025-09-26T09:48:00.906Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/24/6d/333e5458422f12318e3c3e6e7f194353aa68b0d633217c7e89833427ca01/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:455add5ac4f01c9c830ab6771dbfad0fdf373f9b040d3aabe8cca9b6c56654fb", size = 53246314, upload-time = "2025-09-26T09:47:32.536Z" }, - { url = "https://files.pythonhosted.org/packages/56/30/dcd6879d286a35b3c4c8f9e5e0e1bcf4f9e25fe35310fc77ecf97f915a23/nodejs_wheel_binaries-22.20.0-py2.py3-none-macosx_11_0_x86_64.whl", hash = "sha256:5d8c12f97eea7028b34a84446eb5ca81829d0c428dfb4e647e09ac617f4e21fa", size = 53644391, upload-time = "2025-09-26T09:47:36.093Z" }, - { url = "https://files.pythonhosted.org/packages/58/be/c7b2e7aa3bb281d380a1c531f84d0ccfe225832dfc3bed1ca171753b9630/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a2b0989194148f66e9295d8f11bc463bde02cbe276517f4d20a310fb84780ae", size = 60282516, upload-time = "2025-09-26T09:47:39.88Z" }, - { url = "https://files.pythonhosted.org/packages/3e/c5/8befacf4190e03babbae54cb0809fb1a76e1600ec3967ab8ee9f8fc85b65/nodejs_wheel_binaries-22.20.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5c500aa4dc046333ecb0a80f183e069e5c30ce637f1c1a37166b2c0b642dc21", size = 60347290, upload-time = "2025-09-26T09:47:43.712Z" }, - { url = "https://files.pythonhosted.org/packages/c0/bd/cfffd1e334277afa0714962c6ec432b5fe339340a6bca2e5fa8e678e7590/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3279eb1b99521f0d20a850bbfc0159a658e0e85b843b3cf31b090d7da9f10dfc", size = 62178798, upload-time = "2025-09-26T09:47:47.752Z" }, - { url = "https://files.pythonhosted.org/packages/08/14/10b83a9c02faac985b3e9f5e65d63a34fc0f46b48d8a2c3e4caa3e1e7318/nodejs_wheel_binaries-22.20.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d29705797b33bade62d79d8f106c2453c8a26442a9b2a5576610c0f7e7c351ed", size = 62772957, upload-time = "2025-09-26T09:47:51.266Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a9/c6a480259aa0d6b270aac2c6ba73a97444b9267adde983a5b7e34f17e45a/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_amd64.whl", hash = "sha256:4bd658962f24958503541963e5a6f2cc512a8cb301e48a69dc03c879f40a28ae", size = 40120431, upload-time = "2025-09-26T09:47:54.363Z" }, - { url = "https://files.pythonhosted.org/packages/42/b1/6a4eb2c6e9efa028074b0001b61008c9d202b6b46caee9e5d1b18c088216/nodejs_wheel_binaries-22.20.0-py2.py3-none-win_arm64.whl", hash = "sha256:1fccac931faa210d22b6962bcdbc99269d16221d831b9a118bbb80fe434a60b8", size = 38844133, upload-time = "2025-09-26T09:47:57.357Z" }, -] - [[package]] name = "numpy" version = "2.3.3" @@ -2084,23 +2056,23 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, + { name = "alabaster", marker = "python_full_version < '3.12'" }, + { name = "babel", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version < '3.12'" }, + { name = "imagesize", marker = "python_full_version < '3.12'" }, + { name = "jinja2", marker = "python_full_version < '3.12'" }, + { name = "packaging", marker = "python_full_version < '3.12'" }, + { name = "pygments", marker = "python_full_version < '3.12'" }, + { name = "requests", marker = "python_full_version < '3.12'" }, + { name = "roman-numerals", marker = "python_full_version < '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -2115,23 +2087,23 @@ resolution-markers = [ "python_full_version >= '3.12'", ] dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, + { name = "alabaster", marker = "python_full_version >= '3.12'" }, + { name = "babel", marker = "python_full_version >= '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version >= '3.12'" }, + { name = "imagesize", marker = "python_full_version >= '3.12'" }, + { name = "jinja2", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "pygments", marker = "python_full_version >= '3.12'" }, + { name = "requests", marker = "python_full_version >= '3.12'" }, + { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -2261,6 +2233,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, ] +[[package]] +name = "ty" +version = "0.0.73" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/90/c4e1bb4cead3b644c3e258a27f9b05c7dc5eb0ec96a4f5282194edae9e0d/ty-0.0.73.tar.gz", hash = "sha256:823d4ce0d237bfc7eb6bcee70842f2c0706113813a16951077840743712f4b74", size = 6712739, upload-time = "2026-08-19T03:12:43.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/0f/f5e1801e55cc631f2db193276675b30561b963a2403da832bffb5d100267/ty-0.0.73-py3-none-linux_armv6l.whl", hash = "sha256:90a946082bf9bc446b5e72973d9f4ff1222a240b2ca4c9e6eed61eb913e30810", size = 12715452, upload-time = "2026-08-19T03:12:06.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/32/515dd05074c213b433524ab97eb003b0132ae7e358e0d75633ba7a314ed8/ty-0.0.73-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b7d6b5c6a6db7ea95fbbc16af514ef44a27a29a2fe1dc798900790364d170209", size = 12301870, upload-time = "2026-08-19T03:12:08.924Z" }, + { url = "https://files.pythonhosted.org/packages/50/4d/085b4889f0d4bbe4af8b96242d4a1cb209fff95967cfa239ea141983719b/ty-0.0.73-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dd6f657f463e01372d8688f235be164750c8db722c97da27fa4903aa8d40b203", size = 12111741, upload-time = "2026-08-19T03:12:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/95/f6/d6ec277cadfecf03ad4c18551b67c4c6eb7807a0560d801db14be99d7a89/ty-0.0.73-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2de468e33fd44c9ff1c43473a7316f4289480f5cba8995a67b6d22aee39ca9", size = 12196124, upload-time = "2026-08-19T03:12:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/75/b7/ce78d8707563af9cae9bbd25328bfbc4931035085bd20089adf0c418f70e/ty-0.0.73-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2942fa0ef795a66034cdc8d75a72f453442f3b58ff2f69b4da05b7b954765b55", size = 12488557, upload-time = "2026-08-19T03:12:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e8/329b9851b23502758c5c98e8cc875ea2a1b4c9674b4ca3a86da56a5063d3/ty-0.0.73-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e0f1ef14f642e18ac4e7a616a2796dcf7a5d82e28cd17f9796494acc7c4aabb", size = 13215606, upload-time = "2026-08-19T03:12:17.225Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/67fedfd2cb77516ef0066b1642f487dba0eb3006493cf3475b15f5b8b228/ty-0.0.73-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16981e15fdceedb37d0aff76c5ac25914595dfee2675af95335550064251ad22", size = 13665497, upload-time = "2026-08-19T03:12:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b3/154f4dd48ec5eebc186ab4b822c6e62f982fc5ddfd262d6e3903c2acba44/ty-0.0.73-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644b2bec8a2e2e4957a942ae81d6cff5571c489bb5a8675e4d3886de537a694d", size = 13351231, upload-time = "2026-08-19T03:12:21.353Z" }, + { url = "https://files.pythonhosted.org/packages/35/5f/d462496903fbe453fb76363f8478be929c8e6ff21e6928c57dcd7e5fa21f/ty-0.0.73-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338d565be3186f50ff8e9d10483685549c2d23f0754485d5ede3b54f4319188a", size = 12782586, upload-time = "2026-08-19T03:12:23.667Z" }, + { url = "https://files.pythonhosted.org/packages/87/52/ec6d24b74abe3ec324204c1c71e6d0c6c76a17ffc15fd51d603b0a302abe/ty-0.0.73-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:11c7b6d839309d2c102cb3a4c03d817176bbfab5b2fccc95a75ec5c9597421c9", size = 13247134, upload-time = "2026-08-19T03:12:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/26/20/cc74650fec56a54786c6d7c89e09576fcad3092be34cf21715d39a406a9b/ty-0.0.73-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:488572db7ff97fb50ea36a76250f2d617c9727d143da6c7bf0623276eb0fc507", size = 12309344, upload-time = "2026-08-19T03:12:28.122Z" }, + { url = "https://files.pythonhosted.org/packages/89/bd/4b0a9087f4315d7fbadf77a3ce44c816cc9ffabed1ced06cc5be81fbc414/ty-0.0.73-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1b958ebceefbbf594e59eb8d3d55bbd033ce634026fcba3e4bc3179e78e45bb7", size = 12502319, upload-time = "2026-08-19T03:12:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/11/80/0a925074911fe111912ea29d9eed309bcc183f43d2fb3eef07db056a0beb/ty-0.0.73-py3-none-musllinux_1_2_i686.whl", hash = "sha256:91a32993b3c34e42c3f323ad6c0399cb596bd1c27e9b7f20db7cd64c1067b68e", size = 12753688, upload-time = "2026-08-19T03:12:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/6b/aeccaf89efbc2e112bd415340a22e2669ec998aa397242503e747b712ca4/ty-0.0.73-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bab8a19fbf51f479bddb2a12c5fabfe52f918a5590362321ed5d89b44eb62c15", size = 13069050, upload-time = "2026-08-19T03:12:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3e/eae485fd86c1585943fd4e1746b0757b2da01e2c43136ebe8c686fe1c7f1/ty-0.0.73-py3-none-win32.whl", hash = "sha256:03347a612f0fa020b19bfd8dbd521db6ecc75d377a3e4d4f6e6c2e62871da4cc", size = 12053187, upload-time = "2026-08-19T03:12:37.565Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/9b8b983786e3ce34924e372e8b76b92b508273ab65c589fc7e88cc03ee17/ty-0.0.73-py3-none-win_amd64.whl", hash = "sha256:cedd05122ded0b5dcc55431a370e974b747f99c41c290a3d2ab8c1867f197519", size = 12693838, upload-time = "2026-08-19T03:12:39.483Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/25333bbfea6a5dc064371d2002d3d4807db90b84d5448f9106b2712b0fbc/ty-0.0.73-py3-none-win_arm64.whl", hash = "sha256:e47068f8369dea5d641a26a2ad0a947a320b02ff87099b07e95de0323245a4dc", size = 12443573, upload-time = "2026-08-19T03:12:41.449Z" }, +] + [[package]] name = "typer" version = "0.21.1" @@ -2428,12 +2425,12 @@ dependencies = [ [package.dev-dependencies] dev = [ - { name = "basedpyright" }, { name = "filelock" }, { name = "ipython" }, { name = "pip" }, { name = "pytest" }, { name = "ruff" }, + { name = "ty" }, { name = "urllib3" }, ] @@ -2446,12 +2443,12 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ - { name = "basedpyright", specifier = ">=1.31" }, { name = "filelock", specifier = ">=3.20.3" }, { name = "ipython", specifier = ">=8.26.0" }, { name = "pip", specifier = ">=23.3.2" }, { name = "pytest", specifier = ">=7.4.0" }, { name = "ruff", specifier = ">=0.7.1" }, + { name = "ty", specifier = ">=0.0.73" }, { name = "urllib3", specifier = ">=2.6.3" }, ] diff --git a/vortex-python-cuda/python/vortex_cuda/__init__.py b/vortex-python-cuda/python/vortex_cuda/__init__.py index 985516e6204..ffd68709881 100644 --- a/vortex-python-cuda/python/vortex_cuda/__init__.py +++ b/vortex-python-cuda/python/vortex_cuda/__init__.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# pyright: reportAttributeAccessIssue=false, reportMissingModuleSource=false, reportPrivateUsage=false, reportUnknownMemberType=false, reportUnknownVariableType=false import importlib +from types import ModuleType from . import _lib @@ -39,7 +39,7 @@ def _install_vortex_array_methods() -> None: setattr(vortex.Array, "__arrow_c_device_array__", _Array___arrow_c_device_array__) -def _import_cudf_modules() -> tuple[object, object]: +def _import_cudf_modules() -> tuple[ModuleType, ModuleType]: try: cudf = importlib.import_module("cudf") pylibcudf = importlib.import_module("pylibcudf") diff --git a/vortex-python-cuda/test/test_cuda.py b/vortex-python-cuda/test/test_cuda.py index 56678ab1f41..01d5b0b62fb 100644 --- a/vortex-python-cuda/test/test_cuda.py +++ b/vortex-python-cuda/test/test_cuda.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# pyright: reportAny=false, reportExplicitAny=false import gc import sys diff --git a/vortex-python-cuda/test/test_native_bridge.py b/vortex-python-cuda/test/test_native_bridge.py index 960467ea25c..45e6b3da348 100644 --- a/vortex-python-cuda/test/test_native_bridge.py +++ b/vortex-python-cuda/test/test_native_bridge.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# pyright: reportPrivateUsage=false import gc @@ -66,7 +65,7 @@ def test_metadata_bridge_struct_with_children(): arrow_table = pa.table({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) struct_array = vortex.Array.from_arrow( - pa.StructArray.from_arrays( # pyright: ignore[reportUnknownMemberType] + pa.StructArray.from_arrays( [arrow_table.column("a").combine_chunks(), arrow_table.column("b").combine_chunks()], names=["a", "b"], ) @@ -128,7 +127,7 @@ def test_arrow_device_export_struct_array(): arrow_table = pa.table({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]}) struct_array = vortex.Array.from_arrow( - pa.StructArray.from_arrays( # pyright: ignore[reportUnknownMemberType] + pa.StructArray.from_arrays( [arrow_table.column("a").combine_chunks(), arrow_table.column("b").combine_chunks()], names=["a", "b"], ) diff --git a/vortex-python/AGENTS.md b/vortex-python/AGENTS.md index 8d2f59bacdc..536be8352a2 100644 --- a/vortex-python/AGENTS.md +++ b/vortex-python/AGENTS.md @@ -25,7 +25,7 @@ If Python docstrings, `docs/api/python/`, or Sphinx configuration change, also f ## Linting and Formatting ```bash -uv run basedpyright vortex-python +uv run ty check vortex-python uv run ruff format --check uv run ruff check ``` diff --git a/vortex-python/benchmark/conftest.py b/vortex-python/benchmark/conftest.py index 300983cf087..dc44161a333 100644 --- a/vortex-python/benchmark/conftest.py +++ b/vortex-python/benchmark/conftest.py @@ -27,15 +27,15 @@ def vxf( length = 100_000 columns: dict[str, list[int] | list[float] | list[str]] = {} - assert "x" in request.param # pyright: ignore[reportAny] + assert "x" in request.param columns["x"] = list(range(length)) - if "y" in request.param: # pyright: ignore[reportAny] + if "y" in request.param: columns["y"] = [hashlib.md5(x.to_bytes(length=4), usedforsecurity=False).hexdigest() for x in range(length)] - if "z" in request.param: # pyright: ignore[reportAny] + if "z" in request.param: columns["z"] = [math.sqrt(x) for x in range(length)] - a = vx.array(pa.table(columns)) # pyright: ignore[reportCallIssue, reportUnknownArgumentType, reportArgumentType] + a = vx.array(pa.table(columns)) # ty: ignore[no-matching-overload] vx.io.write(a, str(fname)) return vx.open(str(fname)) diff --git a/vortex-python/benchmark/test_aggregation.py b/vortex-python/benchmark/test_aggregation.py index bde346b7b6d..f6d3daf43ea 100644 --- a/vortex-python/benchmark/test_aggregation.py +++ b/vortex-python/benchmark/test_aggregation.py @@ -7,7 +7,7 @@ import pyarrow as pa import pytest from pyarrow.types import is_floating, is_integer -from pytest_benchmark.fixture import BenchmarkFixture # pyright: ignore[reportMissingTypeStubs] +from pytest_benchmark.fixture import BenchmarkFixture import vortex as vx @@ -19,9 +19,7 @@ def _has_mean(t: pa.DataType) -> bool: @pytest.mark.benchmark(group="aggregation", disable_gc=True) def test_arrow_table_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile): aggregations: list[tuple[str, Literal["mean"]]] = [ - (field.name, "mean") - for field in vxf.dtype.to_arrow_schema() # pyright: ignore[reportUnknownVariableType] - if _has_mean(field.type) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + (field.name, "mean") for field in vxf.dtype.to_arrow_schema() if _has_mean(field.type) ] benchmark(lambda: pa.concat_tables(x.to_arrow_table() for x in vxf.scan()).group_by([]).aggregate(aggregations)) @@ -44,7 +42,7 @@ def test_duckdb_aggregation(benchmark: BenchmarkFixture, vxf: vx.VortexFile): ds = vxf.to_dataset() _ = conn.register("ds", ds) aggregations = ",".join( - [f"avg(ds.{field.name}) as {field.name}" for field in vxf.dtype.to_arrow_schema() if _has_mean(field.type)] # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType, reportUnknownArgumentType] + [f"avg(ds.{field.name}) as {field.name}" for field in vxf.dtype.to_arrow_schema() if _has_mean(field.type)] ) print(aggregations) query = f"select {aggregations} from ds" diff --git a/vortex-python/benchmark/test_filter.py b/vortex-python/benchmark/test_filter.py index 53616c29468..b23cb0d67b9 100644 --- a/vortex-python/benchmark/test_filter.py +++ b/vortex-python/benchmark/test_filter.py @@ -5,7 +5,7 @@ import polars as pl import pyarrow as pa import pytest -from pytest_benchmark.fixture import BenchmarkFixture # pyright: ignore[reportMissingTypeStubs] +from pytest_benchmark.fixture import BenchmarkFixture import vortex as vx from vortex.expr import column diff --git a/vortex-python/benchmark/test_scalar_at.py b/vortex-python/benchmark/test_scalar_at.py index 185db6bafc2..4bc3da9746a 100644 --- a/vortex-python/benchmark/test_scalar_at.py +++ b/vortex-python/benchmark/test_scalar_at.py @@ -4,7 +4,7 @@ import duckdb import pyarrow as pa import pytest -from pytest_benchmark.fixture import BenchmarkFixture # pyright: ignore[reportMissingTypeStubs] +from pytest_benchmark.fixture import BenchmarkFixture import vortex as vx diff --git a/vortex-python/benchmark/test_scan.py b/vortex-python/benchmark/test_scan.py index 6d8052d01ab..f460b5e7066 100644 --- a/vortex-python/benchmark/test_scan.py +++ b/vortex-python/benchmark/test_scan.py @@ -4,7 +4,7 @@ import duckdb import pyarrow as pa import pytest -from pytest_benchmark.fixture import BenchmarkFixture # pyright: ignore[reportMissingTypeStubs] +from pytest_benchmark.fixture import BenchmarkFixture import vortex as vx diff --git a/vortex-python/benchmark/test_serialization.py b/vortex-python/benchmark/test_serialization.py index 3107ca814c4..34887b68c44 100644 --- a/vortex-python/benchmark/test_serialization.py +++ b/vortex-python/benchmark/test_serialization.py @@ -5,12 +5,12 @@ from typing import cast import pytest -from pytest_benchmark.fixture import BenchmarkFixture # pyright: ignore[reportMissingTypeStubs] +from pytest_benchmark.fixture import BenchmarkFixture import vortex as vx -@pytest.mark.parametrize("protocol", [4, 5], ids=lambda p: f"p{p}") # pyright: ignore[reportAny] +@pytest.mark.parametrize("protocol", [4, 5], ids=lambda p: f"p{p}") @pytest.mark.parametrize("operation", ["dumps", "loads", "roundtrip"]) @pytest.mark.benchmark(disable_gc=True) def test_pickle( diff --git a/vortex-python/check.sh b/vortex-python/check.sh index ec30c3e7213..b6e0a5446f7 100755 --- a/vortex-python/check.sh +++ b/vortex-python/check.sh @@ -17,7 +17,7 @@ pushd $ROOT/vortex-python maturin develop ruff format --check ruff check -basedpyright +ty check popd pushd $ROOT/docs diff --git a/vortex-python/clickbench.py b/vortex-python/clickbench.py index 57fd1e04db9..723bd2acabd 100755 --- a/vortex-python/clickbench.py +++ b/vortex-python/clickbench.py @@ -16,7 +16,7 @@ import vortex as vx # 0: No., 1: SQL, 2: Polars -queries: list[tuple[str, str, Callable[[pl.LazyFrame], Any]]] = [ # pyright: ignore[reportExplicitAny] +queries: list[tuple[str, str, Callable[[pl.LazyFrame], Any]]] = [ ("Q0", "SELECT COUNT(*) FROM hits;", lambda x: x.select(pl.len()).collect().height), ( "Q1", @@ -35,17 +35,17 @@ ( "Q3", "SELECT AVG(UserID) FROM hits;", - lambda x: x.select(pl.col("UserID").mean()).collect().item(), # pyright: ignore[reportAny] + lambda x: x.select(pl.col("UserID").mean()).collect().item(), ), ( "Q4", "SELECT COUNT(DISTINCT UserID) FROM hits;", - lambda x: x.select(pl.col("UserID").n_unique()).collect().item(), # pyright: ignore[reportAny] + lambda x: x.select(pl.col("UserID").n_unique()).collect().item(), ), ( "Q5", "SELECT COUNT(DISTINCT SearchPhrase) FROM hits;", - lambda x: x.select(pl.col("SearchPhrase").n_unique()).collect().item(), # pyright: ignore[reportAny] + lambda x: x.select(pl.col("SearchPhrase").n_unique()).collect().item(), ), ( "Q6", @@ -187,7 +187,7 @@ ( "Q20", "SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';", - lambda x: x.filter(pl.col("URL").str.contains("google")).select(pl.len()).collect().item(), # pyright: ignore[reportAny] + lambda x: x.filter(pl.col("URL").str.contains("google")).select(pl.len()).collect().item(), ), ( "Q21", @@ -511,7 +511,7 @@ ] -def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> dict[str, Any] | None: # pyright: ignore[reportExplicitAny] +def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> dict[str, Any] | None: queries_times: list[list[float | None]] = [] for q in queries: # if q[0] == "Q19": @@ -523,7 +523,7 @@ def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> for _ in range(3): start = timeit.default_timer() try: - result = q[2](lf) # pyright: ignore[reportAny] + result = q[2](lf) except Exception as e: print("Failed", e) result = None @@ -564,14 +564,14 @@ def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> PARSER = argparse.ArgumentParser() -PARSER.add_argument( # pyright: ignore[reportUnusedCallResult] +PARSER.add_argument( "--path", type=str, default="hits.parquet", help="Path to the parquet file", ) -PARSER.add_argument( # pyright: ignore[reportUnusedCallResult] +PARSER.add_argument( "--formats", nargs="+", choices=("vortex", "parquet"), @@ -579,7 +579,7 @@ def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> help="Formats to run", ) -PARSER.add_argument( # pyright: ignore[reportUnusedCallResult] +PARSER.add_argument( "-q", "--queries", nargs="+", @@ -588,13 +588,13 @@ def run_timings(lf: pl.LazyFrame, name: str, src: str, load_time: int | None) -> help="Queries to run", ) -PARSER.add_argument("-i", "--iterations", type=int, default=3, help="Number of iterations to run") # pyright: ignore[reportUnusedCallResult] +PARSER.add_argument("-i", "--iterations", type=int, default=3, help="Number of iterations to run") def main(args: argparse.Namespace): - assert isinstance(args.path, str) # pyright: ignore[reportAny] - assert isinstance(args.queries, list) # pyright: ignore[reportAny] - assert isinstance(args.formats, list | None) # pyright: ignore[reportAny] + assert isinstance(args.path, str) + assert isinstance(args.queries, list) + assert isinstance(args.formats, list | None) if not os.path.exists(args.path): raise ValueError(f"File {args.path} does not exist") @@ -602,22 +602,22 @@ def main(args: argparse.Namespace): results: defaultdict[str, list[float]] = defaultdict(list) def run_queries(format: str, lf: pl.LazyFrame): - for q in args.queries: # pyright: ignore[reportAny] + for q in args.queries: assert isinstance(q, int) timings = [] - for _ in range(args.iterations): # pyright: ignore[reportAny] + for _ in range(args.iterations): start = timeit.default_timer() try: - _result: Callable[[pl.LazyFrame], Any] = queries[q][2](lf) # pyright: ignore[reportExplicitAny, reportAny] + _result: Callable[[pl.LazyFrame], Any] = queries[q][2](lf) except Exception as e: print(f"Failed Q{q}", e) - timings.append(timeit.default_timer() - start) # pyright: ignore[reportUnknownMemberType] - average = sum(timings) / len(timings) # pyright: ignore[reportUnknownArgumentType] + timings.append(timeit.default_timer() - start) + average = sum(timings) / len(timings) results[format].append(average) print(f"{format} Q{q}", average) - if args.formats is None or "vortex" in args.formats: # pyright: ignore[reportUnknownMemberType] + if args.formats is None or "vortex" in args.formats: vx_base, _ = os.path.splitext(args.path) vx_path = f"{vx_base}.vortex" @@ -628,7 +628,7 @@ def run_queries(format: str, lf: pl.LazyFrame): def _iter(): for i in range(pf.num_row_groups): - arr = pf.read_row_group(i).to_struct_array() # pyright: ignore[reportUnknownMemberType] + arr = pf.read_row_group(i).to_struct_array() arr = vx.Array.from_arrow(arr) yield arr @@ -638,7 +638,7 @@ def _iter(): lf = vx.open(vx_path).to_polars() run_queries("vortex", lf) - if args.formats is None or "parquet" in args.formats: # pyright: ignore[reportUnknownMemberType] + if args.formats is None or "parquet" in args.formats: lf = pl.scan_parquet(args.path) run_queries("parquet", lf) diff --git a/vortex-python/pyproject.toml b/vortex-python/pyproject.toml index 2e40ad9bc0d..9aa5e28899e 100644 --- a/vortex-python/pyproject.toml +++ b/vortex-python/pyproject.toml @@ -91,5 +91,9 @@ dev = [ "protobuf>=6.33.5", ] -[tool.basedpyright] -exclude = ["python/vortex/_lib/store/**.pyi"] +[tool.ty.environment] +root = ["python", "."] + +[tool.ty.src] +# Vendored obstore stubs are not checked. +exclude = ["python/vortex/_lib/store/**/*.pyi"] diff --git a/vortex-python/python/vortex/__init__.py b/vortex-python/python/vortex/__init__.py index acb90629832..dca52a77578 100644 --- a/vortex-python/python/vortex/__init__.py +++ b/vortex-python/python/vortex/__init__.py @@ -6,7 +6,7 @@ import importlib.util from . import _lib, arrays, dataset, expr, file, io, ray, registry, scan -from ._lib.arrays import ( # pyright: ignore[reportMissingModuleSource] +from ._lib.arrays import ( AlpArray, AlpRdArray, BoolArray, @@ -33,8 +33,8 @@ VarBinViewArray, ZigZagArray, ) -from ._lib.compress import compress # pyright: ignore[reportMissingModuleSource] -from ._lib.dtype import ( # pyright: ignore[reportMissingModuleSource] +from ._lib.compress import compress +from ._lib.dtype import ( BinaryDType, BoolDType, DecimalDType, @@ -62,12 +62,12 @@ uint, utf8, ) -from ._lib.iter import ArrayIterator # pyright: ignore[reportMissingModuleSource] -from ._lib.runtime import ( # pyright: ignore[reportMissingModuleSource] +from ._lib.iter import ArrayIterator +from ._lib.runtime import ( set_worker_threads, worker_threads, ) -from ._lib.scalar import ( # pyright: ignore[reportMissingModuleSource] +from ._lib.scalar import ( BinaryScalar, BoolScalar, # TODO(connor): Is this missing a `DecimalScalar`? @@ -80,11 +80,11 @@ Utf8Scalar, scalar, ) -from ._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] +from ._lib.serde import ArrayContext, SerializedArray from .arrays import ( Array, PyArray, - _unpickle_array, # pyright: ignore[reportPrivateUsage] + _unpickle_array, array, ) from .file import VortexFile, open diff --git a/vortex-python/python/vortex/_lib/expr.pyi b/vortex-python/python/vortex/_lib/expr.pyi index 47172e3b87f..6cd2b58bfd5 100644 --- a/vortex-python/python/vortex/_lib/expr.pyi +++ b/vortex-python/python/vortex/_lib/expr.pyi @@ -18,9 +18,9 @@ VariantPath: TypeAlias = str | int | Sequence[str | int] @final class Expr: @override - def __eq__(self, other: IntoExpr) -> Expr: ... # pyright: ignore[reportIncompatibleMethodOverride] + def __eq__(self, other: IntoExpr) -> Expr: ... # ty: ignore[invalid-method-override] @override - def __ne__(self, other: IntoExpr) -> Expr: ... # pyright: ignore[reportIncompatibleMethodOverride] + def __ne__(self, other: IntoExpr) -> Expr: ... # ty: ignore[invalid-method-override] def __lt__(self, other: IntoExpr) -> Expr: ... def __le__(self, other: IntoExpr) -> Expr: ... def __gt__(self, other: IntoExpr) -> Expr: ... diff --git a/vortex-python/python/vortex/_lib/scalar.pyi b/vortex-python/python/vortex/_lib/scalar.pyi index a35b64c8566..6f982ee69bb 100644 --- a/vortex-python/python/vortex/_lib/scalar.pyi +++ b/vortex-python/python/vortex/_lib/scalar.pyi @@ -8,7 +8,7 @@ from .dtype import DType ScalarPyType: TypeAlias = None | int | float | str | Decimal | bytes | list[ScalarPyType] | dict[str, ScalarPyType] -def scalar(value: Any, *, dtype: DType | None = None) -> Scalar: ... # pyright: ignore[reportAny, reportExplicitAny] +def scalar(value: Any, *, dtype: DType | None = None) -> Scalar: ... class Scalar: @property diff --git a/vortex-python/python/vortex/_lib/store/__init__.pyi b/vortex-python/python/vortex/_lib/store/__init__.pyi index 6e88ca35508..d7621b15d3b 100644 --- a/vortex-python/python/vortex/_lib/store/__init__.pyi +++ b/vortex-python/python/vortex/_lib/store/__init__.pyi @@ -178,7 +178,7 @@ class LocalStore: ``` """ - def __eq__(self, value: object, /) -> bool: ... # pyright: ignore[reportImplicitOverride] + def __eq__(self, value: object, /) -> bool: ... def __getnewargs_ex__(self) -> tuple[tuple[()], dict[str, object]]: ... @property def prefix(self) -> Path | None: diff --git a/vortex-python/python/vortex/arrays.py b/vortex-python/python/vortex/arrays.py index aa0bd411bd4..03677c0f56b 100644 --- a/vortex-python/python/vortex/arrays.py +++ b/vortex-python/python/vortex/arrays.py @@ -9,9 +9,9 @@ import pyarrow from typing_extensions import override -import vortex._lib.arrays as _arrays # pyright: ignore[reportMissingModuleSource] -from vortex._lib.dtype import DType # pyright: ignore[reportMissingModuleSource] -from vortex._lib.serde import ( # pyright: ignore[reportMissingModuleSource] +import vortex._lib.arrays as _arrays +from vortex._lib.dtype import DType +from vortex._lib.serde import ( ArrayContext, SerializedArray, decode_ipc_array_buffers, @@ -24,7 +24,7 @@ else: # HACK: monkey-patch a fixed implementation of the pd.ArrowDtype.type property accessor. # See https://github.com/pandas-dev/pandas/issues/60068 for more details - _old_ArrowDtype_type: Callable[[pandas.ArrowDtype], type] = pandas.ArrowDtype.type.fget # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + _old_ArrowDtype_type: Callable[[pandas.ArrowDtype], type] = pandas.ArrowDtype.type.fget # ty: ignore[unresolved-attribute] @property def __ArrowDtype_type_patched(self: pandas.ArrowDtype): @@ -48,7 +48,7 @@ def empty_arrow_table(schema: pyarrow.Schema) -> pyarrow.Table: def empty_array(f: pyarrow.Field[pyarrow.DataType]) -> pyarrow.Array[pyarrow.Scalar[pyarrow.DataType]]: return pyarrow.array([], type=f.type) - return pyarrow.Table.from_arrays([empty_array(field) for field in schema], schema=schema) # pyright: ignore[reportUnknownVariableType, reportUnknownArgumentType] + return pyarrow.Table.from_arrays([empty_array(field) for field in schema], schema=schema) def arrow_table_from_struct_array( @@ -134,7 +134,7 @@ def _Array_to_pandas(self: _arrays.Array) -> pandas.DataFrame: """ import pandas - return self.to_arrow_table().to_pandas(types_mapper=pandas.ArrowDtype) # pyright: ignore[reportUnknownMemberType] + return self.to_arrow_table().to_pandas(types_mapper=pandas.ArrowDtype) Array.to_pandas = _Array_to_pandas @@ -186,7 +186,7 @@ def _Array_to_polars_dataframe( """ import polars - return polars.from_arrow(self.to_arrow_table()) # pyright: ignore[reportUnknownMemberType] + return polars.from_arrow(self.to_arrow_table()) setattr(Array, "to_polars_dataframe", _Array_to_polars_dataframe) @@ -254,7 +254,7 @@ def _Array_to_polars_series(self: _arrays.Array): # -> 'polars.Series': # brea """ import polars - return polars.from_arrow(self.to_arrow_array()) # pyright: ignore[reportUnknownMemberType] + return polars.from_arrow(self.to_arrow_array()) setattr(Array, "to_polars_series", _Array_to_polars_series) @@ -291,7 +291,7 @@ def _Array_to_numpy(self: _arrays.Array, *, zero_copy_only: bool = True) -> nump Array.to_numpy = _Array_to_numpy -def _Array_to_pylist(self: _arrays.Array) -> list[Any]: # pyright: ignore[reportExplicitAny] +def _Array_to_pylist(self: _arrays.Array) -> list[Any]: """Deeply copy an Array into a Python list. Returns @@ -317,10 +317,10 @@ def _Array_to_pylist(self: _arrays.Array) -> list[Any]: # pyright: ignore[repor def array( - obj: pyarrow.Array[pyarrow.Scalar[Any]] # pyright: ignore[reportExplicitAny] - | pyarrow.ChunkedArray[pyarrow.Scalar[Any]] # pyright: ignore[reportExplicitAny] + obj: pyarrow.Array[pyarrow.Scalar[Any]] + | pyarrow.ChunkedArray[pyarrow.Scalar[Any]] | pyarrow.Table - | list[Any] # pyright: ignore[reportExplicitAny] + | list[Any] | pandas.DataFrame | range, ) -> Array: @@ -472,7 +472,7 @@ def decode(cls, parts: SerializedArray, ctx: ArrayContext, dtype: DType, len: in """ -def _unpickle_array( # pyright: ignore[reportUnusedFunction] +def _unpickle_array( array_buffers: Sequence[bytes | memoryview], dtype_buffers: Sequence[bytes | memoryview], ) -> Array: diff --git a/vortex-python/python/vortex/arrow/expression.py b/vortex-python/python/vortex/arrow/expression.py index 92c7e87638f..69c88f77a25 100644 --- a/vortex-python/python/vortex/arrow/expression.py +++ b/vortex-python/python/vortex/arrow/expression.py @@ -5,11 +5,11 @@ import pyarrow as pa import pyarrow.compute as pc -from substrait.proto import ( # pyright: ignore[reportMissingTypeStubs] +from substrait.proto import ( ExtendedExpression, ) -from vortex._lib.expr import Expr # pyright: ignore[reportMissingModuleSource] +from vortex._lib.expr import Expr from ..substrait import extended_expression @@ -34,20 +34,20 @@ def _schema_for_substrait(schema: pa.Schema) -> pa.Schema: # If/When PyArrow to_substrait supports view types, revert. # Workaround for: https://github.com/vortex-data/vortex/issues/5759 fields = [] - for field in schema: # pyright: ignore[reportUnknownVariableType] - if field.type == pa.string_view(): # pyright: ignore[reportUnknownMemberType] - fields.append(field.with_type(pa.string())) # pyright: ignore[reportUnknownMemberType] - elif field.type == pa.binary_view(): # pyright: ignore[reportUnknownMemberType] - fields.append(field.with_type(pa.binary())) # pyright: ignore[reportUnknownMemberType] + for field in schema: + if field.type == pa.string_view(): + fields.append(field.with_type(pa.string())) + elif field.type == pa.binary_view(): + fields.append(field.with_type(pa.binary())) else: - fields.append(field) # pyright: ignore[reportUnknownMemberType] - return pa.schema(fields) # pyright: ignore[reportUnknownArgumentType] + fields.append(field) + return pa.schema(fields) def arrow_to_vortex(arrow_expression: pc.Expression, schema: pa.Schema) -> Expr: compat_schema = _schema_for_substrait(schema) substrait_object = ExtendedExpression() - substrait_object.ParseFromString(bytes(arrow_expression.to_substrait(compat_schema))) # pyright: ignore[reportUnusedCallResult] + substrait_object.ParseFromString(bytes(arrow_expression.to_substrait(compat_schema))) expressions = extended_expression(substrait_object) diff --git a/vortex-python/python/vortex/cli.py b/vortex-python/python/vortex/cli.py index 93012737cdd..444cb33d280 100644 --- a/vortex-python/python/vortex/cli.py +++ b/vortex-python/python/vortex/cli.py @@ -5,6 +5,6 @@ def main() -> None: - from vortex._lib.cli import launch # pyright: ignore[reportMissingModuleSource] + from vortex._lib.cli import launch launch(sys.argv) diff --git a/vortex-python/python/vortex/dataset.py b/vortex-python/python/vortex/dataset.py index a50118bc9a3..9b16b53422a 100644 --- a/vortex-python/python/vortex/dataset.py +++ b/vortex-python/python/vortex/dataset.py @@ -13,10 +13,10 @@ import pyarrow.dataset from typing_extensions import override -from ._lib import dataset as _dataset # pyright: ignore[reportMissingModuleSource] -from ._lib import file as _file # pyright: ignore[reportMissingModuleSource] -from ._lib.runtime import set_worker_threads as _set_worker_threads # pyright: ignore[reportMissingModuleSource] -from ._lib.runtime import worker_threads as _worker_threads # pyright: ignore[reportMissingModuleSource] +from ._lib import dataset as _dataset +from ._lib import file as _file +from ._lib.runtime import set_worker_threads as _set_worker_threads +from ._lib.runtime import worker_threads as _worker_threads from .arrays import array from .arrow.expression import ensure_vortex_expression from .expr import Expr, and_ @@ -281,12 +281,14 @@ def scanner( ) @override - def sort_by(self, sorting: str | list[tuple[str, str]], **kwargs) -> pyarrow.dataset.InMemoryDataset: # pyright: ignore[reportUnknownParameterType, reportMissingParameterType, reportIncompatibleMethodOverride] + def sort_by( # ty: ignore[invalid-method-override] + self, sorting: str | list[tuple[str, str]], **kwargs + ) -> pyarrow.dataset.InMemoryDataset: """Not implemented.""" raise NotImplementedError("sort_by") @override - def take( # pyright: ignore[reportIncompatibleMethodOverride] + def take( # ty: ignore[invalid-method-override] self, indices: pyarrow.Array[ pyarrow.Int8Scalar @@ -654,7 +656,7 @@ def to_table( ) @override - def take( # pyright: ignore[reportIncompatibleMethodOverride] + def take( # ty: ignore[invalid-method-override] self, indices: pyarrow.Array[ pyarrow.Int8Scalar @@ -729,7 +731,7 @@ def head( # regarding the ignore: https://github.com/zen-xu/pyarrow-stubs/pull/258 @override - def count_rows( # pyright: ignore[reportIncompatibleMethodOverride] + def count_rows( # ty: ignore[invalid-method-override] self, filter: pyarrow.dataset.Expression | Expr | None = None, batch_size: int | None = None, @@ -826,10 +828,7 @@ def dataset_schema(self) -> pyarrow.Schema: @override def projected_schema(self) -> pyarrow.Schema: if self._columns: - fields: list[pa.Field[pa.DataType]] = [ - self._dataset.schema.field(c) # pyright: ignore[reportUnknownMemberType] - for c in self._columns - ] + fields: list[pa.Field[pa.DataType]] = [self._dataset.schema.field(c) for c in self._columns] return pyarrow.schema(fields) return self._dataset.schema @@ -876,7 +875,7 @@ def head(self, num_rows: int) -> pyarrow.Table: ) @override - def scan_batches(self) -> Iterator[pyarrow.dataset.TaggedRecordBatch]: # pyright: ignore[reportIncompatibleMethodOverride] + def scan_batches(self) -> Iterator[pyarrow.dataset.TaggedRecordBatch]: # ty: ignore[invalid-method-override] """Not implemented.""" raise NotImplementedError("scan batches") diff --git a/vortex-python/python/vortex/datasets.py b/vortex-python/python/vortex/datasets.py index a09da332def..45098039ad5 100644 --- a/vortex-python/python/vortex/datasets.py +++ b/vortex-python/python/vortex/datasets.py @@ -1,14 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# pyright: reportAny=false -# pyright: reportMissingTypeStubs=false -# pyright: reportMissingTypeArgument=false -# pyright: reportPrivateUsage=false -# pyright: reportUnannotatedClassAttribute=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownMemberType=false -# pyright: reportUnknownParameterType=false -# pyright: reportUnknownVariableType=false from __future__ import annotations @@ -202,19 +193,19 @@ def __init__( super().__init__( ex_iterable=ex_iterable, info=info, - split=hf_datasets.Split(split), + split=hf_datasets.Split(split), # ty: ignore[invalid-argument-type] formatting=formatting, - shuffling=shuffling, # pyright: ignore[reportCallIssue] - distributed=distributed, + shuffling=shuffling, # ty: ignore[unknown-argument] + distributed=distributed, # ty: ignore[invalid-argument-type] token_per_repo_id=token_per_repo_id, ) else: super().__init__( ex_iterable=ex_iterable, info=info, - split=hf_datasets.Split(split), + split=hf_datasets.Split(split), # ty: ignore[invalid-argument-type] formatting=formatting, - distributed=distributed, # pyright: ignore[reportArgumentType] + distributed=distributed, # ty: ignore[invalid-argument-type] token_per_repo_id=token_per_repo_id, ) @@ -300,7 +291,7 @@ def _with_pushdown( limit: int | None | object = ..., formatting: FormattingConfig | None | object = ..., ) -> VortexIterableDataset: - new_columns = self._vortex_columns if columns is ... else _normalize_columns(columns) # pyright: ignore[reportArgumentType] + new_columns = self._vortex_columns if columns is ... else _normalize_columns(columns) # ty: ignore[invalid-argument-type] # Derive the new features from the current ones instead of re-reading the file schema: # select_columns/remove_columns only ever narrow the current selection. if new_columns == self._vortex_columns: @@ -312,12 +303,12 @@ def _with_pushdown( return VortexIterableDataset( self._vortex_files, columns=new_columns, - filter=self._vortex_filter if filter is ... else filter, # pyright: ignore[reportArgumentType] - limit=self._vortex_limit if limit is ... else limit, # pyright: ignore[reportArgumentType] + filter=self._vortex_filter if filter is ... else filter, # ty: ignore[invalid-argument-type] + limit=self._vortex_limit if limit is ... else limit, # ty: ignore[invalid-argument-type] batch_size=self._vortex_batch_size, store=self._vortex_store, split=str(self._split), - formatting=self._formatting if formatting is ... else formatting, # pyright: ignore[reportArgumentType] + formatting=self._formatting if formatting is ... else formatting, # ty: ignore[invalid-argument-type] shuffling=copy.deepcopy(getattr(self, "_shuffling", None)), distributed=copy.deepcopy(self._distributed), token_per_repo_id=self._token_per_repo_id, @@ -468,7 +459,7 @@ def _resumed_batches(self): state["file_row_idx"] = 0 @override - def shuffle_data_sources(self, generator) -> _VortexExamplesIterable: # pyright: ignore[reportMissingParameterType] + def shuffle_data_sources(self, generator) -> _VortexExamplesIterable: if self.limit is not None: # A pushed-down limit (take()) must keep selecting the same rows, so the file order # cannot be permuted — mirroring TakeExamplesIterable. datasets>=5 signals this with @@ -545,13 +536,13 @@ def _materialize_dataset( return hf_datasets.Dataset.from_generator( _generate_rows, features=features, - cache_dir=cache_dir, # pyright: ignore[reportArgumentType] + cache_dir=cache_dir, # ty: ignore[invalid-argument-type] keep_in_memory=keep_in_memory, gen_kwargs=gen_kwargs, # A global row limit cannot be divided across processes without overshooting, so force # single-process generation whenever a limit is set. num_proc=None if limit is not None else num_proc, - split=hf_datasets.Split(split), + split=hf_datasets.Split(split), # ty: ignore[invalid-argument-type] ) diff --git a/vortex-python/python/vortex/expr.py b/vortex-python/python/vortex/expr.py index a2ccefcdece..d02ed5ebab6 100644 --- a/vortex-python/python/vortex/expr.py +++ b/vortex-python/python/vortex/expr.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors -from ._lib.expr import ( # pyright: ignore[reportMissingModuleSource] +from ._lib.expr import ( Expr, add, and_, diff --git a/vortex-python/python/vortex/file.py b/vortex-python/python/vortex/file.py index 4db19c1c05b..c22e73fb64b 100644 --- a/vortex-python/python/vortex/file.py +++ b/vortex-python/python/vortex/file.py @@ -8,11 +8,11 @@ import pyarrow as pa -from ._lib import file as _file # pyright: ignore[reportMissingModuleSource] -from ._lib.arrays import Array # pyright: ignore[reportMissingModuleSource] -from ._lib.dtype import DType # pyright: ignore[reportMissingModuleSource] -from ._lib.expr import Expr # pyright: ignore[reportMissingModuleSource] -from ._lib.iter import ArrayIterator # pyright: ignore[reportMissingModuleSource] +from ._lib import file as _file +from ._lib.arrays import Array +from ._lib.dtype import DType +from ._lib.expr import Expr +from ._lib.iter import ArrayIterator from .dataset import VortexDataset from .scan import RepeatedScan from .store import ( @@ -253,17 +253,17 @@ def _io_source( reader = self.to_arrow(projection=with_columns, expr=vx_predicate, limit=n_rows) for batch in reader: - batch = pl.DataFrame._from_arrow(batch, rechunk=False) # pyright: ignore[reportPrivateUsage] + batch = pl.DataFrame._from_arrow(batch, rechunk=False) # TODO(ngates): set sortedness on DataFrame based on stats? yield batch # Make sure we always yield at least one empty DataFrame - yield pl.DataFrame._from_arrow( # pyright: ignore[reportPrivateUsage] - data=pa.RecordBatch.from_arrays( # pyright: ignore[reportUnknownMemberType] - [pa.array([], type=field.type) for field in reader.schema], # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType] + yield pl.DataFrame._from_arrow( + data=pa.RecordBatch.from_arrays( + [pa.array([], type=field.type) for field in reader.schema], schema=reader.schema, ), ) # https://github.com/pola-rs/polars/pull/24125 - return register_io_source(_io_source, schema=schema) # pyright: ignore[reportArgumentType] + return register_io_source(_io_source, schema=schema) # ty: ignore[invalid-argument-type] diff --git a/vortex-python/python/vortex/io.py b/vortex-python/python/vortex/io.py index 87275078777..9d17ecdd8c4 100644 --- a/vortex-python/python/vortex/io.py +++ b/vortex-python/python/vortex/io.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -from vortex._lib.io import VortexWriteOptions, read_url, write # pyright: ignore[reportMissingModuleSource] +from vortex._lib.io import VortexWriteOptions, read_url, write __all__ = ["read_url", "write", "VortexWriteOptions"] diff --git a/vortex-python/python/vortex/polars_.py b/vortex-python/python/vortex/polars_.py index d1cd708af39..3fafe710127 100644 --- a/vortex-python/python/vortex/polars_.py +++ b/vortex-python/python/vortex/polars_.py @@ -10,14 +10,14 @@ import vortex.expr as ve -from ._lib import dtype as _dtype # pyright: ignore[reportMissingModuleSource] +from ._lib import dtype as _dtype def polars_to_vortex(expr: pl.Expr) -> ve.Expr: """Convert a Polars expression to a Vortex expression.""" - data = json.loads(expr.meta.serialize(format="json")) # pyright: ignore[reportAny] + data = json.loads(expr.meta.serialize(format="json")) assert isinstance(data, dict) - return _polars_to_vortex(data) # pyright: ignore[reportUnknownArgumentType] + return _polars_to_vortex(data) _OPS = { @@ -34,7 +34,7 @@ def polars_to_vortex(expr: pl.Expr) -> ve.Expr: } -_LITERAL_TYPES: dict[str, Callable[[Any | None], _dtype.DType]] = { # pyright: ignore[reportExplicitAny] +_LITERAL_TYPES: dict[str, Callable[[Any | None], _dtype.DType]] = { "Boolean": lambda v: _dtype.bool_(nullable=v is None), "Int": lambda v: _dtype.int_(64, nullable=v is None), "Int8": lambda v: _dtype.int_(8, nullable=v is None), @@ -53,48 +53,48 @@ def polars_to_vortex(expr: pl.Expr) -> ve.Expr: } -def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr: # pyright: ignore[reportExplicitAny] +def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr: """Convert a Polars expression to a Vortex expression.""" if "BinaryExpr" in expr: - expr = expr["BinaryExpr"] # pyright: ignore[reportAny] - lhs = _polars_to_vortex(expr["left"]) # pyright: ignore[reportAny] - rhs = _polars_to_vortex(expr["right"]) # pyright: ignore[reportAny] - op = expr["op"] # pyright: ignore[reportAny] + expr = expr["BinaryExpr"] + lhs = _polars_to_vortex(expr["left"]) + rhs = _polars_to_vortex(expr["right"]) + op = expr["op"] if op not in _OPS: raise NotImplementedError(f"Unsupported Polars binary operator: {op}") - return _OPS[op](lhs, rhs) # pyright: ignore[reportAny] + return _OPS[op](lhs, rhs) if "Column" in expr: - return ve.column(expr["Column"]) # pyright: ignore[reportAny] + return ve.column(expr["Column"]) # See https://github.com/pola-rs/polars/pull/21849 if "Scalar" in expr: - scalar = expr["Scalar"] # pyright: ignore[reportAny] + scalar = expr["Scalar"] if "Null" in scalar: value = None dtype = "Null" elif "String" in scalar: - value = scalar["String"] # pyright: ignore[reportAny] + value = scalar["String"] dtype = "String" elif "Int" in scalar: - value = scalar["Int"] # pyright: ignore[reportAny] + value = scalar["Int"] dtype = "Int64" elif "Float" in scalar: - value = scalar["Float"] # pyright: ignore[reportAny] + value = scalar["Float"] dtype = "Float64" elif "Float32" in scalar: - value = scalar["Float32"] # pyright: ignore[reportAny] + value = scalar["Float32"] dtype = "Float32" elif "Float64" in scalar: - value = scalar["Float64"] # pyright: ignore[reportAny] + value = scalar["Float64"] dtype = "Float64" elif "Int32" in scalar: - value = scalar["Int32"] # pyright: ignore[reportAny] + value = scalar["Int32"] dtype = "Int32" elif "Int64" in scalar: - value = scalar["Int64"] # pyright: ignore[reportAny] + value = scalar["Int64"] dtype = "Int64" else: raise ValueError(f"Cannot convert to Vortex: unsupported Polars scalar value type {scalar}") @@ -102,7 +102,7 @@ def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr: # pyright: ignore[repor return ve.literal(_LITERAL_TYPES[dtype](value), value) if "Literal" in expr: - expr = expr["Literal"] # pyright: ignore[reportAny] + expr = expr["Literal"] literal_type = next(iter(expr.keys()), None) @@ -115,7 +115,7 @@ def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr: # pyright: ignore[repor # Special-case date-times if literal_type == "DateTime": - (value, unit, tz) = expr[literal_type] # pyright: ignore[reportAny, reportAny] + (value, unit, tz) = expr[literal_type] if unit == "Nanoseconds": unit = "ns" elif unit == "Microseconds": @@ -127,37 +127,37 @@ def _polars_to_vortex(expr: dict[str, Any]) -> ve.Expr: # pyright: ignore[repor else: raise NotImplementedError(f"Unsupported Polars date time unit: {unit}") - dtype = _dtype.timestamp(unit, tz=tz, nullable=value) # pyright: ignore[reportAny] - return ve.literal(dtype, value) # pyright: ignore[reportAny] + dtype = _dtype.timestamp(unit, tz=tz, nullable=value) + return ve.literal(dtype, value) # Unwrap 'Dyn' scalars, whose type hasn't been established yet. # (post https://github.com/pola-rs/polars/pull/21849) if literal_type == "Dyn": - expr = expr["Dyn"] # pyright: ignore[reportAny] + expr = expr["Dyn"] literal_type = next(iter(expr.keys()), None) if literal_type not in _LITERAL_TYPES: raise NotImplementedError(f"Unsupported Polars literal type: {literal_type}") - value = expr[literal_type] # pyright: ignore[reportAny] - return ve.literal(_LITERAL_TYPES[literal_type](value), value) # pyright: ignore[reportAny] + value = expr[literal_type] + return ve.literal(_LITERAL_TYPES[literal_type](value), value) if "Function" in expr: - expr = expr["Function"] # pyright: ignore[reportAny] - _inputs = [_polars_to_vortex(e) for e in expr["input"]] # pyright: ignore[reportAny] + expr = expr["Function"] + _inputs = [_polars_to_vortex(e) for e in expr["input"]] - fn = expr["function"] # pyright: ignore[reportAny] + fn = expr["function"] if "Boolean" in fn: - fn = fn["Boolean"] # pyright: ignore[reportAny] + fn = fn["Boolean"] if "IsIn" in fn: - fn = fn["IsIn"] # pyright: ignore[reportAny] + fn = fn["IsIn"] if fn["nulls_equal"]: raise ValueError(f"Unsupported nulls_equal argument in fn {expr}") # Vortex doesn't support is-in, so we need to construct a series of ORs? if "StringExpr" in fn: - fn = fn["StringExpr"] # pyright: ignore[reportAny] + fn = fn["StringExpr"] if "Contains" in fn: raise ValueError("Unsupported Polars StringExpr.Contains") diff --git a/vortex-python/python/vortex/ray/datasource.py b/vortex-python/python/vortex/ray/datasource.py index 966c33b3eaf..95b57e24c81 100644 --- a/vortex-python/python/vortex/ray/datasource.py +++ b/vortex-python/python/vortex/ray/datasource.py @@ -11,7 +11,7 @@ from ray.data.context import DataContext from ray.data.datasource import BaseFileMetadataProvider, DefaultFileMetadataProvider from ray.data.datasource.path_util import ( - _resolve_paths_and_filesystem, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] + _resolve_paths_and_filesystem, ) from typing_extensions import override @@ -54,17 +54,17 @@ def __init__( columns: IntoProjection = None, filter: pc.Expression | VortexExpr | None = None, batch_size: int | None = None, - meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(), # pyright: ignore[reportCallInDefaultInitializer] + meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(), ): super().__init__() self._columns = columns self._filter = filter - urls, fs = _resolve_paths_and_filesystem(url, None) # pyright: ignore[reportUnknownVariableType] + urls, fs = _resolve_paths_and_filesystem(url, None) paths_and_sizes = list( meta_provider.expand_paths( urls, - fs, # pyright: ignore[reportUnknownArgumentType] + fs, None, ignore_missing_paths=False, ) @@ -147,6 +147,6 @@ def read() -> Iterable[pandas.DataFrame]: # # yield pa.Table.from_batches([rb]) # - yield rb.to_pandas() # pyright: ignore[reportUnknownMemberType] + yield rb.to_pandas() return ReadTask(read, metadata, schema) diff --git a/vortex-python/python/vortex/registry.py b/vortex-python/python/vortex/registry.py index 44fdb7fed32..05ef91b543f 100644 --- a/vortex-python/python/vortex/registry.py +++ b/vortex-python/python/vortex/registry.py @@ -3,6 +3,6 @@ """Register Python extension types with the process-wide Vortex registry.""" -from vortex._lib.registry import register # pyright: ignore[reportMissingModuleSource] +from vortex._lib.registry import register __all__ = ["register"] diff --git a/vortex-python/python/vortex/scan.py b/vortex-python/python/vortex/scan.py index 970747c1717..71db2267479 100644 --- a/vortex-python/python/vortex/scan.py +++ b/vortex-python/python/vortex/scan.py @@ -5,9 +5,9 @@ from typing import final -from ._lib import scan as _scan # pyright: ignore[reportMissingModuleSource] -from ._lib.iter import ArrayIterator # pyright: ignore[reportMissingModuleSource] -from ._lib.scalar import Scalar # pyright: ignore[reportMissingModuleSource] +from ._lib import scan as _scan +from ._lib.iter import ArrayIterator +from ._lib.scalar import Scalar @final diff --git a/vortex-python/python/vortex/serde.py b/vortex-python/python/vortex/serde.py index 2220c254a53..c711d824e5d 100644 --- a/vortex-python/python/vortex/serde.py +++ b/vortex-python/python/vortex/serde.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -from vortex._lib.serde import ( # pyright: ignore[reportMissingModuleSource] +from vortex._lib.serde import ( ArrayContext, SerializedArray, decode_ipc_array_buffers, diff --git a/vortex-python/python/vortex/store/__init__.py b/vortex-python/python/vortex/store/__init__.py index f5f3338e3f5..9f3bee567c6 100644 --- a/vortex-python/python/vortex/store/__init__.py +++ b/vortex-python/python/vortex/store/__init__.py @@ -4,7 +4,7 @@ from collections.abc import Callable from typing import TypeAlias, Unpack, overload -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store from ._aws import S3Config, S3Credential, S3CredentialProvider, S3Store from ._azure import ( AzureAccessKey, @@ -127,13 +127,13 @@ def from_url( # type: ignore[misc] # docstring in pyi file kwargs: per-store configuration passed down to store-specific builders. """ - return _store.from_url( # pyright: ignore[reportCallIssue, reportUnknownVariableType] + return _store.from_url( # ty: ignore[no-matching-overload] url, - config=config, # pyright: ignore[reportArgumentType] + config=config, client_options=client_options, retry_config=retry_config, - credential_provider=credential_provider, # pyright: ignore[reportArgumentType] - **kwargs, # pyright: ignore[reportArgumentType] + credential_provider=credential_provider, + **kwargs, ) diff --git a/vortex-python/python/vortex/store/_aws.py b/vortex-python/python/vortex/store/_aws.py index 3f3079e9902..2c806db032b 100644 --- a/vortex-python/python/vortex/store/_aws.py +++ b/vortex-python/python/vortex/store/_aws.py @@ -7,7 +7,7 @@ from typing_extensions import override -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store from ._client import ClientConfig from ._retry import RetryConfig @@ -404,7 +404,7 @@ def __call__(self) -> S3Credential: """ - def __call__(self) -> S3Credential | Coroutine[Any, Any, S3Credential]: # pyright: ignore[reportExplicitAny] + def __call__(self) -> S3Credential | Coroutine[Any, Any, S3Credential]: """Return an ``S3Credential``.""" ... @@ -438,7 +438,7 @@ def __new__( client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, credential_provider: S3CredentialProvider | None = None, - **kwargs: Unpack[S3Config], # pyright: ignore[reportGeneralTypeIssues] + **kwargs: Unpack[S3Config], # ty: ignore[invalid-type-form] ) -> Self: """Create a new S3Store. @@ -459,7 +459,7 @@ def __new__( S3Store """ - return super().__new__( # pyright: ignore[reportUnknownVariableType] + return super().__new__( cls, bucket, prefix=prefix, @@ -467,7 +467,8 @@ def __new__( client_options=client_options, retry_config=retry_config, credential_provider=credential_provider, - **kwargs, # pyright: ignore[reportCallIssue] bucket appears in both S3Config and explicitly above + # bucket appears in both S3Config and explicitly above + **kwargs, # ty: ignore[parameter-already-assigned] ) @override @@ -514,7 +515,7 @@ def from_url( S3Store """ - return super(cls).from_url( + return super(cls).from_url( # ty: ignore[unresolved-attribute] url, config=config, client_options=client_options, @@ -528,8 +529,8 @@ def __eq__(self, value: object) -> bool: return super().__eq__(value) @override - def __getnewargs_ex__(self): # pyright: ignore[reportUnknownParameterType] - return super().__getnewargs_ex__() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + def __getnewargs_ex__(self): + return super().__getnewargs_ex__() @property @override diff --git a/vortex-python/python/vortex/store/_azure.py b/vortex-python/python/vortex/store/_azure.py index 1b3ec1b4e52..9fd76a9a182 100644 --- a/vortex-python/python/vortex/store/_azure.py +++ b/vortex-python/python/vortex/store/_azure.py @@ -7,7 +7,7 @@ from typing_extensions import override -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store from ._client import ClientConfig from ._retry import RetryConfig @@ -252,7 +252,7 @@ class AzureCredentialProvider(Protocol): """ - def __call__(self) -> AzureCredential | Coroutine[Any, Any, AzureCredential]: # pyright: ignore[reportExplicitAny] + def __call__(self) -> AzureCredential | Coroutine[Any, Any, AzureCredential]: """Return an ``AzureCredential``.""" ... @@ -273,7 +273,7 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, credential_provider: AzureCredentialProvider | None = None, - **kwargs: Unpack[AzureConfig], # pyright: ignore[reportGeneralTypeIssues] + **kwargs: Unpack[AzureConfig], # ty: ignore[invalid-type-form] ) -> Self: """Construct a new AzureStore. @@ -294,7 +294,7 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed AzureStore """ - return super().__new__( # pyright: ignore[reportUnknownVariableType] + return super().__new__( cls, container_name, prefix=prefix, @@ -302,7 +302,8 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed client_options=client_options, retry_config=retry_config, credential_provider=credential_provider, - **kwargs, # pyright: ignore[reportCallIssue] + # container_name appears in both AzureConfig and explicitly above + **kwargs, # ty: ignore[parameter-already-assigned] ) @override @@ -373,8 +374,8 @@ def __eq__(self, value: object) -> bool: return super().__eq__(value) @override - def __getnewargs_ex__(self): # pyright: ignore[reportUnknownParameterType] - return super().__getnewargs_ex__() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + def __getnewargs_ex__(self): + return super().__getnewargs_ex__() @property @override diff --git a/vortex-python/python/vortex/store/_gcs.py b/vortex-python/python/vortex/store/_gcs.py index e1512434093..0e53f70716a 100644 --- a/vortex-python/python/vortex/store/_gcs.py +++ b/vortex-python/python/vortex/store/_gcs.py @@ -7,7 +7,7 @@ from typing_extensions import override -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store from ._client import ClientConfig from ._retry import RetryConfig @@ -92,7 +92,7 @@ class GCSCredentialProvider(Protocol): """ - def __call__(self) -> GCSCredential | Coroutine[Any, Any, GCSCredential]: # pyright: ignore[reportExplicitAny] + def __call__(self) -> GCSCredential | Coroutine[Any, Any, GCSCredential]: """Return a ``GCSCredential``.""" ... @@ -117,7 +117,7 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed client_options: ClientConfig | None = None, retry_config: RetryConfig | None = None, credential_provider: GCSCredentialProvider | None = None, - **kwargs: Unpack[GCSConfig], # pyright: ignore[reportGeneralTypeIssues] + **kwargs: Unpack[GCSConfig], # ty: ignore[invalid-type-form] ) -> Self: """Construct a new GCSStore. @@ -138,7 +138,7 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed GCSStore """ - return super().__new__( # pyright: ignore[reportUnknownVariableType] + return super().__new__( cls, bucket, prefix=prefix, @@ -146,7 +146,8 @@ def __new__( # type: ignore[misc] # Overlap between argument names and ** Typed client_options=client_options, retry_config=retry_config, credential_provider=credential_provider, - **kwargs, # pyright: ignore[reportCallIssue] + # bucket appears in both GCSConfig and explicitly above + **kwargs, # ty: ignore[parameter-already-assigned] ) @override @@ -205,8 +206,8 @@ def __eq__(self, value: object) -> bool: return super().__eq__(value) @override - def __getnewargs_ex__(self): # pyright: ignore[reportUnknownParameterType] - return super().__getnewargs_ex__() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + def __getnewargs_ex__(self): + return super().__getnewargs_ex__() @property @override diff --git a/vortex-python/python/vortex/store/_http.py b/vortex-python/python/vortex/store/_http.py index 57d46bb5ebb..4817ea429cf 100644 --- a/vortex-python/python/vortex/store/_http.py +++ b/vortex-python/python/vortex/store/_http.py @@ -5,7 +5,7 @@ from typing_extensions import override -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store from ._client import ClientConfig from ._retry import RetryConfig @@ -53,15 +53,15 @@ def from_url( This is an alias of the :class:`~vortex.store.HTTPStore` constructor. """ - return super(cls).from_url(url, client_options=client_options, retry_config=retry_config) + return super(cls).from_url(url, client_options=client_options, retry_config=retry_config) # ty: ignore[unresolved-attribute] @override def __eq__(self, value: object) -> bool: return super().__eq__(value) @override - def __getnewargs_ex__(self): # pyright: ignore[reportUnknownParameterType] - return super().__getnewargs_ex__() # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + def __getnewargs_ex__(self): + return super().__getnewargs_ex__() @property @override diff --git a/vortex-python/python/vortex/store/_local.py b/vortex-python/python/vortex/store/_local.py index 891c8e968b4..496dc0be137 100644 --- a/vortex-python/python/vortex/store/_local.py +++ b/vortex-python/python/vortex/store/_local.py @@ -6,7 +6,7 @@ from typing_extensions import override -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store class LocalStore(_store.LocalStore): @@ -67,7 +67,7 @@ def from_url( store = LocalStore.from_url(url) """ - return super(cls).from_url(url, automatic_cleanup=automatic_cleanup, mkdir=mkdir) + return super(cls).from_url(url, automatic_cleanup=automatic_cleanup, mkdir=mkdir) # ty: ignore[unresolved-attribute] @override def __eq__(self, value: object, /) -> bool: diff --git a/vortex-python/python/vortex/store/_memory.py b/vortex-python/python/vortex/store/_memory.py index 763424732eb..7dbd48b3d6f 100644 --- a/vortex-python/python/vortex/store/_memory.py +++ b/vortex-python/python/vortex/store/_memory.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MIT # SPDX-FileCopyrightText: Copyright (c) 2024 Development Seed -from .._lib import store as _store # pyright: ignore[reportMissingModuleSource] +from .._lib import store as _store class MemoryStore(_store.MemoryStore): diff --git a/vortex-python/python/vortex/substrait.py b/vortex-python/python/vortex/substrait.py index 1bab4b87d2b..bfad6ba1c11 100644 --- a/vortex-python/python/vortex/substrait.py +++ b/vortex-python/python/vortex/substrait.py @@ -12,7 +12,7 @@ from substrait.extended_expression_pb2 import ExpressionReference, ExtendedExpression from substrait.extensions.extensions_pb2 import ( SimpleExtensionDeclaration, - SimpleExtensionURI, # pyright: ignore[reportDeprecated] + SimpleExtensionURI, # ty: ignore[deprecated] ) from substrait.type_pb2 import NamedStruct, Type else: @@ -29,8 +29,8 @@ from substrait.gen.proto.extensions.extensions_pb2 import SimpleExtensionDeclaration, SimpleExtensionURI from substrait.gen.proto.type_pb2 import NamedStruct, Type -from ._lib import dtype as _dtype # pyright: ignore[reportMissingModuleSource] -from ._lib import expr as _expr # pyright: ignore[reportMissingModuleSource] +from ._lib import dtype as _dtype +from ._lib import expr as _expr ExtensionTypes = dict[int, str] TypeVariations = dict[int, str] @@ -465,7 +465,7 @@ def function_argument( def extension_function( substrait_object: SimpleExtensionDeclaration.ExtensionFunction, - extension_uris: RepeatedCompositeFieldContainer[SimpleExtensionURI], # pyright: ignore[reportDeprecated] + extension_uris: RepeatedCompositeFieldContainer[SimpleExtensionURI], # ty: ignore[deprecated] ) -> Callable[..., _expr.Expr]: # https://github.com/substrait-io/substrait/blob/main/proto/substrait/extensions/extensions.proto#L57 match extension_uris[substrait_object.extension_uri_reference].uri: diff --git a/vortex-python/python/vortex/type_aliases.py b/vortex-python/python/vortex/type_aliases.py index 550f471088c..ed18b01e0a3 100644 --- a/vortex-python/python/vortex/type_aliases.py +++ b/vortex-python/python/vortex/type_aliases.py @@ -4,9 +4,9 @@ import pyarrow as pa -from ._lib.arrays import Array # pyright: ignore[reportMissingModuleSource] -from ._lib.expr import Expr # pyright: ignore[reportMissingModuleSource] -from ._lib.iter import ArrayIterator # pyright: ignore[reportMissingModuleSource] +from ._lib.arrays import Array +from ._lib.expr import Expr +from ._lib.iter import ArrayIterator # TypeAliases do not support __doc__. IntoProjection: TypeAlias = Expr | list[str] | None diff --git a/vortex-python/test/bench.py b/vortex-python/test/bench.py index fe7b4d8ab79..a00ad0b4dc1 100644 --- a/vortex-python/test/bench.py +++ b/vortex-python/test/bench.py @@ -18,11 +18,11 @@ @pytest.fixture(params=[10, 100]) -def vortex_array(request): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] +def vortex_array(request): rows: list[dict[str, list[int | None]]] = [] for _ in range(1_000): r: dict[str, list[int | None]] = {} - for col in range(request.param): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + for col in range(request.param): # Create large arrays of length 100 for each column. r[f"col{col}"] = [1, 2, None, 4] * 25 rows.append(r) @@ -30,11 +30,11 @@ def vortex_array(request): # pyright: ignore[reportUnknownParameterType, report @pytest.fixture(params=[10, 100]) -def arrow_array(request): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] +def arrow_array(request): rows: list[dict[str, list[int | None]]] = [] for _ in range(1_000): r: dict[str, list[int | None]] = {} - for col in range(request.param): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + for col in range(request.param): # Create large arrays of length 100 for each column. r[f"col{col}"] = [1, 2, None, 4] * 25 rows.append(r) @@ -57,6 +57,6 @@ def test_compress_parquet( def compress(): # write to bytes in memory. bout = io.BytesIO() - pq.write_table(arrow_array, bout) # pyright: ignore[reportArgumentType, reportUnknownMemberType] + pq.write_table(arrow_array, bout) # ty: ignore[invalid-argument-type] benchmark(compress) diff --git a/vortex-python/test/test_compress.py b/vortex-python/test/test_compress.py index 5909390d623..bb92e631587 100644 --- a/vortex-python/test/test_compress.py +++ b/vortex-python/test/test_compress.py @@ -48,8 +48,8 @@ def test_chunked_encode(): def test_table_encode(): - table = pa.table( # pyright: ignore[reportCallIssue, reportUnknownVariableType] - { # pyright: ignore[reportArgumentType] + table = pa.table( # ty: ignore[no-matching-overload] + { "number": pa.chunked_array([pa.array([0, 1, 2]), pa.array([3, 4, 5])]), "string": pa.chunked_array( [pa.array(["a", "b", "c"], type=pa.string_view()), pa.array(["d", "e", "f"], type=pa.string_view())] @@ -61,7 +61,7 @@ def test_table_encode(): encoded = vortex.array(table) arrow = encoded.to_arrow_array() assert isinstance(arrow, pa.ChunkedArray) - assert arrow.combine_chunks() == pa.StructArray.from_arrays( # pyright: ignore[reportUnknownMemberType] + assert arrow.combine_chunks() == pa.StructArray.from_arrays( [pa.array([0, 1, 2, 3, 4, 5]), pa.array(["a", "b", "c", "d", "e", "f"], type=pa.string_view())], names=["number", "string"], ) @@ -70,7 +70,7 @@ def test_table_encode(): @pytest.mark.skip(reason="We have no way to guarantee that the vortex-bench data has been downloaded.") def test_taxi(): curdir = Path(os.path.dirname(__file__)).parent.parent - table = pq.read_table(curdir / "vortex-bench/data/yellow-tripdata-2023-11.parquet") # pyright: ignore[reportUnknownMemberType] + table = pq.read_table(curdir / "vortex-bench/data/yellow-tripdata-2023-11.parquet") compressed = vortex.compress(vortex.array(table[:100])) decompressed = compressed.to_arrow_array() assert len(decompressed) == 100 diff --git a/vortex-python/test/test_dataset.py b/vortex-python/test/test_dataset.py index a3e57d7af84..b6cd8107598 100644 --- a/vortex-python/test/test_dataset.py +++ b/vortex-python/test/test_dataset.py @@ -25,14 +25,14 @@ def record(x: int, columns: list[str] | set[str] | None = None) -> dict[str, int @pytest.fixture(scope="session") -def ds(tmpdir_factory) -> vx.dataset.VortexDataset: # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] - fname = tmpdir_factory.mktemp("data") / "foo.vortex" # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] +def ds(tmpdir_factory) -> vx.dataset.VortexDataset: + fname = tmpdir_factory.mktemp("data") / "foo.vortex" - assert not os.path.exists(fname) # pyright: ignore[reportUnknownArgumentType] + assert not os.path.exists(fname) a = pa.array([record(x) for x in range(1_000_000)]) - vx.io.write(vx.array(a), str(fname)) # pyright: ignore[reportUnknownArgumentType] - return vx.dataset.VortexDataset.from_path(str(fname)) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(vx.array(a), str(fname)) + return vx.dataset.VortexDataset.from_path(str(fname)) def test_schema(ds: pd.Dataset): @@ -87,12 +87,12 @@ def fake_set_worker_threads(count: int | None) -> None: monkeypatch.setattr(vx_dataset, "_worker_threads", fake_worker_threads) monkeypatch.setattr(vx_dataset, "_set_worker_threads", fake_set_worker_threads) - with vx_dataset._temporary_worker_threads(True): # pyright: ignore[reportPrivateUsage] + with vx_dataset._temporary_worker_threads(True): assert current_workers == 11 assert current_workers == 3 - with vx_dataset._temporary_worker_threads(False): # pyright: ignore[reportPrivateUsage] + with vx_dataset._temporary_worker_threads(False): assert current_workers == 0 assert current_workers == 3 @@ -107,7 +107,7 @@ def fake_set_worker_threads(count: int | None) -> None: ], ) - batches = list(vx_dataset._read_batches_with_temporary_worker_threads(reader, True)) # pyright: ignore[reportPrivateUsage] + batches = list(vx_dataset._read_batches_with_temporary_worker_threads(reader, True)) assert [batch.to_pylist() for batch in batches] == [[{"x": 1}], [{"x": 2}]] assert current_workers == 3 @@ -144,7 +144,7 @@ def test_to_table(ds: pd.Dataset): def test_to_record_batch_reader_with_polars(ds: pd.Dataset): - pldf = polars.scan_pyarrow_dataset(ds).collect() # pyright: ignore[reportUnknownMemberType] + pldf = polars.scan_pyarrow_dataset(ds).collect() assert len(pldf) == 1_000_000 assert pldf.schema["index"] == polars.Int64 assert pldf.schema["string"] == polars.Utf8 @@ -192,7 +192,7 @@ def test_filter_with_nested_null_dtype(tmp_path: Path): def test_duckdb(ds: vx.dataset.VortexDataset): - assert ds # pyright cannot determine that ds is used by duckdb.execute + assert ds # the type checker cannot determine that ds is used by duckdb.execute tbl = duckdb.execute("select * from ds where string >= '950000' and float < 975.0").arrow().read_all() assert len(tbl) == 6176 diff --git a/vortex-python/test/test_datasource.py b/vortex-python/test/test_datasource.py index fc7fc0f492c..044e0a7a3ec 100644 --- a/vortex-python/test/test_datasource.py +++ b/vortex-python/test/test_datasource.py @@ -4,7 +4,7 @@ import pyarrow as pa import pytest import ray -from ray.data import read_datasource # pyright: ignore[reportUnknownVariableType] +from ray.data import read_datasource import vortex as vx from vortex.ray.datasource import VortexDatasource, partition @@ -22,9 +22,9 @@ def ray_init(): import ray._private.ray_constants as ray_constants ray_constants.RAY_ENABLE_UV_RUN_RUNTIME_ENV = False - _ = ray.init() # pyright: ignore[reportUnknownMemberType] + _ = ray.init() yield None - ray.shutdown() # pyright: ignore[reportUnknownMemberType] + ray.shutdown() def test_partition(): @@ -41,21 +41,21 @@ def test_partition(): assert partition(3, list(range(11))) == [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10]] -def test_vortex_datasource(ray_init, tmpdir_factory): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType, reportUnusedParameter] - folder = tmpdir_factory.mktemp("data") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] +def test_vortex_datasource(ray_init, tmpdir_factory): + folder = tmpdir_factory.mktemp("data") arr1 = vx.array([record(x) for x in range(5)]) - vx.io.write(arr1, str(folder / "01.vortex")) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(arr1, str(folder / "01.vortex")) arr2 = vx.array([record(x) for x in range(5, 10)]) - vx.io.write(arr2, str(folder / "02.vortex")) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(arr2, str(folder / "02.vortex")) - ds = read_datasource(VortexDatasource(url=str(folder))) # pyright: ignore[reportUnknownArgumentType] + ds = read_datasource(VortexDatasource(url=str(folder))) # Without an explicit sort, Ray may reorder rows *even within a single record batch*. ds = ds.sort("index") - tbl = pa.concat_tables(pa.Table.from_pydict(x) for x in ds.iter_batches()) # pyright: ignore[reportArgumentType, reportUnknownMemberType, reportUnknownVariableType] + tbl = pa.concat_tables(pa.Table.from_pydict(x) for x in ds.iter_batches()) expected = pa.Table.from_pylist([record(x) for x in range(0, 10)], schema=tbl.schema) assert tbl == expected diff --git a/vortex-python/test/test_duckdb.py b/vortex-python/test/test_duckdb.py index 1f0ae722dec..0ee40c84aff 100644 --- a/vortex-python/test/test_duckdb.py +++ b/vortex-python/test/test_duckdb.py @@ -20,7 +20,7 @@ def test_duckdb_via_substrait(tmp_path: Path) -> None: path = str(tmp_path / "test_timestamp.vortex") vx.io.write(table, path) - ds = vx.open(path).to_dataset() # noqa: F841 # pyright: ignore[reportUnusedVariable] - used by duckdb via SQL + ds = vx.open(path).to_dataset() # noqa: F841 - used by duckdb via SQL result = con.execute("SELECT * FROM ds WHERE ts > '2024-06-01'").fetchall() assert len(result) == 2 print(result) diff --git a/vortex-python/test/test_expr.py b/vortex-python/test/test_expr.py index f99ab9ffdad..ed5cf3bee76 100644 --- a/vortex-python/test/test_expr.py +++ b/vortex-python/test/test_expr.py @@ -129,7 +129,7 @@ def test_variant_get_paths() -> None: def test_variant_get_rejects_bad_path() -> None: with pytest.raises(TypeError): - _ = ve.variant_get(ve.column("payload"), [1.5]) # pyright: ignore[reportArgumentType] + _ = ve.variant_get(ve.column("payload"), [1.5]) # ty: ignore[invalid-argument-type] def test_ext_storage_returns_expr() -> None: @@ -138,7 +138,7 @@ def test_ext_storage_returns_expr() -> None: def test_merge_rejects_unknown_duplicate_handling() -> None: with pytest.raises(ValueError): - _ = ve.merge([ve.select(["name"])], duplicate_handling="nonsense") # pyright: ignore[reportArgumentType] + _ = ve.merge([ve.select(["name"])], duplicate_handling="nonsense") # ty: ignore[invalid-argument-type] def test_case_when_requires_a_pair() -> None: diff --git a/vortex-python/test/test_expression.py b/vortex-python/test/test_expression.py index 3d15d1daaa8..2e412dc1985 100644 --- a/vortex-python/test/test_expression.py +++ b/vortex-python/test/test_expression.py @@ -8,7 +8,7 @@ import pyarrow as pa import pyarrow.compute as pc import pytest -from vortex.arrow.expression import _schema_for_substrait, arrow_to_vortex # pyright: ignore[reportPrivateUsage] +from vortex.arrow.expression import _schema_for_substrait, arrow_to_vortex import vortex as vx import vortex.expr as ve @@ -41,12 +41,12 @@ class TestSchemaForSubstrait: def test_string_view_mapped_to_string(self): schema = pa.schema([("col", pa.string_view())]) result = _schema_for_substrait(schema) - assert result.field("col").type == pa.string() # pyright: ignore[reportUnknownMemberType] + assert result.field("col").type == pa.string() def test_binary_view_mapped_to_binary(self): schema = pa.schema([("col", pa.binary_view())]) result = _schema_for_substrait(schema) - assert result.field("col").type == pa.binary() # pyright: ignore[reportUnknownMemberType] + assert result.field("col").type == pa.binary() def test_other_types_unchanged(self): schema = pa.schema( @@ -121,10 +121,10 @@ def test_mixed_view_and_regular_types(self): (pa.binary_view(), b"test"), ], ) - def test_view_types_parametrized(self, view_type, value): # pyright: ignore[reportMissingParameterType, reportUnknownParameterType] - schema = pa.schema([("col", view_type)]) # pyright: ignore[reportUnknownArgumentType] - expr = pc.field("col") == value # pyright: ignore[reportUnknownVariableType] - vortex_expr = arrow_to_vortex(expr, schema) # pyright: ignore[reportUnknownArgumentType] + def test_view_types_parametrized(self, view_type, value): + schema = pa.schema([("col", view_type)]) + expr = pc.field("col") == value + vortex_expr = arrow_to_vortex(expr, schema) assert vortex_expr is not None def test_null_literal_expression(self): diff --git a/vortex-python/test/test_file.py b/vortex-python/test/test_file.py index 82fef155367..d7048cac4f9 100644 --- a/vortex-python/test/test_file.py +++ b/vortex-python/test/test_file.py @@ -20,14 +20,14 @@ def record(x: int, columns: list[str] | set[str] | None = None) -> dict[str, int @pytest.fixture(scope="session") -def vxf(tmpdir_factory) -> vx.VortexFile: # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] - fname = tmpdir_factory.mktemp("data") / "foo.vortex" # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] +def vxf(tmpdir_factory) -> vx.VortexFile: + fname = tmpdir_factory.mktemp("data") / "foo.vortex" - if not os.path.exists(fname): # pyright: ignore[reportUnknownArgumentType] + if not os.path.exists(fname): a = pa.array([record(x) for x in range(1_000_000)]) arr = vx.compress(vx.array(a)) - vx.io.write(arr, str(fname)) # pyright: ignore[reportUnknownArgumentType] - return vx.open(str(fname), without_segment_cache=True) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(arr, str(fname)) + return vx.open(str(fname), without_segment_cache=True) def test_dtype(vxf: VortexFile): @@ -72,7 +72,7 @@ def test_to_arrow_offset_string_schema(vxf: VortexFile): ) -def test_empty_file(tmpdir_factory): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] +def test_empty_file(tmpdir_factory): # test for writing empty files with null columns # create an empty table with schema `empty: null` table = pa.Table.from_pydict({"empty": []}) @@ -84,21 +84,21 @@ def test_empty_file(tmpdir_factory): # pyright: ignore[reportUnknownParameterTy assert repr(empty.dtype) == 'struct({"empty": null()}, nullable=False)' # writing file should succeed - empty_file = tmpdir_factory.mktemp("data") / "empty.vortex" # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - vx.io.write(empty, str(empty_file)) # pyright: ignore[reportUnknownArgumentType] + empty_file = tmpdir_factory.mktemp("data") / "empty.vortex" + vx.io.write(empty, str(empty_file)) -def test_stream_pyarrow(tmpdir_factory): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] +def test_stream_pyarrow(tmpdir_factory): import pyarrow.parquet as pq - data_dir = tmpdir_factory.mktemp("data") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + data_dir = tmpdir_factory.mktemp("data") table = pa.Table.from_pydict( { "names": ["Alice", "Bob", "Carol"], "ages": [21, 22, 23], } ) - pq.write_table(table, str(data_dir / "names.parquet")) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] + pq.write_table(table, str(data_dir / "names.parquet")) - df = pq.read_table(str(data_dir / "names.parquet")) # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] - vx.io.write(df, str(data_dir / "names.vortex")) # pyright: ignore[reportUnknownArgumentType] + df = pq.read_table(str(data_dir / "names.parquet")) + vx.io.write(df, str(data_dir / "names.vortex")) diff --git a/vortex-python/test/test_fixed_size_list.py b/vortex-python/test/test_fixed_size_list.py index c9d19c49efa..b41411c8aa2 100644 --- a/vortex-python/test/test_fixed_size_list.py +++ b/vortex-python/test/test_fixed_size_list.py @@ -38,7 +38,7 @@ def test_create_fixed_size_list_array_from_arrow(): # Convert back to Arrow to verify result = vx_array.to_arrow_array() assert pa.types.is_fixed_size_list(result.type) - assert result.type.list_size == 3 # pyright: ignore[reportAny] + assert result.type.list_size == 3 # PyArrow returns FixedSizeListScalar objects that need to be converted assert [x.as_py() for x in result] == [[1, 2, 3], [4, 5, 6], [7, 8, 9]] @@ -52,7 +52,7 @@ def test_create_nullable_fixed_size_list_array(): result = vx_array.to_arrow_array() assert pa.types.is_fixed_size_list(result.type) - assert result.type.list_size == 2 # pyright: ignore[reportAny] + assert result.type.list_size == 2 # Handle nullable values - PyArrow scalars have as_py() method assert [x.as_py() for x in result] == [[10, 20], None, [30, 40]] @@ -66,7 +66,7 @@ def test_fixed_size_list_with_string_elements(): result = vx_array.to_arrow_array() assert pa.types.is_fixed_size_list(result.type) - assert result.type.list_size == 3 # pyright: ignore[reportAny] + assert result.type.list_size == 3 # Convert FixedSizeListScalar to Python lists assert [x.as_py() for x in result] == [["a", "b", "c"], ["d", "e", "f"], ["g", "h", "i"]] @@ -80,7 +80,7 @@ def test_empty_fixed_size_list_array(): result = vx_array.to_arrow_array() assert pa.types.is_fixed_size_list(result.type) - assert result.type.list_size == 3 # pyright: ignore[reportAny] + assert result.type.list_size == 3 assert len(result) == 0 @@ -116,8 +116,8 @@ def test_fixed_size_list_with_f64_elements(): # Verify type is preserved assert pa.types.is_fixed_size_list(result.type) - assert result.type.list_size == 4 # pyright: ignore[reportAny] - assert result.type.value_type == pa.float64() # pyright: ignore[reportAny] + assert result.type.list_size == 4 + assert result.type.value_type == pa.float64() # Verify data is preserved # Use as_py() to convert PyArrow scalars to Python objects diff --git a/vortex-python/test/test_hf_datasets.py b/vortex-python/test/test_hf_datasets.py index 5ae43a8bf8b..0a47c5cc1c6 100644 --- a/vortex-python/test/test_hf_datasets.py +++ b/vortex-python/test/test_hf_datasets.py @@ -1,9 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# pyright: reportMissingTypeStubs=false -# pyright: reportUnknownMemberType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownVariableType=false import re import threading @@ -134,7 +130,7 @@ def test_streaming_select_columns_pushes_projection(tmp_path: Path): selected = dataset.select_columns(["text"]) assert isinstance(selected, vx_datasets.VortexIterableDataset) - assert selected._vortex_columns == ("text",) # pyright: ignore[reportPrivateUsage] + assert selected._vortex_columns == ("text",) assert list(selected) == [{"text": "zero"}, {"text": "one"}] @@ -182,18 +178,18 @@ def test_streaming_resume_with_limit_reads_full_limit(tmp_path: Path): vx_datasets.load_dataset(tmp_path / "train.vortex", split="train", limit=8, batch_size=2), ) examples = cast( - vx_datasets._VortexExamplesIterable, # pyright: ignore[reportPrivateUsage] - dataset._ex_iterable, # pyright: ignore[reportPrivateUsage] + vx_datasets._VortexExamplesIterable, + dataset._ex_iterable, ) # Simulate resuming after the first two rows of the file were already yielded. - examples._state_dict = { # pyright: ignore[reportPrivateUsage] + examples._state_dict = { "file_idx": 0, "file_row_idx": 2, "num_yielded": 2, "type": type(examples).__name__, } - produced = [row for _key, table in examples._iter_arrow() for row in table.to_pylist()] # pyright: ignore[reportPrivateUsage] + produced = [row for _key, table in examples._iter_arrow() for row in table.to_pylist()] # The limit of 8 must still be honored: six rows remain after the two already yielded. assert produced == rows[2:8] @@ -262,7 +258,7 @@ def test_streaming_take_splits_limit_across_shards(tmp_path: Path): ) limited = dataset.take(4) assert isinstance(limited, vx_datasets.VortexIterableDataset) - examples = limited._ex_iterable # pyright: ignore[reportPrivateUsage] + examples = limited._ex_iterable # DataLoader worker / distributed sharding must split the pushed-down limit so the shards # together yield exactly take(n) rows, mirroring TakeExamplesIterable.split_number. @@ -449,7 +445,7 @@ def list_repo_files(self, repo_id: str, *, repo_type: str | None = None, revisio def test_hub_streaming_resolves_to_hf_uris_without_download(token: bool | None, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(vx_datasets, "HfApi", _FakeHfApi) - files, store = vx_datasets._resolve_data_files( # pyright: ignore[reportPrivateUsage] + files, store = vx_datasets._resolve_data_files( "org/name", data_files=None, split="train", @@ -474,7 +470,7 @@ def test_hub_streaming_resolves_to_hf_uris_without_download(token: bool | None, def test_hub_streaming_with_token_false_forces_anonymous_store(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(vx_datasets, "HfApi", _FakeHfApi) - files, store = vx_datasets._resolve_data_files( # pyright: ignore[reportPrivateUsage] + files, store = vx_datasets._resolve_data_files( "org/name", data_files=None, split="train", @@ -494,7 +490,7 @@ def test_hub_streaming_with_token_false_forces_anonymous_store(monkeypatch: pyte def test_hub_streaming_with_token_uses_authenticated_store(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(vx_datasets, "HfApi", _FakeHfApi) - files, store = vx_datasets._resolve_data_files( # pyright: ignore[reportPrivateUsage] + files, store = vx_datasets._resolve_data_files( "org/name", data_files=None, split="train", @@ -525,20 +521,20 @@ def test_hub_streaming_with_token_uses_authenticated_store(monkeypatch: pytest.M ], ) def test_parse_hf_uri(uri: str, expected: tuple[str, str | None, str | None]): - assert vx_datasets._parse_hf_uri(uri) == expected # pyright: ignore[reportPrivateUsage] + assert vx_datasets._parse_hf_uri(uri) == expected @pytest.mark.parametrize("uri", ["hf://org/name/file.vortex", "hf://datasets/name-only", "hf://datasets/org/@main"]) def test_parse_hf_uri_invalid(uri: str): with pytest.raises(ValueError, match="hf://"): - _ = vx_datasets._parse_hf_uri(uri) # pyright: ignore[reportPrivateUsage] + _ = vx_datasets._parse_hf_uri(uri) def test_hf_uri_streaming_resolves_file_and_directory(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(vx_datasets, "HfApi", _FakeHfApi) def resolve(path: str): - return vx_datasets._resolve_data_files( # pyright: ignore[reportPrivateUsage] + return vx_datasets._resolve_data_files( path, data_files=None, split="train", @@ -561,7 +557,7 @@ def resolve(path: str): def test_hf_uri_slash_revision_stays_percent_encoded(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(vx_datasets, "HfApi", _FakeHfApi) - files, store = vx_datasets._resolve_data_files( # pyright: ignore[reportPrivateUsage] + files, store = vx_datasets._resolve_data_files( "hf://datasets/org/name@refs%2Fconvert%2Fparquet", data_files=None, split="train", @@ -623,7 +619,7 @@ def test_local_directory_in_data_files(tmp_path: Path): ], ) def test_glob_match(pattern: str, path: str, expected: bool): - assert vx_datasets._glob_match(path, pattern) is expected # pyright: ignore[reportPrivateUsage] + assert vx_datasets._glob_match(path, pattern) is expected @pytest.mark.parametrize("repo_type", ["dataset", "datasets", "model", "space"]) diff --git a/vortex-python/test/test_polars_.py b/vortex-python/test/test_polars_.py index fd15751aa42..25d14baced4 100644 --- a/vortex-python/test/test_polars_.py +++ b/vortex-python/test/test_polars_.py @@ -42,13 +42,13 @@ def test_exprs(polars: pl.Expr, vortex: ve.Expr): @pytest.fixture(scope="module") -def vxf(tmpdir_factory): # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] - fname = tmpdir_factory.mktemp("data") / "polars_test.vortex" # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] +def vxf(tmpdir_factory): + fname = tmpdir_factory.mktemp("data") / "polars_test.vortex" - if not os.path.exists(fname): # pyright: ignore[reportUnknownArgumentType] + if not os.path.exists(fname): a = pa.array([{"index": x, "value": math.sqrt(x)} for x in range(1_000_000)]) - vx.io.write(vx.compress(vx.array(a)), str(fname)) # pyright: ignore[reportUnknownArgumentType] - return vx.open(str(fname), without_segment_cache=True) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(vx.compress(vx.array(a)), str(fname)) + return vx.open(str(fname), without_segment_cache=True) def test_to_polars_with_limit(vxf: vx.VortexFile): diff --git a/vortex-python/test/test_pyarray.py b/vortex-python/test/test_pyarray.py index f29ca41408f..123ebaf501d 100644 --- a/vortex-python/test/test_pyarray.py +++ b/vortex-python/test/test_pyarray.py @@ -8,11 +8,11 @@ import numpy as np import pyarrow as pa import pytest -from pcodec import ( # pyright: ignore[reportMissingTypeStubs] - ChunkConfig, # pyright: ignore[reportAttributeAccessIssue, reportUnknownVariableType] +from pcodec import ( + ChunkConfig, # ty: ignore[unresolved-import] ) -from pcodec import ( # pyright: ignore[reportMissingTypeStubs] - wrapped as pco, # pyright: ignore[reportAttributeAccessIssue, reportUnknownVariableType] +from pcodec import ( + wrapped as pco, # ty: ignore[unresolved-import] ) from typing_extensions import override @@ -43,17 +43,17 @@ def __init__( chunk_header: memoryview, data: memoryview, ): - (fd, _bytes_read) = pco.FileDecompressor.new(file_header) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + (fd, _bytes_read) = pco.FileDecompressor.new(file_header) if dtype == vx.int_(64, nullable=True): dt = "i64" else: raise ValueError(f"Unsupported dtype: {dtype}") - (cd, _bytes_read) = fd.read_chunk_meta(chunk_header, dt) # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] + (cd, _bytes_read) = fd.read_chunk_meta(chunk_header, dt) dst = np.array([0] * length, dtype=np.int64) - cd.read_page_into( # pyright: ignore[reportUnknownMemberType] + cd.read_page_into( data, page_n=length, dst=dst, @@ -66,27 +66,27 @@ def __init__( self._data = data @classmethod - def encode(cls, array: pa.Array[pa.Scalar[pa.DataType]], config: ChunkConfig | None = None) -> PCodecArray: # pyright: ignore[reportUnknownParameterType] + def encode(cls, array: pa.Array[pa.Scalar[pa.DataType]], config: ChunkConfig | None = None) -> PCodecArray: assert array.null_count == 0, "Cannot compress arrays with nulls" - config = config or ChunkConfig() # pyright: ignore[reportUnknownVariableType] + config = config or ChunkConfig() - fc = pco.FileCompressor() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - file_header = fc.write_header() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + fc = pco.FileCompressor() + file_header = fc.write_header() - cc = fc.chunk_compressor(array.to_numpy(), config) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] - chunk_header = cc.write_chunk_meta() # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + cc = fc.chunk_compressor(array.to_numpy(), config) + chunk_header = cc.write_chunk_meta() data = b"" - for i, _n in enumerate(cc.n_per_page()): # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType, reportUnknownVariableType] - data += cc.write_page(i) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + for i, _n in enumerate(cc.n_per_page()): + data += cc.write_page(i) return PCodecArray( len(array), vx.DType.from_arrow(array.type), - file_header, # pyright: ignore[reportUnknownArgumentType] - chunk_header, # pyright: ignore[reportUnknownArgumentType] - memoryview(data), # pyright: ignore[reportUnknownArgumentType] + file_header, + chunk_header, + memoryview(data), ) @override @@ -99,6 +99,6 @@ def decode(cls, parts: vx.SerializedArray, ctx: vx.ArrayContext, dtype: vx.DType @pytest.mark.skip(reason="Not implemented yet") def test_pcodec(): - _ = PCodecArray.encode(pa.array([0, 1, 2, 3, 4])) # pyright: ignore[reportUnknownMemberType] + _ = PCodecArray.encode(pa.array([0, 1, 2, 3, 4])) vx.registry.register(PCodecArray) diff --git a/vortex-python/test/test_scan.py b/vortex-python/test/test_scan.py index 3bf1efbd220..bff1f7d7429 100644 --- a/vortex-python/test/test_scan.py +++ b/vortex-python/test/test_scan.py @@ -26,14 +26,14 @@ def vxscan(vxfile: vx.VortexFile) -> vx.RepeatedScan: @pytest.fixture(scope="session") -def vxfile(tmpdir_factory) -> vx.VortexFile: # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] - fname = tmpdir_factory.mktemp("data") / "foo.vortex" # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] +def vxfile(tmpdir_factory) -> vx.VortexFile: + fname = tmpdir_factory.mktemp("data") / "foo.vortex" - if not os.path.exists(fname): # pyright: ignore[reportUnknownArgumentType] + if not os.path.exists(fname): a = pa.array([record(x) for x in range(1_000)]) arr = vx.compress(vx.array(a)) - vx.io.write(arr, str(fname)) # pyright: ignore[reportUnknownArgumentType] - return vx.open(str(fname)) # pyright: ignore[reportUnknownArgumentType] + vx.io.write(arr, str(fname)) + return vx.open(str(fname)) def test_execute(vxscan: RepeatedScan): From 6f50bcbfe38cbab6715a0345e0ef95a27d636789 Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 22 Aug 2026 00:46:25 +0000 Subject: [PATCH 2/2] Skip the Rust extension build in the Python lint job ty resolves the vortex package from the in-tree sources and stubs via the configured source roots, so the lint job only needs the Python dependencies. Syncing with --no-install-package for the two maturin packages avoids the ~4.5 min extension build. Signed-off-by: "Robert" --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da4464e4700..d34dfb0e78d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,13 +85,13 @@ jobs: run: uvx ruff format --check . - name: Python Lint - Ruff run: uvx ruff check . - # ty needs the project for type information, so use uv run + # ty needs the Python dependencies for type information, but resolves the + # vortex package itself from the in-tree sources and stubs, so skip the + # Rust extension build entirely. - name: Python Lint - ty - env: - MATURIN_PEP517_ARGS: "--profile ci" run: | - uv sync --all-packages - uv run ty check vortex-python + uv sync --all-packages --no-install-package vortex-data --no-install-package vortex-data-cuda + uv run --no-sync ty check vortex-python python-test: name: "Python (test)"