From 18bccdab947fc992114c9f3a48fa257c2ca6deab Mon Sep 17 00:00:00 2001 From: suni72 Date: Fri, 30 Jan 2026 14:09:02 +0000 Subject: [PATCH 1/5] Update imports for python sdk Update grpc_client since MRD and AAOW now accept AsyncGrpcClient object --- gcsfs/core.py | 2 +- gcsfs/extended_gcsfs.py | 9 +++++---- gcsfs/tests/conftest.py | 2 +- gcsfs/tests/test_extended_gcsfs.py | 5 +++-- gcsfs/tests/test_zonal_file.py | 3 +++ gcsfs/zb_hns_utils.py | 3 ++- gcsfs/zonal_file.py | 5 ++++- 7 files changed, 19 insertions(+), 10 deletions(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index 564141b4f..c9d7353aa 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -2157,7 +2157,7 @@ async def upload_chunk(fs, location, data, offset, size, content_type): Uploads a chunk of data. This function has a conditional path to support experimental features for Zonal buckets to append data using gRPC. """ - from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( + from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) diff --git a/gcsfs/extended_gcsfs.py b/gcsfs/extended_gcsfs.py index e52c73594..f8e163f09 100644 --- a/gcsfs/extended_gcsfs.py +++ b/gcsfs/extended_gcsfs.py @@ -13,11 +13,12 @@ from google.api_core.client_info import ClientInfo from google.auth.credentials import AnonymousCredentials from google.cloud import storage_control_v2 -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) -from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) @@ -85,7 +86,7 @@ async def _get_grpc_client(self): self._grpc_client = AsyncGrpcClient( credentials=self.credential, client_info=ClientInfo(user_agent=f"{USER_AGENT}/{version}"), - ).grpc_client + ) return self._grpc_client async def _get_control_plane_client(self): diff --git a/gcsfs/tests/conftest.py b/gcsfs/tests/conftest.py index 4f4aaf742..669badf68 100644 --- a/gcsfs/tests/conftest.py +++ b/gcsfs/tests/conftest.py @@ -11,7 +11,7 @@ import pytest_asyncio import requests from google.cloud import storage -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) diff --git a/gcsfs/tests/test_extended_gcsfs.py b/gcsfs/tests/test_extended_gcsfs.py index b357920c2..cfe6dd4b8 100644 --- a/gcsfs/tests/test_extended_gcsfs.py +++ b/gcsfs/tests/test_extended_gcsfs.py @@ -11,10 +11,11 @@ import pytest from google.api_core.exceptions import NotFound -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) from google.cloud.storage.exceptions import DataCorruption diff --git a/gcsfs/tests/test_zonal_file.py b/gcsfs/tests/test_zonal_file.py index 76c0bfcf4..8cd485d17 100644 --- a/gcsfs/tests/test_zonal_file.py +++ b/gcsfs/tests/test_zonal_file.py @@ -4,6 +4,9 @@ from unittest import mock import pytest +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + _DEFAULT_FLUSH_INTERVAL_BYTES, +) from gcsfs.tests.settings import TEST_ZONAL_BUCKET from gcsfs.tests.utils import tempdir, tmpfile diff --git a/gcsfs/zb_hns_utils.py b/gcsfs/zb_hns_utils.py index 57d5d842e..4fbef554a 100644 --- a/gcsfs/zb_hns_utils.py +++ b/gcsfs/zb_hns_utils.py @@ -1,6 +1,7 @@ from io import BytesIO -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) diff --git a/gcsfs/zonal_file.py b/gcsfs/zonal_file.py index f78c155c5..3bf461779 100644 --- a/gcsfs/zonal_file.py +++ b/gcsfs/zonal_file.py @@ -1,7 +1,10 @@ import logging from fsspec import asyn -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + _DEFAULT_FLUSH_INTERVAL_BYTES, +) +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) From 696521afea93b53f2ca02058039164808df57b2b Mon Sep 17 00:00:00 2001 From: suni72 Date: Fri, 30 Jan 2026 14:21:33 +0000 Subject: [PATCH 2/5] update ci tests to use latest changes from python-storage --- .github/workflows/ci.yml | 1 + cloudbuild/e2e-tests-cloudbuild.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b9c0200d..4df3108d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: - name: install run: | pip install -e . + pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main - name: Run Standard Tests run: | export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 84e2fee3e..5597a1c3d 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -99,6 +99,7 @@ steps: pip install --upgrade pip > /dev/null # Install testing libraries explicitly, as they are not in setup.py pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null + pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main pip install -e . > /dev/null " From 57dbfbe88ec5f5dc8f914072b488e54aa277d92a Mon Sep 17 00:00:00 2001 From: suni72 Date: Mon, 2 Feb 2026 20:24:57 +0000 Subject: [PATCH 3/5] update requirements to use latest python sdk release --- gcsfs/extended_gcsfs.py | 1 - gcsfs/tests/test_extended_gcsfs.py | 1 - gcsfs/tests/test_zonal_file.py | 3 --- gcsfs/zb_hns_utils.py | 1 - gcsfs/zonal_file.py | 3 --- requirements.txt | 2 +- 6 files changed, 1 insertion(+), 10 deletions(-) diff --git a/gcsfs/extended_gcsfs.py b/gcsfs/extended_gcsfs.py index f8e163f09..0ca0f7df5 100644 --- a/gcsfs/extended_gcsfs.py +++ b/gcsfs/extended_gcsfs.py @@ -14,7 +14,6 @@ from google.auth.credentials import AnonymousCredentials from google.cloud import storage_control_v2 from google.cloud.storage.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient diff --git a/gcsfs/tests/test_extended_gcsfs.py b/gcsfs/tests/test_extended_gcsfs.py index cfe6dd4b8..886c964cb 100644 --- a/gcsfs/tests/test_extended_gcsfs.py +++ b/gcsfs/tests/test_extended_gcsfs.py @@ -12,7 +12,6 @@ import pytest from google.api_core.exceptions import NotFound from google.cloud.storage.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) from google.cloud.storage.asyncio.async_multi_range_downloader import ( diff --git a/gcsfs/tests/test_zonal_file.py b/gcsfs/tests/test_zonal_file.py index 8cd485d17..76c0bfcf4 100644 --- a/gcsfs/tests/test_zonal_file.py +++ b/gcsfs/tests/test_zonal_file.py @@ -4,9 +4,6 @@ from unittest import mock import pytest -from google.cloud.storage.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, -) from gcsfs.tests.settings import TEST_ZONAL_BUCKET from gcsfs.tests.utils import tempdir, tmpfile diff --git a/gcsfs/zb_hns_utils.py b/gcsfs/zb_hns_utils.py index 4fbef554a..b7638ed26 100644 --- a/gcsfs/zb_hns_utils.py +++ b/gcsfs/zb_hns_utils.py @@ -1,7 +1,6 @@ from io import BytesIO from google.cloud.storage.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, AsyncAppendableObjectWriter, ) diff --git a/gcsfs/zonal_file.py b/gcsfs/zonal_file.py index 3bf461779..b5f69b858 100644 --- a/gcsfs/zonal_file.py +++ b/gcsfs/zonal_file.py @@ -1,9 +1,6 @@ import logging from fsspec import asyn -from google.cloud.storage.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, -) from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) diff --git a/requirements.txt b/requirements.txt index 9ce803f91..08ace2205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ decorator>4.1.2 fsspec==2026.1.0 google-auth>=1.2 google-auth-oauthlib -google-cloud-storage>=3.8.0 +google-cloud-storage>=3.9.0 google-cloud-storage-control requests From cc87a9603e17acf7774446bf5002a3667f4d1520 Mon Sep 17 00:00:00 2001 From: Ankita Luthra Date: Tue, 3 Feb 2026 04:45:25 +0000 Subject: [PATCH 4/5] retriggering ci checks From 816085f295221fff49b1849fac06fc140728815d Mon Sep 17 00:00:00 2001 From: suni72 Date: Tue, 3 Feb 2026 05:04:47 +0000 Subject: [PATCH 5/5] update ulimit to avoid too many open file error --- cloudbuild/e2e-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 5597a1c3d..4f24d98c7 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -147,7 +147,7 @@ steps: export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running Zonal tests on VM ---' && \ - ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + ulimit -n 4096 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main && \ pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py gcsfs/tests/test_zonal_file.py gcsfs/tests/test_async_gcsfs.py "