diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b9c0200d..4df3108d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: - name: install run: | pip install -e . + pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main - name: Run Standard Tests run: | export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json diff --git a/cloudbuild/e2e-tests-cloudbuild.yaml b/cloudbuild/e2e-tests-cloudbuild.yaml index 84e2fee3e..4f24d98c7 100644 --- a/cloudbuild/e2e-tests-cloudbuild.yaml +++ b/cloudbuild/e2e-tests-cloudbuild.yaml @@ -99,6 +99,7 @@ steps: pip install --upgrade pip > /dev/null # Install testing libraries explicitly, as they are not in setup.py pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null + pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main pip install -e . > /dev/null " @@ -146,7 +147,7 @@ steps: export GCSFS_TEST_PROJECT=${PROJECT_ID} && \ export GCSFS_TEST_KMS_KEY=projects/${PROJECT_ID}/locations/${_REGION}/keyRings/${_GCSFS_KEY_RING_NAME}/cryptoKeys/${_GCSFS_KEY_NAME} && \ echo '--- Running Zonal tests on VM ---' && \ - ulimit -n 2048 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ + ulimit -n 4096 && export GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT='true' && \ pip install --upgrade --force-reinstall git+https://github.com/googleapis/python-storage.git@main && \ pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' --color=no gcsfs/tests/test_extended_gcsfs.py gcsfs/tests/test_zonal_file.py gcsfs/tests/test_async_gcsfs.py " diff --git a/gcsfs/core.py b/gcsfs/core.py index 564141b4f..c9d7353aa 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -2157,7 +2157,7 @@ async def upload_chunk(fs, location, data, offset, size, content_type): Uploads a chunk of data. This function has a conditional path to support experimental features for Zonal buckets to append data using gRPC. """ - from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( + from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) diff --git a/gcsfs/extended_gcsfs.py b/gcsfs/extended_gcsfs.py index e52c73594..0ca0f7df5 100644 --- a/gcsfs/extended_gcsfs.py +++ b/gcsfs/extended_gcsfs.py @@ -13,11 +13,11 @@ from google.api_core.client_info import ClientInfo from google.auth.credentials import AnonymousCredentials from google.cloud import storage_control_v2 -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) -from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) @@ -85,7 +85,7 @@ async def _get_grpc_client(self): self._grpc_client = AsyncGrpcClient( credentials=self.credential, client_info=ClientInfo(user_agent=f"{USER_AGENT}/{version}"), - ).grpc_client + ) return self._grpc_client async def _get_control_plane_client(self): diff --git a/gcsfs/tests/conftest.py b/gcsfs/tests/conftest.py index 4f4aaf742..669badf68 100644 --- a/gcsfs/tests/conftest.py +++ b/gcsfs/tests/conftest.py @@ -11,7 +11,7 @@ import pytest_asyncio import requests from google.cloud import storage -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) diff --git a/gcsfs/tests/test_extended_gcsfs.py b/gcsfs/tests/test_extended_gcsfs.py index b357920c2..886c964cb 100644 --- a/gcsfs/tests/test_extended_gcsfs.py +++ b/gcsfs/tests/test_extended_gcsfs.py @@ -11,10 +11,10 @@ import pytest from google.api_core.exceptions import NotFound -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) from google.cloud.storage.exceptions import DataCorruption diff --git a/gcsfs/zb_hns_utils.py b/gcsfs/zb_hns_utils.py index 57d5d842e..b7638ed26 100644 --- a/gcsfs/zb_hns_utils.py +++ b/gcsfs/zb_hns_utils.py @@ -1,6 +1,6 @@ from io import BytesIO -from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( +from google.cloud.storage.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) diff --git a/gcsfs/zonal_file.py b/gcsfs/zonal_file.py index f78c155c5..b5f69b858 100644 --- a/gcsfs/zonal_file.py +++ b/gcsfs/zonal_file.py @@ -1,7 +1,7 @@ import logging from fsspec import asyn -from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( +from google.cloud.storage.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, ) diff --git a/requirements.txt b/requirements.txt index 9ce803f91..08ace2205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ decorator>4.1.2 fsspec==2026.1.0 google-auth>=1.2 google-auth-oauthlib -google-cloud-storage>=3.8.0 +google-cloud-storage>=3.9.0 google-cloud-storage-control requests