diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java index 7907aafe9fca..c50df9436cc5 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java @@ -38,7 +38,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import com.google.common.hash.HashingInputStream; import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; import com.google.gson.Gson; import com.google.gson.stream.JsonReader; import java.io.IOException; @@ -61,6 +63,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +@SuppressWarnings("UnstableApiUsage") class ApiaryUnbufferedReadableByteChannel implements UnbufferedReadableByteChannel { private final ApiaryReadRequest apiaryReadRequest; @@ -68,6 +71,7 @@ class ApiaryUnbufferedReadableByteChannel implements UnbufferedReadableByteChann private final SettableApiFuture result; private final ResultRetryAlgorithm resultRetryAlgorithm; private final Retrier retrier; + private final Hasher hasher; private long position; private ScatteringByteChannel sbc; @@ -77,16 +81,21 @@ class ApiaryUnbufferedReadableByteChannel implements UnbufferedReadableByteChann // returned X-Goog-Generation header value private Long xGoogGeneration; + private HashingInputStream hashingInputStream; + private String expectedCrc32cBase64; + ApiaryUnbufferedReadableByteChannel( ApiaryReadRequest apiaryReadRequest, Storage storage, SettableApiFuture result, Retrier retrier, - ResultRetryAlgorithm resultRetryAlgorithm) { + ResultRetryAlgorithm resultRetryAlgorithm, + Hasher hasher) { this.apiaryReadRequest = apiaryReadRequest; this.storage = storage; this.result = result; this.retrier = retrier; + this.hasher = hasher; this.resultRetryAlgorithm = new BasicResultRetryAlgorithm() { @Override @@ -126,6 +135,16 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { long read = sbc.read(dsts, offset, length); if (read == -1) { returnEOF = true; + if (hashingInputStream != null && expectedCrc32cBase64 != null) { + int calculatedCrc32c = hashingInputStream.hash().asInt(); + byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64); + int expectedVal = Ints.fromByteArray(decoded); + + Crc32cValue expected = Crc32cValue.of(expectedVal, 0); + Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0); + + hasher.validate(expected, actual); + } } else { totalRead += read; } @@ -211,9 +230,19 @@ private ScatteringByteChannel open() { if (!result.isDone()) { result.set(clone); } + + Map hashes = ChecksumResponseParser.extractHashesFromHeader(media); + this.expectedCrc32cBase64 = hashes.get("crc32c"); } } + boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher); + boolean isFullObjectDownload = (request.getByteRangeSpec().getHttpRangeHeader() == null); + if (isHasherEnabled && isFullObjectDownload && expectedCrc32cBase64 != null) { + this.hashingInputStream = new HashingInputStream(Hashing.crc32c(), content); + content = this.hashingInputStream; + } + ReadableByteChannel rbc = Channels.newChannel(content); return StorageByteChannels.readable().asScatteringByteChannel(rbc); } catch (HttpResponseException e) { diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java index 2b81fac694da..202cf13b3051 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java @@ -54,10 +54,11 @@ public static final class ReadableByteChannelSessionBuilder { private final BlobReadChannelContext blobReadChannelContext; private boolean autoGzipDecompression; - // private Hasher hasher; // TODO: wire in Hasher + private final Hasher hasher; private ReadableByteChannelSessionBuilder(BlobReadChannelContext blobReadChannelContext) { this.blobReadChannelContext = blobReadChannelContext; + this.hasher = Hasher.defaultHasher(); this.autoGzipDecompression = false; } @@ -96,7 +97,8 @@ public UnbufferedReadableByteChannelSessionBuilder unbuffered() { blobReadChannelContext.getApiaryClient(), resultFuture, blobReadChannelContext.getRetrier(), - blobReadChannelContext.getRetryAlgorithmManager().idempotent()), + blobReadChannelContext.getRetryAlgorithmManager().idempotent(), + hasher), ApiFutures.transform( resultFuture, StorageObject::getContentEncoding, MoreExecutors.directExecutor())); } else { @@ -105,7 +107,8 @@ public UnbufferedReadableByteChannelSessionBuilder unbuffered() { blobReadChannelContext.getApiaryClient(), resultFuture, blobReadChannelContext.getRetrier(), - blobReadChannelContext.getRetryAlgorithmManager().idempotent()); + blobReadChannelContext.getRetryAlgorithmManager().idempotent(), + hasher); } }; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java new file mode 100644 index 000000000000..fe31d8678c1d --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.core.SettableApiFuture; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.storage.ApiaryUnbufferedReadableByteChannel.ApiaryReadRequest; +import com.google.cloud.storage.Retrying.RetrierWithAlg; +import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import org.junit.Test; + +public class ApiaryUnbufferedReadableByteChannelTest { + + private static final byte[] CONTENT_BYTES = "Hello, World!".getBytes(); + private static final String CORRECT_CRC32C_BASE64 = "TVUQaA=="; + private static final String WRONG_CRC32C_BASE64 = "AAAAAA=="; + + private Storage createMockStorageClient(String googHashHeader) { + HttpTransport transport = + new HttpTransport() { + @Override + protected com.google.api.client.http.LowLevelHttpRequest buildRequest( + String method, String url) throws IOException { + return new com.google.api.client.testing.http.MockLowLevelHttpRequest() { + @Override + public com.google.api.client.http.LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse lowLevelResponse = + new MockLowLevelHttpResponse() + .setContent(CONTENT_BYTES) + .setContentLength(CONTENT_BYTES.length) + .addHeader("Content-Length", String.valueOf(CONTENT_BYTES.length)) + .addHeader("x-goog-generation", "12345"); + if (googHashHeader != null) { + lowLevelResponse.addHeader("x-goog-hash", googHashHeader); + } + return lowLevelResponse; + } + }; + } + }; + return new Storage.Builder(transport, GsonFactory.getDefaultInstance(), null) + .setApplicationName("test") + .build(); + } + + @Test + public void testRead_successfulCrc32cValidation() throws IOException { + Storage storageClient = createMockStorageClient("crc32c=" + CORRECT_CRC32C_BASE64); + + StorageObject from = new StorageObject().setBucket("bucket").setName("blob"); + ApiaryReadRequest apiaryReadRequest = + new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange()); + + SettableApiFuture resultFuture = SettableApiFuture.create(); + try (ApiaryUnbufferedReadableByteChannel channel = + new ApiaryUnbufferedReadableByteChannel( + apiaryReadRequest, + storageClient, + resultFuture, + RetrierWithAlg.attemptOnce(), + Retrying.neverRetry(), + Hasher.defaultHasher()); ) { + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (WritableByteChannel w = Channels.newChannel(out)) { + ByteBuffer buf = ByteBuffer.allocate(4096); + while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) { + buf.flip(); + w.write(buf); + buf.clear(); + } + } + + assertArrayEquals(CONTENT_BYTES, out.toByteArray()); + } + } + + @Test + public void testRead_mismatchedCrc32cValidation_throwsChecksumMismatch() throws IOException { + Storage storageClient = createMockStorageClient("crc32c=" + WRONG_CRC32C_BASE64); + + StorageObject from = new StorageObject().setBucket("bucket").setName("blob"); + ApiaryReadRequest apiaryReadRequest = + new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange()); + + SettableApiFuture resultFuture = SettableApiFuture.create(); + try (ApiaryUnbufferedReadableByteChannel channel = + new ApiaryUnbufferedReadableByteChannel( + apiaryReadRequest, + storageClient, + resultFuture, + RetrierWithAlg.attemptOnce(), + Retrying.neverRetry(), + Hasher.defaultHasher()); ) { + + ByteBuffer buf = ByteBuffer.allocate(4096); + Hasher.ChecksumMismatchException expected = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> { + while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) { + buf.clear(); + } + }); + + assertTrue(expected.getMessage().contains("Mismatch checksum value")); + } + } +}