diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java index c1b506de2f7e..97947e78563c 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java @@ -73,6 +73,8 @@ default Crc32cLengthKnown hash(Supplier b) { void validateUnchecked(Crc32cValue expected, ByteString byteString) throws UncheckedChecksumMismatchException; + void validate(Crc32cValue expected, Crc32cLengthKnown actual) throws ChecksumMismatchException; + @Nullable > C nullSafeConcat( @Nullable C r1, @Nullable Crc32cLengthKnown r2); @@ -122,6 +124,9 @@ public void validate(Crc32cValue expected, ByteString b) {} @Override public void validateUnchecked(Crc32cValue expected, ByteString byteString) {} + @Override + public void validate(Crc32cValue expected, Crc32cLengthKnown actual) {} + @Override public > @Nullable C nullSafeConcat( @Nullable C r1, @Nullable Crc32cLengthKnown r2) { @@ -189,6 +194,14 @@ public void validateUnchecked(Crc32cValue expected, ByteString byteString) } } + @Override + public void validate(Crc32cValue expected, Crc32cLengthKnown actual) + throws ChecksumMismatchException { + if (!actual.eqValue(expected)) { + throw new ChecksumMismatchException(expected, actual); + } + } + @SuppressWarnings("unchecked") @Override public > @Nullable C nullSafeConcat( @@ -212,7 +225,7 @@ final class ChecksumMismatchException extends IOException { private final Crc32cValue expected; private final Crc32cLengthKnown actual; - private ChecksumMismatchException(Crc32cValue expected, Crc32cLengthKnown actual) { + ChecksumMismatchException(Crc32cValue expected, Crc32cLengthKnown actual) { super( String.format( Locale.US, diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java new file mode 100644 index 000000000000..131cc161c007 --- /dev/null +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.client.http.HttpResponse; +import com.google.api.core.InternalApi; +import com.google.common.hash.Hashing; +import com.google.common.hash.HashingOutputStream; +import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Internal utility class to perform client-side CRC32C checksum validation on downloaded data + * specifically for the {@code HttpStorageRpc} transport layer. + */ +@InternalApi +public final class HttpStorageRpcHasherHelper { + + public static final HttpStorageRpcHasherHelper INSTANCE = new HttpStorageRpcHasherHelper(); + + private final Hasher hasher; + + private HttpStorageRpcHasherHelper() { + hasher = Hasher.defaultHasher(); + } + + /** + * Returns a wrapping output stream that hashes the written content if validation is enabled, or + * the original output stream otherwise. + */ + @SuppressWarnings("UnstableApiUsage") + public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) { + boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher); + return (isChecksumValidationEnabled && isHasherEnabled) + ? new HashingOutputStream(Hashing.crc32c(), out) + : out; + } + + /** + * Validates a raw byte array against GCS's expected base64-encoded value in response headers. + * + * @throws IOException if the checksums do not match. + */ + public void validate(HttpResponse response, byte[] content) throws IOException { + Map hashes = ChecksumResponseParser.extractHashesFromHeader(response); + String expectedCrc32cBase64 = hashes.get("crc32c"); + if (expectedCrc32cBase64 != null) { + validateCrc32c(expectedCrc32cBase64, content); + } + } + + /** + * Validates the downloaded output stream against GCS's expected base64-encoded value in response + * headers. + * + * @throws IOException if the checksums do not match. + */ + @SuppressWarnings("UnstableApiUsage") + public void validate(HttpResponse response, OutputStream activeStream) throws IOException { + if (activeStream instanceof HashingOutputStream) { + HashingOutputStream targetStream = (HashingOutputStream) activeStream; + + Map hashes = ChecksumResponseParser.extractHashesFromHeader(response); + String expectedCrc32cBase64 = hashes.get("crc32c"); + if (expectedCrc32cBase64 != null) { + validateCrc32c(expectedCrc32cBase64, targetStream.hash().asInt()); + } + } + } + + /** + * Validates a calculated CRC32C value against GCS's expected base64-encoded value. + * + * @throws IOException if the checksums do not match. + */ + public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) throws IOException { + if (expectedCrc32cBase64 == null) { + return; + } + byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64); + int expectedVal = Ints.fromByteArray(decoded); + + Crc32cValue expected = Crc32cValue.of(expectedVal, 0); + Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0); + + hasher.validate(expected, actual); + } + + /** + * Validates a downloaded raw byte array against GCS's expected base64-encoded value. + * + * @throws IOException if the checksums do not match. + */ + public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws IOException { + if (expectedCrc32cBase64 == null) { + return; + } + byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64); + int expectedVal = Ints.fromByteArray(decoded); + + Crc32cValue expected = Crc32cValue.of(expectedVal, 0); + hasher.validate( + expected, + new Supplier() { + @Override + public ByteBuffer get() { + return ByteBuffer.wrap(content); + } + }); + } +} diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 97814b597c37..e8e8b31c0e07 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -80,6 +80,7 @@ import com.google.cloud.Tuple; import com.google.cloud.http.CensusHttpModule; import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.storage.HttpStorageRpcHasherHelper; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.common.base.Function; @@ -860,9 +861,12 @@ public byte[] load(StorageObject from, Map options) { if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) { getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options)); } + HttpResponse response = getRequest.executeMedia(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - getRequest.executeMedia().download(out); - return out.toByteArray(); + response.download(out); + byte[] content = out.toByteArray(); + HttpStorageRpcHasherHelper.INSTANCE.validate(response, content); + return content; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); @@ -919,7 +923,18 @@ public long read( } MediaHttpDownloader mediaHttpDownloader = req.getMediaHttpDownloader(); mediaHttpDownloader.setDirectDownloadEnabled(true); - req.executeMedia().download(outputStream); + + // Check if this is a full object download (no Range header set) + boolean isFullObjectDownload = (req.getRequestHeaders().getRange() == null); + + OutputStream activeStream = + HttpStorageRpcHasherHelper.INSTANCE.wrap(outputStream, isFullObjectDownload); + + HttpResponse response = req.executeMedia(); + response.download(activeStream); + // Validate checksum + HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream); + return mediaHttpDownloader.getNumBytesDownloaded(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java new file mode 100644 index 000000000000..665fe44605f4 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.common.hash.Hashing; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import org.junit.Test; + +public class HttpStorageRpcHasherHelperTest { + + private static final byte[] CONTENT_BYTES = "Hello, World!".getBytes(); + private static final String CONTENT_CRC32C_BASE64 = + "TVUQaA=="; // expected CRC32C of "Hello, World!" + + @Test + public void testWrap_disabled_returnsOriginalStream() { + ByteArrayOutputStream original = new ByteArrayOutputStream(); + OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, false); + assertSame(original, wrapped); + } + + @Test + public void testWrap_enabled_returnsHashingStream() throws IOException { + ByteArrayOutputStream original = new ByteArrayOutputStream(); + OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, true); + assertNotEquals(original, wrapped); + + wrapped.write(CONTENT_BYTES); + wrapped.flush(); + wrapped.close(); + + byte[] writtenBytes = original.toByteArray(); + assertArrayEquals(CONTENT_BYTES, writtenBytes); + } + + @Test + public void testValidateCrc32c_int_expectSuccess() throws IOException { + int calculatedCrc32c = Hashing.crc32c().hashBytes(CONTENT_BYTES).asInt(); + // Should complete cleanly without throwing + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, calculatedCrc32c); + } + + @Test + public void testValidateCrc32c_int_expectMismatchFailure() { + int calculatedCrc32c = 12345; // Incorrect hash + Hasher.ChecksumMismatchException ex = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c( + CONTENT_CRC32C_BASE64, calculatedCrc32c)); + assertTrue(ex.getMessage().contains("Mismatch checksum value")); + } + + @Test + public void testValidateCrc32c_byteArray_expectSuccess() throws IOException { + // Should complete cleanly without throwing + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, CONTENT_BYTES); + } + + @Test + public void testValidateCrc32c_byteArray_expectMismatchFailure() { + byte[] wrongBytes = "Wrong bytes!".getBytes(); + Hasher.ChecksumMismatchException ex = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c( + CONTENT_CRC32C_BASE64, wrongBytes)); + assertTrue(ex.getMessage().contains("Mismatch checksum value")); + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java new file mode 100644 index 000000000000..aebd6c6f8e85 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.spi.v1; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.cloud.NoCredentials; +import com.google.cloud.TransportOptions; +import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableList; +import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public final class HttpStorageRpcChecksumTest { + + private static final String BUCKET = "test-bucket"; + private static final String BLOB = "test-blob"; + private static final byte[] CONTENT = + "hello world checksum test".getBytes(StandardCharsets.UTF_8); + + private static final int CONTENT_CRC32C_VAL = + com.google.common.hash.Hashing.crc32c().hashBytes(CONTENT).asInt(); + private static final String CONTENT_CRC32C_BASE64 = + BaseEncoding.base64().encode(Ints.toByteArray(CONTENT_CRC32C_VAL)); + + private static final String BAD_CRC32C_BASE64 = + BaseEncoding.base64().encode(Ints.toByteArray(999999)); + + @Test + public void testReadAllBytes_successfulCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + CONTENT_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + byte[] bytes = storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + assertThat(bytes).isEqualTo(CONTENT); + } + } + + @Test + public void testReadAllBytes_failedCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + BAD_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + StorageException ex = + assertThrows( + StorageException.class, + () -> { + storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + }); + assertThat(ex.getMessage()).contains("Mismatch checksum value"); + assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value"); + } + } + + @Test + public void testDownloadTo_successfulCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + CONTENT_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + assertThat(out.toByteArray()).isEqualTo(CONTENT); + } + } + + @Test + public void testDownloadTo_failedCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + BAD_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + StorageException ex = + assertThrows( + StorageException.class, + () -> { + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + }); + assertThat(ex.getMessage()).contains("Mismatch checksum value"); + assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value"); + } + } + + @Test + public void testReadAllBytes_noChecksumHeader_expectSuccess() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + byte[] bytes = storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + assertThat(bytes).isEqualTo(CONTENT); + } + } + + @Test + public void testDownloadTo_noChecksumHeader_expectSuccess() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + assertThat(out.toByteArray()).isEqualTo(CONTENT); + } + } + + private Storage createMockStorage(MockLowLevelHttpResponse response) { + AuditingHttpTransport transport = new AuditingHttpTransport(response); + TransportOptions transportOptions = + HttpTransportOptions.newBuilder().setHttpTransportFactory(() -> transport).build(); + return StorageOptions.getDefaultInstance().toBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setTransportOptions(transportOptions) + .build() + .getService(); + } +}