From 6c3a69614fffc8ddb02640533270c094fc205877 Mon Sep 17 00:00:00 2001 From: nidhiii-27 Date: Tue, 26 May 2026 12:14:00 +0530 Subject: [PATCH 1/3] feat(storage): add checksum validation on read paths --- .../java/com/google/cloud/storage/Hasher.java | 15 +- .../storage/HttpStorageRpcHasherHelper.java | 176 ++++++++++++++++++ .../cloud/storage/spi/v1/HttpStorageRpc.java | 21 ++- .../HttpStorageRpcHasherHelperTest.java | 94 ++++++++++ .../spi/v1/HttpStorageRpcChecksumTest.java | 170 +++++++++++++++++ 5 files changed, 472 insertions(+), 4 deletions(-) create mode 100644 java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java index c1b506de2f7e..97947e78563c 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java @@ -73,6 +73,8 @@ default Crc32cLengthKnown hash(Supplier b) { void validateUnchecked(Crc32cValue expected, ByteString byteString) throws UncheckedChecksumMismatchException; + void validate(Crc32cValue expected, Crc32cLengthKnown actual) throws ChecksumMismatchException; + @Nullable > C nullSafeConcat( @Nullable C r1, @Nullable Crc32cLengthKnown r2); @@ -122,6 +124,9 @@ public void validate(Crc32cValue expected, ByteString b) {} @Override public void validateUnchecked(Crc32cValue expected, ByteString byteString) {} + @Override + public void validate(Crc32cValue expected, Crc32cLengthKnown actual) {} + @Override public > @Nullable C nullSafeConcat( @Nullable C r1, @Nullable Crc32cLengthKnown r2) { @@ -189,6 +194,14 @@ public void validateUnchecked(Crc32cValue expected, ByteString byteString) } } + @Override + public void validate(Crc32cValue expected, Crc32cLengthKnown actual) + throws ChecksumMismatchException { + if (!actual.eqValue(expected)) { + throw new ChecksumMismatchException(expected, actual); + } + } + @SuppressWarnings("unchecked") @Override public > @Nullable C nullSafeConcat( @@ -212,7 +225,7 @@ final class ChecksumMismatchException extends IOException { private final Crc32cValue expected; private final Crc32cLengthKnown actual; - private ChecksumMismatchException(Crc32cValue expected, Crc32cLengthKnown actual) { + ChecksumMismatchException(Crc32cValue expected, Crc32cLengthKnown actual) { super( String.format( Locale.US, diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java new file mode 100644 index 000000000000..31de47c804da --- /dev/null +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java @@ -0,0 +1,176 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.client.http.HttpResponse; +import com.google.api.core.InternalApi; +import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Internal utility class to perform client-side CRC32C checksum validation on downloaded data + * specifically for the {@code HttpStorageRpc} transport layer. + * + *

Since this class resides in the {@code com.google.cloud.storage} package, it has full, + * package-private compile-time access to internal components (like {@link Hasher} and {@link + * Crc32cValue}) without leaking GCS internal types into public client API surfaces. + */ +@InternalApi +public final class HttpStorageRpcHasherHelper { + + public static final HttpStorageRpcHasherHelper INSTANCE = new HttpStorageRpcHasherHelper(); + + private final Hasher hasher; + + private HttpStorageRpcHasherHelper() { + hasher = Hasher.defaultHasher(); + } + + /** + * Returns a wrapping output stream that hashes the written content if validation is enabled, or + * the original output stream otherwise. + */ + public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) { + boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher); + return (isChecksumValidationEnabled && isHasherEnabled) + ? new Crc32cHashingOutputStream(out) + : out; + } + + /** + * Validates a raw byte array against GCS's expected base64-encoded value in response headers. + * + * @throws IOException if the checksums do not match. + */ + public void validate(HttpResponse response, byte[] content) throws IOException { + Map hashes = extractHashesFromHeader(response); + String expectedCrc32cBase64 = hashes.get("crc32c"); + if (expectedCrc32cBase64 != null) { + validateCrc32c(expectedCrc32cBase64, content); + } + } + + /** + * Validates the downloaded output stream against GCS's expected base64-encoded value in response + * headers. + * + * @throws IOException if the checksums do not match. + */ + public void validate(HttpResponse response, OutputStream activeStream) throws IOException { + if (activeStream instanceof Crc32cHashingOutputStream) { + Crc32cHashingOutputStream targetStream = (Crc32cHashingOutputStream) activeStream; + Map hashes = extractHashesFromHeader(response); + String expectedCrc32cBase64 = hashes.get("crc32c"); + if (expectedCrc32cBase64 != null) { + validateCrc32c(expectedCrc32cBase64, targetStream.hash()); + } + } + } + + /** + * Validates a calculated CRC32C value against GCS's expected base64-encoded value. + * + * @throws IOException if the checksums do not match. + */ + public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) throws IOException { + if (expectedCrc32cBase64 == null) { + return; + } + byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64); + int expectedVal = Ints.fromByteArray(decoded); + + Crc32cValue expected = Crc32cValue.of(expectedVal, 0); + Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0); + + // Invoke standard package-private validate path natively + System.out.println("validating checksum"); + hasher.validate(expected, actual); + } + + /** + * Validates a downloaded raw byte array against GCS's expected base64-encoded value. + * + * @throws IOException if the checksums do not match. + */ + public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws IOException { + if (expectedCrc32cBase64 == null) { + return; + } + byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64); + int expectedVal = Ints.fromByteArray(decoded); + + Crc32cValue expected = Crc32cValue.of(expectedVal, 0); + System.out.println("validating checksum"); + hasher.validate( + expected, + new Supplier() { + @Override + public ByteBuffer get() { + return ByteBuffer.wrap(content); + } + }); + } + + @SuppressWarnings("UnstableApiUsage") + private static class Crc32cHashingOutputStream extends java.io.FilterOutputStream { + private final com.google.common.hash.Hasher hasher; + + Crc32cHashingOutputStream(OutputStream out) { + super(out); + this.hasher = com.google.common.hash.Hashing.crc32c().newHasher(); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + hasher.putByte((byte) b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + hasher.putBytes(b, off, len); + } + + int hash() { + return hasher.hash().asInt(); + } + } + + private static Map extractHashesFromHeader(HttpResponse response) { + List hashHeaders = response.getHeaders().getHeaderStringValues("x-goog-hash"); + if (hashHeaders == null || hashHeaders.isEmpty()) { + return java.util.Collections.emptyMap(); + } + + return hashHeaders.stream() + .flatMap(h -> java.util.Arrays.stream(h.split(","))) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(s -> s.split("=", 2)) + .filter(a -> a.length == 2) + .filter(a -> "crc32c".equalsIgnoreCase(a[0]) || "md5".equalsIgnoreCase(a[0])) + .collect( + java.util.stream.Collectors.toMap(a -> a[0].toLowerCase(), a -> a[1], (v1, v2) -> v1)); + } +} diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 97814b597c37..e8e8b31c0e07 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -80,6 +80,7 @@ import com.google.cloud.Tuple; import com.google.cloud.http.CensusHttpModule; import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.storage.HttpStorageRpcHasherHelper; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.common.base.Function; @@ -860,9 +861,12 @@ public byte[] load(StorageObject from, Map options) { if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) { getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options)); } + HttpResponse response = getRequest.executeMedia(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - getRequest.executeMedia().download(out); - return out.toByteArray(); + response.download(out); + byte[] content = out.toByteArray(); + HttpStorageRpcHasherHelper.INSTANCE.validate(response, content); + return content; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); @@ -919,7 +923,18 @@ public long read( } MediaHttpDownloader mediaHttpDownloader = req.getMediaHttpDownloader(); mediaHttpDownloader.setDirectDownloadEnabled(true); - req.executeMedia().download(outputStream); + + // Check if this is a full object download (no Range header set) + boolean isFullObjectDownload = (req.getRequestHeaders().getRange() == null); + + OutputStream activeStream = + HttpStorageRpcHasherHelper.INSTANCE.wrap(outputStream, isFullObjectDownload); + + HttpResponse response = req.executeMedia(); + response.download(activeStream); + // Validate checksum + HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream); + return mediaHttpDownloader.getNumBytesDownloaded(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java new file mode 100644 index 000000000000..665fe44605f4 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/HttpStorageRpcHasherHelperTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.common.hash.Hashing; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import org.junit.Test; + +public class HttpStorageRpcHasherHelperTest { + + private static final byte[] CONTENT_BYTES = "Hello, World!".getBytes(); + private static final String CONTENT_CRC32C_BASE64 = + "TVUQaA=="; // expected CRC32C of "Hello, World!" + + @Test + public void testWrap_disabled_returnsOriginalStream() { + ByteArrayOutputStream original = new ByteArrayOutputStream(); + OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, false); + assertSame(original, wrapped); + } + + @Test + public void testWrap_enabled_returnsHashingStream() throws IOException { + ByteArrayOutputStream original = new ByteArrayOutputStream(); + OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, true); + assertNotEquals(original, wrapped); + + wrapped.write(CONTENT_BYTES); + wrapped.flush(); + wrapped.close(); + + byte[] writtenBytes = original.toByteArray(); + assertArrayEquals(CONTENT_BYTES, writtenBytes); + } + + @Test + public void testValidateCrc32c_int_expectSuccess() throws IOException { + int calculatedCrc32c = Hashing.crc32c().hashBytes(CONTENT_BYTES).asInt(); + // Should complete cleanly without throwing + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, calculatedCrc32c); + } + + @Test + public void testValidateCrc32c_int_expectMismatchFailure() { + int calculatedCrc32c = 12345; // Incorrect hash + Hasher.ChecksumMismatchException ex = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c( + CONTENT_CRC32C_BASE64, calculatedCrc32c)); + assertTrue(ex.getMessage().contains("Mismatch checksum value")); + } + + @Test + public void testValidateCrc32c_byteArray_expectSuccess() throws IOException { + // Should complete cleanly without throwing + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, CONTENT_BYTES); + } + + @Test + public void testValidateCrc32c_byteArray_expectMismatchFailure() { + byte[] wrongBytes = "Wrong bytes!".getBytes(); + Hasher.ChecksumMismatchException ex = + assertThrows( + Hasher.ChecksumMismatchException.class, + () -> + HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c( + CONTENT_CRC32C_BASE64, wrongBytes)); + assertTrue(ex.getMessage().contains("Mismatch checksum value")); + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java new file mode 100644 index 000000000000..aebd6c6f8e85 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/spi/v1/HttpStorageRpcChecksumTest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.spi.v1; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.cloud.NoCredentials; +import com.google.cloud.TransportOptions; +import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableList; +import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import org.junit.Test; + +public final class HttpStorageRpcChecksumTest { + + private static final String BUCKET = "test-bucket"; + private static final String BLOB = "test-blob"; + private static final byte[] CONTENT = + "hello world checksum test".getBytes(StandardCharsets.UTF_8); + + private static final int CONTENT_CRC32C_VAL = + com.google.common.hash.Hashing.crc32c().hashBytes(CONTENT).asInt(); + private static final String CONTENT_CRC32C_BASE64 = + BaseEncoding.base64().encode(Ints.toByteArray(CONTENT_CRC32C_VAL)); + + private static final String BAD_CRC32C_BASE64 = + BaseEncoding.base64().encode(Ints.toByteArray(999999)); + + @Test + public void testReadAllBytes_successfulCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + CONTENT_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + byte[] bytes = storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + assertThat(bytes).isEqualTo(CONTENT); + } + } + + @Test + public void testReadAllBytes_failedCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + BAD_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + StorageException ex = + assertThrows( + StorageException.class, + () -> { + storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + }); + assertThat(ex.getMessage()).contains("Mismatch checksum value"); + assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value"); + } + } + + @Test + public void testDownloadTo_successfulCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + CONTENT_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + assertThat(out.toByteArray()).isEqualTo(CONTENT); + } + } + + @Test + public void testDownloadTo_failedCrc32cValidation() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setHeaderNames(ImmutableList.of("x-goog-hash")) + .setHeaderValues(ImmutableList.of("crc32c=" + BAD_CRC32C_BASE64)) + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + StorageException ex = + assertThrows( + StorageException.class, + () -> { + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + }); + assertThat(ex.getMessage()).contains("Mismatch checksum value"); + assertThat(ex.getCause().getMessage()).contains("Mismatch checksum value"); + } + } + + @Test + public void testReadAllBytes_noChecksumHeader_expectSuccess() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + byte[] bytes = storage.readAllBytes(BlobId.of(BUCKET, BLOB)); + assertThat(bytes).isEqualTo(CONTENT); + } + } + + @Test + public void testDownloadTo_noChecksumHeader_expectSuccess() throws Exception { + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setContentType("application/octet-stream") + .setContent(new String(CONTENT, StandardCharsets.UTF_8)) + .setStatusCode(200); + + try (Storage storage = createMockStorage(response)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + storage.downloadTo(BlobId.of(BUCKET, BLOB), out); + assertThat(out.toByteArray()).isEqualTo(CONTENT); + } + } + + private Storage createMockStorage(MockLowLevelHttpResponse response) { + AuditingHttpTransport transport = new AuditingHttpTransport(response); + TransportOptions transportOptions = + HttpTransportOptions.newBuilder().setHttpTransportFactory(() -> transport).build(); + return StorageOptions.getDefaultInstance().toBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setTransportOptions(transportOptions) + .build() + .getService(); + } +} From b610455d784b89ac1a25518ac59df7b78796e210 Mon Sep 17 00:00:00 2001 From: nidhiii-27 Date: Tue, 26 May 2026 12:43:57 +0530 Subject: [PATCH 2/3] remove debug statements --- .../com/google/cloud/storage/HttpStorageRpcHasherHelper.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java index 31de47c804da..9c9cadc58400 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java @@ -102,8 +102,6 @@ public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) th Crc32cValue expected = Crc32cValue.of(expectedVal, 0); Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0); - // Invoke standard package-private validate path natively - System.out.println("validating checksum"); hasher.validate(expected, actual); } @@ -120,7 +118,6 @@ public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws I int expectedVal = Ints.fromByteArray(decoded); Crc32cValue expected = Crc32cValue.of(expectedVal, 0); - System.out.println("validating checksum"); hasher.validate( expected, new Supplier() { From 109008acbbf533c833bb59948220bb1416d337f9 Mon Sep 17 00:00:00 2001 From: nidhiii-27 Date: Tue, 26 May 2026 14:39:17 +0530 Subject: [PATCH 3/3] Use HashingOutputStream instead of custom Crc32cHashingOutputStream --- .../storage/HttpStorageRpcHasherHelper.java | 65 ++++--------------- 1 file changed, 11 insertions(+), 54 deletions(-) diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java index 9c9cadc58400..131cc161c007 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageRpcHasherHelper.java @@ -18,22 +18,19 @@ import com.google.api.client.http.HttpResponse; import com.google.api.core.InternalApi; +import com.google.common.hash.Hashing; +import com.google.common.hash.HashingOutputStream; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.List; import java.util.Map; import java.util.function.Supplier; /** * Internal utility class to perform client-side CRC32C checksum validation on downloaded data * specifically for the {@code HttpStorageRpc} transport layer. - * - *

Since this class resides in the {@code com.google.cloud.storage} package, it has full, - * package-private compile-time access to internal components (like {@link Hasher} and {@link - * Crc32cValue}) without leaking GCS internal types into public client API surfaces. */ @InternalApi public final class HttpStorageRpcHasherHelper { @@ -50,10 +47,11 @@ private HttpStorageRpcHasherHelper() { * Returns a wrapping output stream that hashes the written content if validation is enabled, or * the original output stream otherwise. */ + @SuppressWarnings("UnstableApiUsage") public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) { boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher); return (isChecksumValidationEnabled && isHasherEnabled) - ? new Crc32cHashingOutputStream(out) + ? new HashingOutputStream(Hashing.crc32c(), out) : out; } @@ -63,7 +61,7 @@ public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) * @throws IOException if the checksums do not match. */ public void validate(HttpResponse response, byte[] content) throws IOException { - Map hashes = extractHashesFromHeader(response); + Map hashes = ChecksumResponseParser.extractHashesFromHeader(response); String expectedCrc32cBase64 = hashes.get("crc32c"); if (expectedCrc32cBase64 != null) { validateCrc32c(expectedCrc32cBase64, content); @@ -76,13 +74,15 @@ public void validate(HttpResponse response, byte[] content) throws IOException { * * @throws IOException if the checksums do not match. */ + @SuppressWarnings("UnstableApiUsage") public void validate(HttpResponse response, OutputStream activeStream) throws IOException { - if (activeStream instanceof Crc32cHashingOutputStream) { - Crc32cHashingOutputStream targetStream = (Crc32cHashingOutputStream) activeStream; - Map hashes = extractHashesFromHeader(response); + if (activeStream instanceof HashingOutputStream) { + HashingOutputStream targetStream = (HashingOutputStream) activeStream; + + Map hashes = ChecksumResponseParser.extractHashesFromHeader(response); String expectedCrc32cBase64 = hashes.get("crc32c"); if (expectedCrc32cBase64 != null) { - validateCrc32c(expectedCrc32cBase64, targetStream.hash()); + validateCrc32c(expectedCrc32cBase64, targetStream.hash().asInt()); } } } @@ -127,47 +127,4 @@ public ByteBuffer get() { } }); } - - @SuppressWarnings("UnstableApiUsage") - private static class Crc32cHashingOutputStream extends java.io.FilterOutputStream { - private final com.google.common.hash.Hasher hasher; - - Crc32cHashingOutputStream(OutputStream out) { - super(out); - this.hasher = com.google.common.hash.Hashing.crc32c().newHasher(); - } - - @Override - public void write(int b) throws IOException { - out.write(b); - hasher.putByte((byte) b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - hasher.putBytes(b, off, len); - } - - int hash() { - return hasher.hash().asInt(); - } - } - - private static Map extractHashesFromHeader(HttpResponse response) { - List hashHeaders = response.getHeaders().getHeaderStringValues("x-goog-hash"); - if (hashHeaders == null || hashHeaders.isEmpty()) { - return java.util.Collections.emptyMap(); - } - - return hashHeaders.stream() - .flatMap(h -> java.util.Arrays.stream(h.split(","))) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .map(s -> s.split("=", 2)) - .filter(a -> a.length == 2) - .filter(a -> "crc32c".equalsIgnoreCase(a[0]) || "md5".equalsIgnoreCase(a[0])) - .collect( - java.util.stream.Collectors.toMap(a -> a[0].toLowerCase(), a -> a[1], (v1, v2) -> v1)); - } }