Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ default Crc32cLengthKnown hash(Supplier<ByteBuffer> b) {
void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
throws UncheckedChecksumMismatchException;

void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) throws ChecksumMismatchException;

@Nullable <C extends Crc32cValue<?>> C nullSafeConcat(
@Nullable C r1, @Nullable Crc32cLengthKnown r2);

Expand Down Expand Up @@ -122,6 +124,9 @@ public void validate(Crc32cValue<?> expected, ByteString b) {}
@Override
public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString) {}

@Override
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual) {}

@Override
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(
@Nullable C r1, @Nullable Crc32cLengthKnown r2) {
Expand Down Expand Up @@ -189,6 +194,14 @@ public void validateUnchecked(Crc32cValue<?> expected, ByteString byteString)
}
}

@Override
public void validate(Crc32cValue<?> expected, Crc32cLengthKnown actual)
throws ChecksumMismatchException {
if (!actual.eqValue(expected)) {
throw new ChecksumMismatchException(expected, actual);
}
}

@SuppressWarnings("unchecked")
@Override
public <C extends Crc32cValue<?>> @Nullable C nullSafeConcat(
Expand All @@ -212,7 +225,7 @@ final class ChecksumMismatchException extends IOException {
private final Crc32cValue<?> expected;
private final Crc32cLengthKnown actual;

private ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
ChecksumMismatchException(Crc32cValue<?> expected, Crc32cLengthKnown actual) {
super(
String.format(
Locale.US,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.storage;

import com.google.api.client.http.HttpResponse;
import com.google.api.core.InternalApi;
import com.google.common.hash.Hashing;
import com.google.common.hash.HashingOutputStream;
import com.google.common.io.BaseEncoding;
import com.google.common.primitives.Ints;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.function.Supplier;

/**
* Internal utility class to perform client-side CRC32C checksum validation on downloaded data
* specifically for the {@code HttpStorageRpc} transport layer.
*/
@InternalApi
public final class HttpStorageRpcHasherHelper {

public static final HttpStorageRpcHasherHelper INSTANCE = new HttpStorageRpcHasherHelper();

private final Hasher hasher;

private HttpStorageRpcHasherHelper() {
hasher = Hasher.defaultHasher();
}

/**
* Returns a wrapping output stream that hashes the written content if validation is enabled, or
* the original output stream otherwise.
*/
@SuppressWarnings("UnstableApiUsage")
public OutputStream wrap(OutputStream out, boolean isChecksumValidationEnabled) {
boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher);
return (isChecksumValidationEnabled && isHasherEnabled)
? new HashingOutputStream(Hashing.crc32c(), out)
: out;
}

/**
* Validates a raw byte array against GCS's expected base64-encoded value in response headers.
*
* @throws IOException if the checksums do not match.
*/
public void validate(HttpResponse response, byte[] content) throws IOException {
Map<String, String> hashes = ChecksumResponseParser.extractHashesFromHeader(response);
String expectedCrc32cBase64 = hashes.get("crc32c");
if (expectedCrc32cBase64 != null) {
validateCrc32c(expectedCrc32cBase64, content);
}
}

/**
* Validates the downloaded output stream against GCS's expected base64-encoded value in response
* headers.
*
* @throws IOException if the checksums do not match.
*/
@SuppressWarnings("UnstableApiUsage")
public void validate(HttpResponse response, OutputStream activeStream) throws IOException {
if (activeStream instanceof HashingOutputStream) {
HashingOutputStream targetStream = (HashingOutputStream) activeStream;

Map<String, String> hashes = ChecksumResponseParser.extractHashesFromHeader(response);
String expectedCrc32cBase64 = hashes.get("crc32c");
if (expectedCrc32cBase64 != null) {
validateCrc32c(expectedCrc32cBase64, targetStream.hash().asInt());
}
}
}

/**
* Validates a calculated CRC32C value against GCS's expected base64-encoded value.
*
* @throws IOException if the checksums do not match.
*/
public void validateCrc32c(String expectedCrc32cBase64, int calculatedCrc32c) throws IOException {
if (expectedCrc32cBase64 == null) {
return;
}
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
int expectedVal = Ints.fromByteArray(decoded);

Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
Crc32cValue.Crc32cLengthKnown actual = Crc32cValue.of(calculatedCrc32c, 0);

hasher.validate(expected, actual);
}

/**
* Validates a downloaded raw byte array against GCS's expected base64-encoded value.
*
* @throws IOException if the checksums do not match.
*/
public void validateCrc32c(String expectedCrc32cBase64, byte[] content) throws IOException {
if (expectedCrc32cBase64 == null) {
return;
}
byte[] decoded = BaseEncoding.base64().decode(expectedCrc32cBase64);
int expectedVal = Ints.fromByteArray(decoded);

Crc32cValue<?> expected = Crc32cValue.of(expectedVal, 0);
hasher.validate(
expected,
new Supplier<ByteBuffer>() {
@Override
public ByteBuffer get() {
return ByteBuffer.wrap(content);
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
import com.google.cloud.Tuple;
import com.google.cloud.http.CensusHttpModule;
import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.storage.HttpStorageRpcHasherHelper;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
import com.google.common.base.Function;
Expand Down Expand Up @@ -860,9 +861,12 @@ public byte[] load(StorageObject from, Map<Option, ?> options) {
if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) {
getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options));
}
HttpResponse response = getRequest.executeMedia();
ByteArrayOutputStream out = new ByteArrayOutputStream();
getRequest.executeMedia().download(out);
return out.toByteArray();
response.download(out);
byte[] content = out.toByteArray();
HttpStorageRpcHasherHelper.INSTANCE.validate(response, content);
return content;
} catch (IOException ex) {
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
throw translate(ex);
Expand Down Expand Up @@ -919,7 +923,18 @@ public long read(
}
MediaHttpDownloader mediaHttpDownloader = req.getMediaHttpDownloader();
mediaHttpDownloader.setDirectDownloadEnabled(true);
req.executeMedia().download(outputStream);

// Check if this is a full object download (no Range header set)
boolean isFullObjectDownload = (req.getRequestHeaders().getRange() == null);

OutputStream activeStream =
HttpStorageRpcHasherHelper.INSTANCE.wrap(outputStream, isFullObjectDownload);

HttpResponse response = req.executeMedia();
response.download(activeStream);
// Validate checksum
HttpStorageRpcHasherHelper.INSTANCE.validate(response, activeStream);

return mediaHttpDownloader.getNumBytesDownloaded();
} catch (IOException ex) {
span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright 2026 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.storage;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;

import com.google.common.hash.Hashing;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import org.junit.Test;

public class HttpStorageRpcHasherHelperTest {

private static final byte[] CONTENT_BYTES = "Hello, World!".getBytes();
private static final String CONTENT_CRC32C_BASE64 =
"TVUQaA=="; // expected CRC32C of "Hello, World!"

@Test
public void testWrap_disabled_returnsOriginalStream() {
ByteArrayOutputStream original = new ByteArrayOutputStream();
OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, false);
assertSame(original, wrapped);
}

@Test
public void testWrap_enabled_returnsHashingStream() throws IOException {
ByteArrayOutputStream original = new ByteArrayOutputStream();
OutputStream wrapped = HttpStorageRpcHasherHelper.INSTANCE.wrap(original, true);
assertNotEquals(original, wrapped);

wrapped.write(CONTENT_BYTES);
wrapped.flush();
wrapped.close();

byte[] writtenBytes = original.toByteArray();
assertArrayEquals(CONTENT_BYTES, writtenBytes);
}

@Test
public void testValidateCrc32c_int_expectSuccess() throws IOException {
int calculatedCrc32c = Hashing.crc32c().hashBytes(CONTENT_BYTES).asInt();
// Should complete cleanly without throwing
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, calculatedCrc32c);
}

@Test
public void testValidateCrc32c_int_expectMismatchFailure() {
int calculatedCrc32c = 12345; // Incorrect hash
Hasher.ChecksumMismatchException ex =
assertThrows(
Hasher.ChecksumMismatchException.class,
() ->
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(
CONTENT_CRC32C_BASE64, calculatedCrc32c));
assertTrue(ex.getMessage().contains("Mismatch checksum value"));
}

@Test
public void testValidateCrc32c_byteArray_expectSuccess() throws IOException {
// Should complete cleanly without throwing
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(CONTENT_CRC32C_BASE64, CONTENT_BYTES);
}

@Test
public void testValidateCrc32c_byteArray_expectMismatchFailure() {
byte[] wrongBytes = "Wrong bytes!".getBytes();
Hasher.ChecksumMismatchException ex =
assertThrows(
Hasher.ChecksumMismatchException.class,
() ->
HttpStorageRpcHasherHelper.INSTANCE.validateCrc32c(
CONTENT_CRC32C_BASE64, wrongBytes));
assertTrue(ex.getMessage().contains("Mismatch checksum value"));
}
}
Loading
Loading