Storage Content Validation - adding fuzzy tests for content validation decoder

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "java",
   "TagPrefix": "java/storage/azure-storage-blob",
-  "Tag": "java/storage/azure-storage-blob_beac25b9c7"
+  "Tag": "java/storage/azure-storage-blob_dc7e5d385e"
 }

sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java

@@ -14,6 +14,7 @@
 import com.azure.storage.blob.options.BlobDownloadContentOptions;
 import com.azure.storage.blob.options.BlobDownloadStreamOptions;
 import com.azure.storage.blob.options.BlobDownloadToFileOptions;
+import com.azure.storage.blob.options.BlobUploadFromFileOptions;
 import com.azure.storage.common.ParallelTransferOptions;
 import com.azure.storage.common.ContentValidationAlgorithm;
 import com.azure.storage.common.implementation.Constants;
@@ -23,6 +24,8 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 import reactor.core.publisher.Flux;
 import reactor.test.StepVerifier;
@@ -34,10 +37,12 @@
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicLong;
 
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -50,6 +55,23 @@
 public class BlobContentValidationAsyncDownloadTests extends BlobTestBase {
     private static final int TEN_MB = 10 * Constants.MB;
     private static final int BLOCK_SIZE = 4 * Constants.MB;
+    /**
+     * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy
+     * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full
+     * payload never lives twice in heap.
+     */
+    private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB;
+
+    /**
+     * Live-only random payload band for the dedicated random-size parallel-download fuzzy test
+     * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run
+     * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message
+     * decoder is exercised against payloads whose size varies per run in addition to the random byte contents.
+     */
+    private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB;
+    private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB;
+
+    private final List<File> createdFiles = new ArrayList<>();
 
     private File createRandomFile(Path tempDir, int size) throws IOException {
         File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile();
@@ -499,4 +521,161 @@ long getReportedByteCount() {
             return this.reportedByteCount.get();
         }
     }
+
+    // ---------- Fuzzy parallel download (deterministic grids) ----------
+
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases")
+    public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("replayable", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases")
+    public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases")
+    public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases")
+    public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases")
+    public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // payload >> blockSize; ~96-320 MiB downloads.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases")
+    public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    @LiveOnly // ~1 GiB single case; far too large for the test proxy.
+    @ParameterizedTest
+    @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases")
+    public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency)
+        throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency);
+    }
+
+    /**
+     * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in
+     * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO
+     * content-validation algorithms so the structured-message decoder is tested against payloads whose total size
+     * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept
+     * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the
+     * deterministic per-grid round-trips and the randomized round-trip don't share work or cost.
+     */
+    @LiveOnly
+    @ParameterizedTest
+    @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" })
+    public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException {
+        int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1,
+            LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1);
+        assertParallelDownloadFuzzyRoundTripAsync("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm);
+    }
+
+    private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes,
+        int maxConcurrency) throws IOException {
+        assertParallelDownloadFuzzyRoundTripAsync(caseKind, payloadBytes, blockSizeBytes, maxConcurrency,
+            ContentValidationAlgorithm.CRC64);
+    }
+
+    private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes,
+        int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException {
+        List<HttpHeaders> recorded = new CopyOnWriteArrayList<>();
+        BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded);
+
+        ParallelTransferOptions parallelOptions
+            = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency);
+
+        String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes
+            + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm;
+
+        if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) {
+            File sourceFile = getRandomFile(payloadBytes);
+            sourceFile.deleteOnExit();
+            createdFiles.add(sourceFile);
+            File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile();
+            outFile.deleteOnExit();
+            createdFiles.add(outFile);
+            Files.deleteIfExists(outFile.toPath());
+
+            BlobUploadFromFileOptions uploadOptions
+                = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions(
+                    new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes)
+                        .setMaxConcurrency(maxConcurrency));
+            assertNotNull(client.uploadFromFileWithResponse(uploadOptions).block().getValue().getETag(),
+                assertionMessage);
+
+            BlobDownloadToFileOptions downloadOptions
+                = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions)
+                    .setContentValidationAlgorithm(algorithm);
+
+            StepVerifier.create(client.downloadToFileWithResponse(downloadOptions))
+                .assertNext(r -> assertNotNull(r.getValue(), assertionMessage))
+                .verifyComplete();
+
+            assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage);
+        } else {
+            byte[] randomData = getRandomByteArray(payloadBytes);
+            client.upload(BinaryData.fromBytes(randomData), true).block();
+
+            if (payloadBytes > blockSizeBytes) {
+                File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async-mp", ".bin").toFile();
+                outFile.deleteOnExit();
+                createdFiles.add(outFile);
+                Files.deleteIfExists(outFile.toPath());
+
+                BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString())
+                    .setParallelTransferOptions(parallelOptions)
+                    .setContentValidationAlgorithm(algorithm);
+
+                StepVerifier.create(client.downloadToFileWithResponse(downloadOptions))
+                    .assertNext(r -> assertNotNull(r.getValue(), assertionMessage))
+                    .verifyComplete();
+
+                byte[] downloaded = Files.readAllBytes(outFile.toPath());
+                assertArrayEquals(randomData, downloaded, assertionMessage);
+            } else {
+                BlobDownloadContentOptions downloadOptions
+                    = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm);
+
+                StepVerifier.create(client.downloadContentWithResponse(downloadOptions))
+                    .assertNext(r -> assertArrayEquals(randomData, r.getValue().toBytes(), assertionMessage))
+                    .verifyComplete();
+
+                BlobDownloadStreamOptions streamOptions
+                    = new BlobDownloadStreamOptions().setContentValidationAlgorithm(algorithm);
+                StepVerifier
+                    .create(client.downloadStreamWithResponse(streamOptions)
+                        .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())))
+                    .assertNext(bytes -> assertArrayEquals(randomData, bytes, assertionMessage))
+                    .verifyComplete();
+            }
+        }
+        assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage);
+    }
+
 }