CLDSRV-925: UploadPartCopy handle checksums

lib/api/objectPutPart.js

@@ -44,6 +44,12 @@ function _getPartKey(uploadId, splitter, paddedPartNumber) {
     return `${uploadId}${splitter}${paddedPartNumber}`;
 }
 
+function checksumTypeMismatchErr(expected, actual) {
+    return errors.InvalidRequest.customizeDescription(
+        `Checksum Type mismatch occurred, expected checksum Type: ${expected}, ` +
+        `actual checksum Type: ${actual}`);
+}
+
 /**
  * PUT part of object during a multipart upload. Steps include:
  * validating metadata for authorization, bucket existence
@@ -118,7 +124,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
     const requestType = request.apiMethods || 'objectPutPart';
     let partChecksum;
     let mpuChecksumAlgo;
+    let mpuChecksumType;
     let mpuChecksumIsDefault;
+    let clientSuppliedChecksum;
 
     return async.waterfall([
         // Get the destination bucket.
@@ -203,6 +211,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
                     }
 
                     mpuChecksumAlgo = res.checksumAlgorithm;
+                    mpuChecksumType = res.checksumType;
                     mpuChecksumIsDefault = res.checksumIsDefault;
 
                     const objectLocationConstraint =
@@ -330,16 +339,21 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
             if (headerChecksum && headerChecksum.error) {
                 return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket);
             }
+            // Whether the client sent a per-part checksum header (vs. one the
+            // server computes implicitly). Drives whether we echo it back.
+            clientSuppliedChecksum = !!headerChecksum;
 
             // If the MPU specifies a non-default checksum algo and the
             // client sends a different algo, reject the request.
             if (headerChecksum && mpuChecksumAlgo && !mpuChecksumIsDefault
                 && headerChecksum.algorithm !== mpuChecksumAlgo) {
-                return next(errors.InvalidRequest.customizeDescription(
-                    `Checksum algorithm '${headerChecksum.algorithm}' is not the same ` +
-                    `as the checksum algorithm '${mpuChecksumAlgo}' specified during ` +
-                    'CreateMultipartUpload.'
-                ), destinationBucket);
+                return next(checksumTypeMismatchErr(mpuChecksumAlgo, headerChecksum.algorithm), destinationBucket);
+            }
+
+            // A COMPOSITE MPU's final checksum is composed from the per-part
+            // checksums, so every part must carry one.
+            if (!headerChecksum && mpuChecksumType === 'COMPOSITE') {
+                return next(checksumTypeMismatchErr(mpuChecksumAlgo, 'null'), destinationBucket);
             }
 
             const primaryAlgo = mpuChecksumAlgo || 'crc64nvme';
@@ -500,7 +514,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log,
                 'putObjectPart');
             return cb(err, null, corsHeaders);
         }
-        if (partChecksum) {
+        // Surface the part checksum unless it is the server-computed default.
+        // A client-supplied checksum, and any explicit-algorithm MPU, is still echoed - matching AWS.
+        if (partChecksum && (!mpuChecksumIsDefault || clientSuppliedChecksum)) {
             const { algorithm, value } = partChecksum;
             corsHeaders[`x-amz-checksum-${algorithm}`] = value;
         }

tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js

@@ -184,6 +184,65 @@ describe('CompleteMultipartUpload final-object checksum', () =>
             assert.strictEqual(head.ChecksumType, 'FULL_OBJECT');
         });
 
+        describe('SDK-style checksum forwarding', () => {
+            let fwdS3;
+            const checksumFields = [
+                'ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME',
+                'ChecksumSHA1', 'ChecksumSHA256',
+            ];
+            // explicit algorithms + the no-algorithm default (null)
+            const configs = ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256', null];
+
+            before(() => {
+                // WHEN_REQUIRED: the SDK sends a per-part checksum only when we
+                // explicitly provide one, and nothing for the default MPU.
+                fwdS3 = new BucketUtility('default', {
+                    ...sigCfg,
+                    requestChecksumCalculation: 'WHEN_REQUIRED',
+                    responseChecksumValidation: 'WHEN_REQUIRED',
+                }).s3;
+            });
+
+            configs.forEach(algo => {
+                const label = algo || 'no algorithm (default)';
+                it(`should forward the UploadPart checksum and complete (${label})`, async () => {
+                    const key = `complete-forward-${(algo || 'default').toLowerCase()}-${Date.now()}`;
+                    const create = await fwdS3.send(new CreateMultipartUploadCommand({
+                        Bucket: bucket, Key: key,
+                        ...(algo ? { ChecksumAlgorithm: algo } : {}),
+                    }));
+
+                    const uploadParams = {
+                        Bucket: bucket, Key: key, UploadId: create.UploadId,
+                        PartNumber: 1, Body: partBody,
+                    };
+                    // Explicit-algo MPUs require the matching per-part checksum.
+                    if (algo) {
+                        uploadParams[tagField(algo)] = await algorithms[algo.toLowerCase()].digest(partBody);
+                    }
+                    const uploadPart = await fwdS3.send(new UploadPartCommand(uploadParams));
+
+                    // Forward whatever checksum the UploadPart response surfaced.
+                    const completedPart = { PartNumber: 1, ETag: uploadPart.ETag };
+                    checksumFields.forEach(f => {
+                        if (uploadPart[f] !== undefined) {
+                            completedPart[f] = uploadPart[f];
+                        }
+                    });
+
+                    const complete = await fwdS3.send(new CompleteMultipartUploadCommand({
+                        Bucket: bucket, Key: key, UploadId: create.UploadId,
+                        MultipartUpload: { Parts: [completedPart] },
+                    }));
+                    const expectedField = algo ? tagField(algo) : 'ChecksumCRC64NVME';
+                    assert(
+                        complete[expectedField],
+                        `expected ${expectedField} on CompleteMPU response, got: ${JSON.stringify(complete)}`,
+                    );
+                });
+            });
+        });
+
         // AWS S3 rejects any per-part
         // Checksum<X> field on a default MPU (one created without an
         // explicit ChecksumAlgorithm) with InvalidPart — even when the

tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js

@@ -129,7 +129,16 @@ describe('UploadPart checksum validation', () =>
                                 PartNumber: 3 + idx, Body: partBody,
                                 [checksumField[otherAlgo]]: correctDigest[otherAlgo],
                             })),
-                            { name: 'InvalidRequest' },
+                            err => {
+                                assert.strictEqual(err.name, 'InvalidRequest',
+                                    `expected InvalidRequest, got ${err.name}: ${err.message}`);
+                                // AWS names the expected (MPU) and actual (sent) algorithms.
+                                assert(err.message.includes(
+                                    `expected checksum Type: ${mpuAlgo.toLowerCase()}, ` +
+                                    `actual checksum Type: ${otherAlgo.toLowerCase()}`),
+                                    `unexpected message: ${err.message}`);
+                                return true;
+                            },
                         );
                     });
                 });
@@ -175,12 +184,85 @@ describe('UploadPart checksum validation', () =>
                 });
             });
 
-            it('should accept part with no checksum header', async () => {
-                const res = await s3.send(new UploadPartCommand({
+            it('should return no per-part checksum when none is sent', async () => {
+                // WHEN_REQUIRED so the SDK does not auto-attach a crc32: the
+                // part is genuinely uploaded with no checksum.
+                const noCksumS3 = new BucketUtility('default', {
+                    ...sigCfg,
+                    requestChecksumCalculation: 'WHEN_REQUIRED',
+                    responseChecksumValidation: 'WHEN_REQUIRED',
+                }).s3;
+                const res = await noCksumS3.send(new UploadPartCommand({
                     Bucket: bucket, Key: key, UploadId: uploadId,
                     PartNumber: 2 * allAlgos.length + 1, Body: partBody,
                 }));
                 assert(res.ETag);
+                const present = ['ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME',
+                    'ChecksumSHA1', 'ChecksumSHA256'].filter(f => res[f] !== undefined);
+                assert.deepStrictEqual(present, [],
+                    `default MPU UploadPart should return no checksum, got: ${present.join(', ')}`);
+            });
+        });
+
+        describe('per-part checksum requirement by checksum type', () => {
+            // WHEN_REQUIRED so the SDK does not auto-attach a checksum, letting
+            // us upload a genuinely checksum-less part.
+            let noCksumS3;
+            const openUploads = [];
+
+            before(() => {
+                noCksumS3 = new BucketUtility('default', {
+                    ...sigCfg,
+                    requestChecksumCalculation: 'WHEN_REQUIRED',
+                    responseChecksumValidation: 'WHEN_REQUIRED',
+                }).s3;
+            });
+
+            after(async () => {
+                await Promise.all(openUploads.map(uploadId =>
+                    noCksumS3.send(new AbortMultipartUploadCommand({
+                        Bucket: bucket, Key: key, UploadId: uploadId,
+                    })).catch(() => undefined)));
+            });
+
+            async function createMpu(algo, type) {
+                const res = await noCksumS3.send(new CreateMultipartUploadCommand({
+                    Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type,
+                }));
+                openUploads.push(res.UploadId);
+                return res.UploadId;
+            }
+
+            ['CRC32', 'CRC32C', 'SHA1', 'SHA256'].forEach(algo => {
+                it(`should reject UploadPart with no checksum on a ${algo}/COMPOSITE MPU`, async () => {
+                    const uploadId = await createMpu(algo, 'COMPOSITE');
+                    await assert.rejects(
+                        noCksumS3.send(new UploadPartCommand({
+                            Bucket: bucket, Key: key, UploadId: uploadId,
+                            PartNumber: 1, Body: partBody,
+                        })),
+                        err => {
+                            assert.strictEqual(err.name, 'InvalidRequest',
+                                `expected InvalidRequest, got ${err.name}: ${err.message}`);
+                            assert(err.message.includes(`expected checksum Type: ${algo.toLowerCase()}`),
+                                `unexpected message: ${err.message}`);
+                            return true;
+                        },
+                    );
+                });
+            });
+
+            ['CRC32', 'CRC32C', 'CRC64NVME'].forEach(algo => {
+                it(`should accept UploadPart with no checksum on a ${algo}/FULL_OBJECT MPU`, async () => {
+                    const uploadId = await createMpu(algo, 'FULL_OBJECT');
+                    const res = await noCksumS3.send(new UploadPartCommand({
+                        Bucket: bucket, Key: key, UploadId: uploadId,
+                        PartNumber: 1, Body: partBody,
+                    }));
+                    assert(res.ETag);
+                    assert(res[`Checksum${algo}`],
+                        `expected Checksum${algo} echoed, got: ${JSON.stringify(res)}`);
+                });
             });
         });
     })

tests/functional/raw-node/test/checksumPutObjectUploadPart.js

@@ -154,7 +154,7 @@ let crc64nvmeOfTrailerContent;
 
 // Create the common protocol-scenario tests for a given URL factory.
 // urlFn() is called lazily at test runtime so that uploadId is available.
-function makeScenarioTests(urlFn) {
+function makeScenarioTests(urlFn, { expectsImplicitChecksum = true } = {}) {
     before(async () => {
         if (!crc64nvmeOfTestContent2) {
             crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2);
@@ -164,6 +164,18 @@ function makeScenarioTests(urlFn) {
         }
     });
 
+    // When no client checksum is sent, PutObject echoes the server-computed
+    // default crc64nvme, but a default-MPU UploadPart does not (matching AWS).
+    function assertImplicitChecksum(res, expected) {
+        if (expectsImplicitChecksum) {
+            assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expected,
+                `expected x-amz-checksum-crc64nvme: ${expected}`);
+        } else {
+            assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], undefined,
+                'default-MPU UploadPart should not echo an implicit checksum');
+        }
+    }
+
     itSkipIfAWS(
         'should return 200 for signed sha256 in x-amz-content-sha256, no x-amz-checksum header',
         done => {
@@ -172,8 +184,7 @@ function makeScenarioTests(urlFn) {
                 'content-length': testContent2.length,
             }, testContent2, (err, res) => {
                 assertStatus(200)(err, res, () => {
-                    assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2,
-                        `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`);
+                    assertImplicitChecksum(res, crc64nvmeOfTestContent2);
                     done();
                 });
             });
@@ -459,8 +470,7 @@ function makeScenarioTests(urlFn) {
                 'content-length': Buffer.byteLength(body),
             }, body, (err, res) => {
                 assertStatus(200)(err, res, () => {
-                    assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent,
-                        `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`);
+                    assertImplicitChecksum(res, crc64nvmeOfTrailerContent);
                     done();
                 });
             });
@@ -479,8 +489,7 @@ function makeScenarioTests(urlFn) {
                 'content-length': Buffer.byteLength(body),
             }, body, (err, res) => {
                 assertStatus(200)(err, res, () => {
-                    assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent,
-                        `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`);
+                    assertImplicitChecksum(res, crc64nvmeOfTrailerContent);
                     done();
                 });
             });
@@ -731,7 +740,8 @@ describe('UploadPart: trailer and checksum protocol scenarios', () => {
     });
 
     makeScenarioTests(
-        () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}`
+        () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}`,
+        { expectsImplicitChecksum: false },
     );
 });
 

tests/unit/api/multipartUpload.js

@@ -3431,17 +3431,6 @@ describe('objectPutPart checksum response headers', () => {
             done();
         });
     });
-
-    it('should return x-amz-checksum-crc64nvme response header when no checksum header is provided', done => {
-        const expectedCrc64nvme = '5evlCr2wyO4=';
-        const partRequest = _createPutPartRequest(testUploadId, '1', postBody);
-
-        objectPutPart(authInfo, partRequest, undefined, log, (err, _hexDigest, resHeaders) => {
-            assert.ifError(err);
-            assert.strictEqual(resHeaders['x-amz-checksum-crc64nvme'], expectedCrc64nvme);
-            done();
-        });
-    });
 });
 
 describe('initiateMultipartUpload checksum headers', () => {