diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index f115b8b727..aca2222dd7 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -44,6 +44,12 @@ function _getPartKey(uploadId, splitter, paddedPartNumber) { return `${uploadId}${splitter}${paddedPartNumber}`; } +function checksumTypeMismatchErr(expected, actual) { + return errors.InvalidRequest.customizeDescription( + `Checksum Type mismatch occurred, expected checksum Type: ${expected}, ` + + `actual checksum Type: ${actual}`); +} + /** * PUT part of object during a multipart upload. Steps include: * validating metadata for authorization, bucket existence @@ -118,7 +124,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, const requestType = request.apiMethods || 'objectPutPart'; let partChecksum; let mpuChecksumAlgo; + let mpuChecksumType; let mpuChecksumIsDefault; + let clientSuppliedChecksum; return async.waterfall([ // Get the destination bucket. @@ -203,6 +211,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log, } mpuChecksumAlgo = res.checksumAlgorithm; + mpuChecksumType = res.checksumType; mpuChecksumIsDefault = res.checksumIsDefault; const objectLocationConstraint = @@ -330,16 +339,21 @@ function objectPutPart(authInfo, request, streamingV4Params, log, if (headerChecksum && headerChecksum.error) { return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); } + // Whether the client sent a per-part checksum header (vs. one the + // server computes implicitly). Drives whether we echo it back. + clientSuppliedChecksum = !!headerChecksum; // If the MPU specifies a non-default checksum algo and the // client sends a different algo, reject the request. if (headerChecksum && mpuChecksumAlgo && !mpuChecksumIsDefault && headerChecksum.algorithm !== mpuChecksumAlgo) { - return next(errors.InvalidRequest.customizeDescription( - `Checksum algorithm '${headerChecksum.algorithm}' is not the same ` + - `as the checksum algorithm '${mpuChecksumAlgo}' specified during ` + - 'CreateMultipartUpload.' - ), destinationBucket); + return next(checksumTypeMismatchErr(mpuChecksumAlgo, headerChecksum.algorithm), destinationBucket); + } + + // A COMPOSITE MPU's final checksum is composed from the per-part + // checksums, so every part must carry one. + if (!headerChecksum && mpuChecksumType === 'COMPOSITE') { + return next(checksumTypeMismatchErr(mpuChecksumAlgo, 'null'), destinationBucket); } const primaryAlgo = mpuChecksumAlgo || 'crc64nvme'; @@ -500,7 +514,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, 'putObjectPart'); return cb(err, null, corsHeaders); } - if (partChecksum) { + // Surface the part checksum unless it is the server-computed default. + // A client-supplied checksum, and any explicit-algorithm MPU, is still echoed - matching AWS. + if (partChecksum && (!mpuChecksumIsDefault || clientSuppliedChecksum)) { const { algorithm, value } = partChecksum; corsHeaders[`x-amz-checksum-${algorithm}`] = value; } diff --git a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js index 8ed43e9a80..e3b1fc7d64 100644 --- a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js @@ -184,6 +184,65 @@ describe('CompleteMultipartUpload final-object checksum', () => assert.strictEqual(head.ChecksumType, 'FULL_OBJECT'); }); + describe('SDK-style checksum forwarding', () => { + let fwdS3; + const checksumFields = [ + 'ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME', + 'ChecksumSHA1', 'ChecksumSHA256', + ]; + // explicit algorithms + the no-algorithm default (null) + const configs = ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256', null]; + + before(() => { + // WHEN_REQUIRED: the SDK sends a per-part checksum only when we + // explicitly provide one, and nothing for the default MPU. + fwdS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + }); + + configs.forEach(algo => { + const label = algo || 'no algorithm (default)'; + it(`should forward the UploadPart checksum and complete (${label})`, async () => { + const key = `complete-forward-${(algo || 'default').toLowerCase()}-${Date.now()}`; + const create = await fwdS3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, + ...(algo ? { ChecksumAlgorithm: algo } : {}), + })); + + const uploadParams = { + Bucket: bucket, Key: key, UploadId: create.UploadId, + PartNumber: 1, Body: partBody, + }; + // Explicit-algo MPUs require the matching per-part checksum. + if (algo) { + uploadParams[tagField(algo)] = await algorithms[algo.toLowerCase()].digest(partBody); + } + const uploadPart = await fwdS3.send(new UploadPartCommand(uploadParams)); + + // Forward whatever checksum the UploadPart response surfaced. + const completedPart = { PartNumber: 1, ETag: uploadPart.ETag }; + checksumFields.forEach(f => { + if (uploadPart[f] !== undefined) { + completedPart[f] = uploadPart[f]; + } + }); + + const complete = await fwdS3.send(new CompleteMultipartUploadCommand({ + Bucket: bucket, Key: key, UploadId: create.UploadId, + MultipartUpload: { Parts: [completedPart] }, + })); + const expectedField = algo ? tagField(algo) : 'ChecksumCRC64NVME'; + assert( + complete[expectedField], + `expected ${expectedField} on CompleteMPU response, got: ${JSON.stringify(complete)}`, + ); + }); + }); + }); + // AWS S3 rejects any per-part // Checksum field on a default MPU (one created without an // explicit ChecksumAlgorithm) with InvalidPart — even when the diff --git a/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js index 1bcf410912..8d250db59d 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js @@ -129,7 +129,16 @@ describe('UploadPart checksum validation', () => PartNumber: 3 + idx, Body: partBody, [checksumField[otherAlgo]]: correctDigest[otherAlgo], })), - { name: 'InvalidRequest' }, + err => { + assert.strictEqual(err.name, 'InvalidRequest', + `expected InvalidRequest, got ${err.name}: ${err.message}`); + // AWS names the expected (MPU) and actual (sent) algorithms. + assert(err.message.includes( + `expected checksum Type: ${mpuAlgo.toLowerCase()}, ` + + `actual checksum Type: ${otherAlgo.toLowerCase()}`), + `unexpected message: ${err.message}`); + return true; + }, ); }); }); @@ -175,12 +184,85 @@ describe('UploadPart checksum validation', () => }); }); - it('should accept part with no checksum header', async () => { - const res = await s3.send(new UploadPartCommand({ + it('should return no per-part checksum when none is sent', async () => { + // WHEN_REQUIRED so the SDK does not auto-attach a crc32: the + // part is genuinely uploaded with no checksum. + const noCksumS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + const res = await noCksumS3.send(new UploadPartCommand({ Bucket: bucket, Key: key, UploadId: uploadId, PartNumber: 2 * allAlgos.length + 1, Body: partBody, })); assert(res.ETag); + const present = ['ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME', + 'ChecksumSHA1', 'ChecksumSHA256'].filter(f => res[f] !== undefined); + assert.deepStrictEqual(present, [], + `default MPU UploadPart should return no checksum, got: ${present.join(', ')}`); + }); + }); + + describe('per-part checksum requirement by checksum type', () => { + // WHEN_REQUIRED so the SDK does not auto-attach a checksum, letting + // us upload a genuinely checksum-less part. + let noCksumS3; + const openUploads = []; + + before(() => { + noCksumS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + }); + + after(async () => { + await Promise.all(openUploads.map(uploadId => + noCksumS3.send(new AbortMultipartUploadCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + })).catch(() => undefined))); + }); + + async function createMpu(algo, type) { + const res = await noCksumS3.send(new CreateMultipartUploadCommand({ + Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type, + })); + openUploads.push(res.UploadId); + return res.UploadId; + } + + ['CRC32', 'CRC32C', 'SHA1', 'SHA256'].forEach(algo => { + it(`should reject UploadPart with no checksum on a ${algo}/COMPOSITE MPU`, async () => { + const uploadId = await createMpu(algo, 'COMPOSITE'); + await assert.rejects( + noCksumS3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: 1, Body: partBody, + })), + err => { + assert.strictEqual(err.name, 'InvalidRequest', + `expected InvalidRequest, got ${err.name}: ${err.message}`); + assert(err.message.includes(`expected checksum Type: ${algo.toLowerCase()}`), + `unexpected message: ${err.message}`); + return true; + }, + ); + }); + }); + + ['CRC32', 'CRC32C', 'CRC64NVME'].forEach(algo => { + it(`should accept UploadPart with no checksum on a ${algo}/FULL_OBJECT MPU`, async () => { + const uploadId = await createMpu(algo, 'FULL_OBJECT'); + const res = await noCksumS3.send(new UploadPartCommand({ + Bucket: bucket, Key: key, UploadId: uploadId, + PartNumber: 1, Body: partBody, + })); + assert(res.ETag); + assert(res[`Checksum${algo}`], + `expected Checksum${algo} echoed, got: ${JSON.stringify(res)}`); + }); }); }); }) diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 855eb3ebe8..570b15208c 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -154,7 +154,7 @@ let crc64nvmeOfTrailerContent; // Create the common protocol-scenario tests for a given URL factory. // urlFn() is called lazily at test runtime so that uploadId is available. -function makeScenarioTests(urlFn) { +function makeScenarioTests(urlFn, { expectsImplicitChecksum = true } = {}) { before(async () => { if (!crc64nvmeOfTestContent2) { crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2); @@ -164,6 +164,18 @@ function makeScenarioTests(urlFn) { } }); + // When no client checksum is sent, PutObject echoes the server-computed + // default crc64nvme, but a default-MPU UploadPart does not (matching AWS). + function assertImplicitChecksum(res, expected) { + if (expectsImplicitChecksum) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expected, + `expected x-amz-checksum-crc64nvme: ${expected}`); + } else { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], undefined, + 'default-MPU UploadPart should not echo an implicit checksum'); + } + } + itSkipIfAWS( 'should return 200 for signed sha256 in x-amz-content-sha256, no x-amz-checksum header', done => { @@ -172,8 +184,7 @@ function makeScenarioTests(urlFn) { 'content-length': testContent2.length, }, testContent2, (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`); + assertImplicitChecksum(res, crc64nvmeOfTestContent2); done(); }); }); @@ -459,8 +470,7 @@ function makeScenarioTests(urlFn) { 'content-length': Buffer.byteLength(body), }, body, (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + assertImplicitChecksum(res, crc64nvmeOfTrailerContent); done(); }); }); @@ -479,8 +489,7 @@ function makeScenarioTests(urlFn) { 'content-length': Buffer.byteLength(body), }, body, (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + assertImplicitChecksum(res, crc64nvmeOfTrailerContent); done(); }); }); @@ -731,7 +740,8 @@ describe('UploadPart: trailer and checksum protocol scenarios', () => { }); makeScenarioTests( - () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` + () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}`, + { expectsImplicitChecksum: false }, ); }); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index c3a24cfb7b..e101c5315b 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -3431,17 +3431,6 @@ describe('objectPutPart checksum response headers', () => { done(); }); }); - - it('should return x-amz-checksum-crc64nvme response header when no checksum header is provided', done => { - const expectedCrc64nvme = '5evlCr2wyO4='; - const partRequest = _createPutPartRequest(testUploadId, '1', postBody); - - objectPutPart(authInfo, partRequest, undefined, log, (err, _hexDigest, resHeaders) => { - assert.ifError(err); - assert.strictEqual(resHeaders['x-amz-checksum-crc64nvme'], expectedCrc64nvme); - done(); - }); - }); }); describe('initiateMultipartUpload checksum headers', () => { diff --git a/tests/unit/api/objectPutPartChecksum.js b/tests/unit/api/objectPutPartChecksum.js index f518ca4885..e09e2884fe 100644 --- a/tests/unit/api/objectPutPartChecksum.js +++ b/tests/unit/api/objectPutPartChecksum.js @@ -123,11 +123,28 @@ describe('objectPutPart checksum validation', () => { }); }); - it('should accept part with no checksum on non-default MPU', done => { + it('should reject part with no checksum on a COMPOSITE MPU', done => { + // sha256 is COMPOSITE-only; a COMPOSITE MPU's final checksum is + // composed from the per-part checksums, so every part must carry + // one and AWS rejects a part sent without it. initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { assert.ifError(err); // No checksum header sent const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, err => { + assert(err, 'Expected an error'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + }); + + it('should accept part with no checksum on a FULL_OBJECT MPU', done => { + // crc64nvme is FULL_OBJECT-only; the server computes the + // full-object checksum, so a missing per-part checksum is allowed. + initiateMPU({ 'x-amz-checksum-algorithm': 'crc64nvme' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); objectPutPart(authInfo, request, undefined, log, err => { assert.ifError(err); done(); @@ -298,4 +315,62 @@ describe('objectPutPart checksum validation', () => { }); }); }); + + describe('response checksum header', () => { + const algos = Object.keys(algorithms); + + it('should not return a checksum header on a default MPU when none is sent', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + algos.forEach(algo => { + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], undefined); + }); + // The part checksum is still stored so CompleteMPU can + // compute the final object checksum. + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert(partMD.checksumValue); + done(); + }); + }); + }); + + algos.forEach(algo => { + it(`should echo a client-supplied ${algo} checksum on a default MPU`, done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + Promise.resolve(algorithms[algo].digest(partBody)).then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + [`x-amz-checksum-${algo}`]: digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], digest); + done(); + }); + }).catch(done); + }); + }); + + it(`should echo the ${algo} checksum on an explicit ${algo} MPU`, done => { + initiateMPU({ 'x-amz-checksum-algorithm': algo }, (err, uploadId) => { + assert.ifError(err); + Promise.resolve(algorithms[algo].digest(partBody)).then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + [`x-amz-checksum-${algo}`]: digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], digest); + done(); + }); + }).catch(done); + }); + }); + }); + }); });