Skip to content

HDDS-15304. DeleteObjects should enforce the request key limit #10309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sreejasahithi wants to merge 1 commit into
base: master
Choose a base branch
from
Open

HDDS-15304. DeleteObjects should enforce the request key limit #10309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.ContinueToken;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.util.Time;
Expand Down Expand Up @@ -341,6 +342,11 @@ public MultiDeleteResponse multiDelete(
) throws OS3Exception, IOException {
S3GAction s3GAction = S3GAction.MULTI_DELETE;

if (request.getObjects() != null
&& request.getObjects().size() > S3Consts.S3_DELETE_OBJECTS_MAX_KEYS) {
throw newError(S3ErrorTable.MALFORMED_XML, bucketName);
Copy link
Copy Markdown
Member

@peterxcli peterxcli May 19, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should throw clear exception

Suggested change
throw newError(S3ErrorTable.MALFORMED_XML, bucketName);
throw newError(S3ErrorTable.INVALID_ARGUMENT, "too many objects");

}

OzoneBucket bucket = getVolume().getBucket(bucketName);
MultiDeleteResponse result = new MultiDeleteResponse();
List<String> deleteKeys = new ArrayList<>();
Expand Down
3 changes: 3 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ public final class S3Consts {
public static final Pattern TAG_REGEX_PATTERN = Pattern.compile("^([\\p{L}\\p{Z}\\p{N}_.:/=+\\-]*)$");
public static final String MP_PARTS_COUNT = "x-amz-mp-parts-count";

/** AWS S3 maximum number of keys per DeleteObjects request. */
public static final int S3_DELETE_OBJECTS_MAX_KEYS = 1000;

// Bucket owner condition headers
// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-owner-condition.html
public static final String EXPECTED_BUCKET_OWNER_HEADER = "x-amz-expected-bucket-owner";
Expand Down
42 changes: 42 additions & 0 deletions ...ne/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@

package org.apache.hadoop.ozone.s3.endpoint;

import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.util.Collections.singleton;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;

import com.google.common.collect.Sets;
import java.io.IOException;
Expand All @@ -32,6 +34,7 @@
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.Test;

/**
Expand Down Expand Up @@ -101,6 +104,45 @@ public void deleteQuiet() throws IOException, OS3Exception, JAXBException {
assertEquals(0, response.getErrors().size());
}

@Test
public void multiDeleteRejectsMoreThanMaxKeysPerRequest() throws Exception {
OzoneClient client = new OzoneClientStub();
BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder()
.setClient(client)
.build();

MultiDeleteRequest mdr = new MultiDeleteRequest();
for (int i = 0; i < S3Consts.S3_DELETE_OBJECTS_MAX_KEYS + 1; i++) {
mdr.getObjects().add(new DeleteObject("key-" + i));
}

OS3Exception ex = assertThrows(OS3Exception.class,
() -> rest.multiDelete("b1", "", mdr));
assertEquals("MalformedXML", ex.getCode());
assertEquals(HTTP_BAD_REQUEST, ex.getHttpCode());
}

@Test
public void multiDeleteAllowsMaxKeysPerRequest() throws Exception {
OzoneClient client = new OzoneClientStub();
OzoneBucket bucket = initTestData(client);
BucketEndpoint rest = EndpointBuilder.newBucketEndpointBuilder()
.setClient(client)
.build();

MultiDeleteRequest mdr = new MultiDeleteRequest();
mdr.setQuiet(true);
for (int i = 0; i < S3Consts.S3_DELETE_OBJECTS_MAX_KEYS; i++) {
mdr.getObjects().add(new DeleteObject("missing-" + i));
}

MultiDeleteResponse response = rest.multiDelete("b1", "", mdr);
assertEquals(0, response.getDeletedObjects().size());
assertEquals(0, response.getErrors().size());

assertEquals(3, Sets.newHashSet(bucket.listKeys("")).size());
}

private OzoneBucket initTestData(OzoneClient client) throws IOException {
client.getObjectStore().createS3Bucket("b1");

Expand Down