Skip to content

Add pagination for long queries in CustomControllerClients #1809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
theghost5800 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add pagination for long queries in CustomControllerClients #1809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0437da1
Add pagination for long queries in CustomControllerClients
theghost5800 Apr 1, 2026
769c5fa
Fix comments 1
theghost5800 Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions ...in/java/org/cloudfoundry/multiapps/controller/core/cf/clients/CustomControllerClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

public abstract class CustomControllerClient {

protected static final int MAX_URI_QUERY_LENGTH = 4000;

private final WebClient webClient;
private String correlationId = StringUtils.EMPTY;
private final CloudControllerHeaderConfiguration headerConfiguration;
Expand Down Expand Up @@ -61,6 +63,74 @@ private MultiValueMap<String, String> generateRequestHeaders() {
return result;
}

protected <T> List<T> getListOfResourcesInBatches(ResourcesResponseMapper<T> responseMapper, String uriPrefix,
List<String> batchValues, Object... urlVariables) {
int expandedPrefixLength = calculateExpandedLength(uriPrefix, urlVariables);
List<List<String>> batches = splitIntoBatches(batchValues, expandedPrefixLength);
return batches.stream()
.map(batch ->
getListOfResources(responseMapper, uriPrefix, batch, urlVariables)
)
.flatMap(List::stream)
.toList();
}

private int calculateExpandedLength(String uriPrefix, Object[] urlVariables) {
if (urlVariables == null || urlVariables.length == 0) {
return uriPrefix.length();
}
int urlVariablesLength = 0;
for (Object variable : urlVariables) {
urlVariablesLength += String.valueOf(variable)
.length();
}
int placeholdersLength = calculatePlaceholdersLength(uriPrefix);
return (uriPrefix.length() - placeholdersLength) + urlVariablesLength;
Comment on lines +78 to +88
Copy link
Copy Markdown
Contributor

@IvanBorislavovDimitrov IvanBorislavovDimitrov Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should consider first to URL encode the string then measure the length. Because if urlVariable contains for example cyrillic the length of the string will actually be more because cyrillic becomes 6 chars (%D1%81% -> я).

}

private int calculatePlaceholdersLength(String uriPrefix) {
int length = 0;
int searchFrom = 0;
int start;
int end;
while ((start = uriPrefix.indexOf('{', searchFrom)) >= 0 && (end = uriPrefix.indexOf('}', start)) >= 0) {
length += end - start + 1;
searchFrom = end + 1;
}
return length;
}

protected List<List<String>> splitIntoBatches(List<String> values, int fixedUriLength) {
int maxBatchLength = Math.max(1, MAX_URI_QUERY_LENGTH - fixedUriLength);
List<List<String>> batches = new ArrayList<>();
List<String> currentBatch = new ArrayList<>();
int currentLength = 0;

for (String value : values) {
// Account for the comma separator between values
int addedLength = currentBatch.isEmpty() ? value.length() : value.length() + 1;
Copy link
Copy Markdown
Contributor

@IvanBorislavovDimitrov IvanBorislavovDimitrov Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is comma 1 symbol or should be count as "%2C"?

if (!currentBatch.isEmpty() && currentLength + addedLength > maxBatchLength) {
batches.add(currentBatch);
currentBatch = new ArrayList<>();
currentLength = 0;
addedLength = value.length();
}
currentBatch.add(value);
currentLength += addedLength;
}

if (!currentBatch.isEmpty()) {
batches.add(currentBatch);
}
return batches;
}

private <T> List<T> getListOfResources(ResourcesResponseMapper<T> responseMapper, String uriPrefix,
List<String> batch, Object... urlVariables) {
String uri = uriPrefix + String.join(",", batch);
return getListOfResources(responseMapper, uri, urlVariables);
}

public static abstract class ResourcesResponseMapper<T> {
List<Map<String, Object>> queriedResources = new ArrayList<>();
Map<String, List<Object>> includedResources = new HashMap<>();
Expand Down
28 changes: 7 additions & 21 deletions ...n/java/org/cloudfoundry/multiapps/controller/core/cf/clients/CustomServiceKeysClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@
import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;

public class CustomServiceKeysClient extends CustomControllerClient {

private static final String SERVICE_KEYS_RESOURCE_BASE_URI = "/v3/service_credential_bindings";
private static final String SERVICE_KEYS_BY_METADATA_SELECTOR_URI = SERVICE_KEYS_RESOURCE_BASE_URI + "?type=key&label_selector={value}";
private static final String INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM = "&include=service_instance";

private static final String SERVICE_INSTANCE_GUIDS_PARAM_PREFIX = "&service_instance_guids=";
private final CloudEntityResourceMapper resourceMapper = new CloudEntityResourceMapper();

public CustomServiceKeysClient(ApplicationConfiguration configuration, WebClientFactory webClientFactory, CloudCredentials credentials,
Expand All @@ -35,17 +34,13 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndExistingGuids(
String mtaId,
String mtaNamespace,
List<String> existingServiceGuids) {

String labelSelector = buildMtaMetadataLabelSelector(spaceGuid, mtaId, mtaNamespace);

List<String> allServiceGuids = existingServiceGuids.stream()
.filter(Objects::nonNull)
.toList();

if (allServiceGuids.isEmpty()) {
return List.of();
}

return new CustomControllerClientErrorHandler()
.handleErrorsOrReturnResult(
() -> getServiceKeysByMetadataInternal(labelSelector, allServiceGuids)
Expand All @@ -57,15 +52,11 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndManagedServices(
String mtaId,
String mtaNamespace,
List<DeployedMtaService> services) {

String labelSelector = buildMtaMetadataLabelSelector(spaceGuid, mtaId, mtaNamespace);

List<String> managedGuids = extractManagedServiceGuids(services);

if (managedGuids.isEmpty()) {
return List.of();
}

return new CustomControllerClientErrorHandler()
.handleErrorsOrReturnResult(
() -> getServiceKeysByMetadataInternal(labelSelector, managedGuids)
Expand All @@ -75,7 +66,6 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndManagedServices(
private String buildMtaMetadataLabelSelector(String spaceGuid,
String mtaId,
String mtaNamespace) {

return MtaMetadataCriteriaBuilder.builder()
.label(MtaMetadataLabels.SPACE_GUID)
.hasValue(spaceGuid)
Expand All @@ -98,13 +88,11 @@ private List<String> extractManagedServiceGuids(List<DeployedMtaService> service
}

private List<DeployedMtaServiceKey> getServiceKeysByMetadataInternal(String labelSelector, List<String> guids) {

String uriSuffix = INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM
+ "&service_instance_guids=" + String.join(",", guids);

return getListOfResources(new ServiceKeysResponseMapper(),
SERVICE_KEYS_BY_METADATA_SELECTOR_URI + uriSuffix,
labelSelector);
String expandedUriPrefix =
SERVICE_KEYS_BY_METADATA_SELECTOR_URI + INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM + SERVICE_INSTANCE_GUIDS_PARAM_PREFIX;
return getListOfResourcesInBatches(new ServiceKeysResponseMapper(),
expandedUriPrefix,
guids, labelSelector);
}

private List<DeployedMtaService> getManagedServices(List<DeployedMtaService> services) {
Expand Down Expand Up @@ -132,12 +120,10 @@ public List<DeployedMtaServiceKey> getMappedResources() {

public Map<String, CloudServiceInstance> getIncludedServiceInstancesMapping() {
List<Object> serviceInstances = getIncludedResources().getOrDefault("service_instances", Collections.emptyList());

return serviceInstances.stream()
.distinct()
.map(service -> (Map<String, Object>) service)
.collect(Collectors.toMap(service -> (String) service.get("guid"), resourceMapper::mapService));

}
}
}
}
42 changes: 7 additions & 35 deletions ...va/org/cloudfoundry/multiapps/controller/core/cf/clients/ServiceInstanceRoutesGetter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package org.cloudfoundry.multiapps.controller.core.cf.clients;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
Expand All @@ -13,48 +11,22 @@

public class ServiceInstanceRoutesGetter extends CustomControllerClient {

private static final int MAX_CHAR_LENGTH_FOR_PARAMS_IN_REQUEST = 4000;
private static final String SERVICE_ROUTE_BINDINGS_URI_PREFIX = "/v3/service_route_bindings?";
private static final String ROUTE_GUIDS_PARAM_PREFIX = "route_guids=";

public ServiceInstanceRoutesGetter(ApplicationConfiguration configuration, WebClientFactory webClientFactory,
CloudCredentials credentials, String correlationId) {
super(configuration, webClientFactory, credentials, correlationId);
}

public List<ServiceRouteBinding> getServiceRouteBindings(Collection<String> routeGuids) {
public List<ServiceRouteBinding> getServiceRouteBindings(List<String> routeGuids) {
return new CustomControllerClientErrorHandler().handleErrorsOrReturnResult(() -> doGetServiceRouteBindings(routeGuids));
}

private List<ServiceRouteBinding> doGetServiceRouteBindings(Collection<String> routeGuids) {
var batchedRouteGuids = getBatchedRouteGuids(routeGuids);
var responseMapper = new ServiceRouteBindingsResponseMapper();
return batchedRouteGuids.stream()
.map(this::getServiceRouteBindingsUrl)
.map(url -> getListOfResources(responseMapper, url))
.flatMap(List::stream)
.collect(Collectors.toList());
}

private List<List<String>> getBatchedRouteGuids(Collection<String> routeGuids) {
List<List<String>> batches = new ArrayList<>();
int currentBatchLength = 0, currentBatchIndex = 0;
batches.add(new ArrayList<>());

for (String routeGuid : routeGuids) {
int elementLength = routeGuid.length();
if (elementLength + currentBatchLength >= MAX_CHAR_LENGTH_FOR_PARAMS_IN_REQUEST) {
batches.add(new ArrayList<>());
currentBatchIndex++;
currentBatchLength = 0;
}
batches.get(currentBatchIndex)
.add(routeGuid);
currentBatchLength += elementLength;
}
return batches;
}

private String getServiceRouteBindingsUrl(Collection<String> routeGuids) {
return "/v3/service_route_bindings?route_guids=" + String.join(",", routeGuids);
private List<ServiceRouteBinding> doGetServiceRouteBindings(List<String> routeGuids) {
return getListOfResourcesInBatches(new ServiceRouteBindingsResponseMapper(),
SERVICE_ROUTE_BINDINGS_URI_PREFIX + ROUTE_GUIDS_PARAM_PREFIX,
routeGuids);
}

protected static class ServiceRouteBindingsResponseMapper extends ResourcesResponseMapper<ServiceRouteBinding> {
Expand Down
Loading
Loading