diff --git a/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml b/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml new file mode 100644 index 000000000000..33c359b108e1 --- /dev/null +++ b/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml @@ -0,0 +1,11 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Deprecate SortingMergePolicyFactory in favor of configuring the index sort directly with + in +type: changed +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-12230 + url: https://issues.apache.org/jira/browse/SOLR-12230 diff --git a/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml new file mode 100644 index 000000000000..fde7d60839bd --- /dev/null +++ b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml @@ -0,0 +1,10 @@ +title: > + New RESORTINDEX core-admin action re-sorts an existing core's index into a configured sort order, + so index sorting can be enabled on a collection created without it, without deleting and reindexing + from source. Not supported in SolrCloud mode; indexes with child/nested documents are rejected. +type: added +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-12239 + url: https://issues.apache.org/jira/browse/SOLR-12239 diff --git a/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml b/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml new file mode 100644 index 000000000000..4f15ec70f8f4 --- /dev/null +++ b/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Allow configuring Lucene's index sort directly with an element in , + instead of indirectly through a SortingMergePolicyFactory +type: added +authors: + - name: Serhiy Bzhezytskyy + - name: Christine Poerschke +links: + - name: SOLR-13681 + url: https://issues.apache.org/jira/browse/SOLR-13681 diff --git a/changelog/unreleased/SOLR-15390-native-early-termination.yml b/changelog/unreleased/SOLR-15390-native-early-termination.yml new file mode 100644 index 000000000000..64779af6a13f --- /dev/null +++ b/changelog/unreleased/SOLR-15390-native-early-termination.yml @@ -0,0 +1,11 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Use Lucene's native TopFieldCollector early termination for the segmentTerminateEarly query + parameter, replacing the deprecated EarlyTerminatingSortingCollector +type: changed +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-15390 + url: https://issues.apache.org/jira/browse/SOLR-15390 diff --git a/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml new file mode 100644 index 000000000000..a04533eb6a83 --- /dev/null +++ b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml @@ -0,0 +1,13 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Add a index configuration option that orders segments (leaf readers) at search + time -- by creation time (TIME_ASC/TIME_DESC) or by a numeric docValues field (" asc|desc") + -- exposing Lucene's leaf sorter for early termination +type: added +authors: + - name: Serhiy Bzhezytskyy + - name: Wei Wang +links: + - name: SOLR-17310 + url: https://issues.apache.org/jira/browse/SOLR-17310 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java new file mode 100644 index 000000000000..6a2521f993a8 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; + +/** V2 API definition for re-sorting an existing core index (SOLR-12239). */ +@Path("/cores/{coreName}/resort") +public interface ResortCoreIndexApi { + + @POST + @Operation( + summary = + "Re-sort the existing index of the specified core into the target sort order, so index " + + "sorting can be enabled without reindexing from source.", + tags = {"cores"}) + ResortCoreIndexResponse resortCoreIndex( + @Parameter(description = "The name of the core whose index should be re-sorted") + @PathParam("coreName") + String coreName, + ResortCoreIndexRequestBody requestBody) + throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java new file mode 100644 index 000000000000..6df9919f2526 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** Request body for the {@code RESORTINDEX} core-admin action. */ +public class ResortCoreIndexRequestBody { + + @Schema( + description = + "The target index sort, in Solr sort syntax (e.g. 'timestamp desc'). If omitted, the " + + "sort configured for the core is used: the element if present, " + + "otherwise a (deprecated) SortingMergePolicy sort. Fields must have docValues.") + @JsonProperty + public String sort; + + @Schema(description = "Request ID to track this action which will be processed asynchronously.") + @JsonProperty + public String async; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java new file mode 100644 index 000000000000..a978210f05f4 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** Response for the {@code RESORTINDEX} core-admin action. */ +public class ResortCoreIndexResponse extends SolrJerseyResponse { + + @Schema(description = "The name of the core whose index was re-sorted.") + @JsonProperty + public String core; + + @Schema(description = "The index sort that was applied.") + @JsonProperty + public String indexSort; +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 494e3f898414..de2d6e7c7019 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -69,6 +69,7 @@ import org.apache.solr.handler.admin.api.RequestBufferUpdatesAPI; import org.apache.solr.handler.admin.api.RequestCoreRecoveryAPI; import org.apache.solr.handler.admin.api.RequestSyncShardAPI; +import org.apache.solr.handler.admin.api.ResortCoreIndex; import org.apache.solr.handler.admin.api.RestoreCore; import org.apache.solr.handler.admin.api.SplitCoreAPI; import org.apache.solr.handler.admin.api.SwapCores; @@ -339,6 +340,7 @@ public Collection> getJerseyResources() { SwapCores.class, RenameCore.class, MergeIndexes.class, + ResortCoreIndex.class, GetNodeCommandStatus.class); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 2edc70aef30b..5213e3027898 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -33,6 +33,7 @@ import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTRECOVERY; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSTATUS; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSYNCSHARD; +import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESORTINDEX; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESTORECORE; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SPLIT; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.STATUS; @@ -258,7 +259,8 @@ public enum CoreAdminOperation implements CoreAdminOp { V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); }), - UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()); + UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()), + RESORTINDEX_OP(RESORTINDEX, new ResortIndexOp()); final CoreAdminParams.CoreAdminAction action; final CoreAdminOp fun; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java new file mode 100644 index 000000000000..3a4c0efc2651 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.api.ResortCoreIndex; +import org.apache.solr.handler.api.V2ApiUtils; + +/** + * v1 ({@code action=RESORTINDEX}) wrapper delegating to the {@link ResortCoreIndex} V2 API + * (SOLR-12239). + */ +class ResortIndexOp implements CoreAdminHandler.CoreAdminOp { + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public void execute(CoreAdminHandler.CallInfo it) throws Exception { + final SolrParams params = it.req.getParams(); + final String cname = params.required().get(CoreAdminParams.CORE); + final var requestBody = new ResortCoreIndexRequestBody(); + // "async" is intentionally omitted because CoreAdminHandler has already processed it. + requestBody.sort = params.get(CommonParams.SORT); + + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final ResortCoreIndex api = + new ResortCoreIndex(coreContainer, it.handler.getCoreAdminAsyncTracker(), it.req, it.rsp); + final ResortCoreIndexResponse response = api.resortCoreIndex(cname, requestBody); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java new file mode 100644 index 000000000000..3d3babf8b6a4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SlowCodecReaderWrapper; +import org.apache.lucene.index.SortingCodecReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.Directory; +import org.apache.solr.client.api.endpoint.ResortCoreIndexApi; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.IndexFetcher; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortSpecParsing; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * V2 API implementation for re-sorting an existing core index (SOLR-12239). + * + *

Re-sorts a (possibly unsorted) core index into the target {@link Sort} using the LUCENE-9484 + * mechanism — each segment reader is wrapped in a {@link SortingCodecReader} and merged into a + * fresh sort-configured {@link IndexWriter} via {@link IndexWriter#addIndexes(CodecReader...)} — + * then swaps it in via {@code modifyIndexProps} and reopens the writer/searcher (mirroring + * RestoreCore). + * + *

Not supported in SolrCloud mode. Indexes with child/nested documents are rejected. + */ +public class ResortCoreIndex extends CoreAdminAPIBase implements ResortCoreIndexApi { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ResortCoreIndex( + CoreContainer coreContainer, + CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker, + SolrQueryRequest req, + SolrQueryResponse rsp) { + super(coreContainer, coreAdminAsyncTracker, req, rsp); + } + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public ResortCoreIndexResponse resortCoreIndex( + String coreName, ResortCoreIndexRequestBody requestBody) throws Exception { + ensureRequiredParameterProvided("coreName", coreName); + if (coreContainer.isZooKeeperAware()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "resort is not supported in SolrCloud mode"); + } + final ResortCoreIndexResponse response = + instantiateJerseyResponse(ResortCoreIndexResponse.class); + final String sortParam = requestBody == null ? null : requestBody.sort; + final String async = requestBody == null ? null : requestBody.async; + + return handlePotentiallyAsynchronousTask( + response, + coreName, + async, + "resort-index", + () -> { + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Core not found: " + coreName); + } + final Sort indexSort = resolveTargetSort(core, sortParam); + assertNoChildDocs(core); + resortAndSwap(core, indexSort); + response.core = coreName; + response.indexSort = indexSort.toString(); + } catch (Exception e) { + throw new CoreAdminAPIBaseException(e); + } + return response; + }); + } + + private Sort resolveTargetSort(SolrCore core, String sortParam) throws IOException { + if (sortParam != null && !sortParam.isBlank()) { + final Sort sort = SortSpecParsing.parseSortSpec(sortParam, core.getLatestSchema()).getSort(); + if (sort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not parse a usable index sort from sort=" + sortParam); + } + return sort; + } + // Fall back to the sort configured for the core: the directly configured + // (preferred), or a SortingMergePolicy's sort (deprecated). This mirrors how the searcher + // resolves the index sort, so the migration workflow "set , then RESORTINDEX" works. + final String configuredSpec = core.getSolrConfig().indexConfig.indexSort; + if (configuredSpec != null && !configuredSpec.isBlank()) { + final Sort sort = + SortSpecParsing.parseSortSpec(configuredSpec, core.getLatestSchema()).getSort(); + if (sort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not parse a usable index sort from the configured =" + + configuredSpec); + } + return sort; + } + final Sort mergePolicySort = core.getSolrCoreState().getMergePolicySort(); + if (mergePolicySort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No 'sort' parameter given and the core has no configured index sort " + + "(via or a SortingMergePolicy) to fall back to"); + } + return mergePolicySort; + } + + private void assertNoChildDocs(SolrCore core) throws IOException { + RefCounted ref = core.getSearcher(); + try { + SolrIndexSearcher searcher = ref.get(); + if (!searcher.getSchema().isUsableForChildDocs()) { + return; + } + String uniqueKeyField = searcher.getSchema().getUniqueKeyField().getName(); + for (LeafReaderContext leaf : searcher.getIndexReader().leaves()) { + Terms rootTerms = leaf.reader().terms(IndexSchema.ROOT_FIELD_NAME); + if (rootTerms == null) { + continue; + } + long uniqueRootValues = rootTerms.size(); + Terms idTerms = leaf.reader().terms(uniqueKeyField); + long uniqueIdValues = (idTerms != null) ? idTerms.size() : -1; + if (uniqueRootValues == -1 || uniqueIdValues == -1 || uniqueRootValues < uniqueIdValues) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "resort does not support indexes containing child/nested documents. " + + "Consider reindexing your data from the original source."); + } + } + } finally { + ref.decref(); + } + } + + private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { + final String resortIndexName = "resort." + System.nanoTime(); + final String resortIndexPath = core.getDataDir() + resortIndexName; + final String currentIndexPath = core.getIndexDir(); + final DirectoryFactory df = core.getDirectoryFactory(); + final String lockType = core.getSolrConfig().indexConfig.lockType; + + Directory currentDir = null; + Directory resortDir = null; + try { + currentDir = df.get(currentIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + resortDir = df.get(resortIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + + final IndexWriterConfig iwc = new IndexWriterConfig().setIndexSort(indexSort); + // addIndexes rewrites every segment through this writer, so it must use the core's codec and + // similarity; otherwise a per-field (e.g. SchemaCodecFactory) index would be silently + // rewritten with Lucene defaults. + iwc.setCodec(core.getCodec()); + iwc.setSimilarity(core.getLatestSchema().getSimilarity()); + // Mirror SolrIndexConfig: a child-doc-capable schema records a parent field, which the source + // segments carry, so the re-sort writer must declare the same parent field or addIndexes + // fails. + if (core.getLatestSchema().isUsableForChildDocs()) { + iwc.setParentField(IndexSchema.IS_ROOT_FIELD_NAME); + } + + try (DirectoryReader reader = DirectoryReader.open(currentDir); + IndexWriter resortWriter = new IndexWriter(resortDir, iwc)) { + final List wrapped = new ArrayList<>(reader.leaves().size()); + for (LeafReaderContext ctx : reader.leaves()) { + // addIndexes does NOT auto-sort (LUCENE-8505); the SortingCodecReader wrap performs the + // (merge-based) re-sort of each segment. + wrapped.add( + SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(ctx.reader()), indexSort)); + } + resortWriter.addIndexes(wrapped.toArray(new CodecReader[0])); + resortWriter.commit(); + } + } finally { + if (currentDir != null) df.release(currentDir); + if (resortDir != null) df.release(resortDir); + } + + if (!core.modifyIndexProps(resortIndexName)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to point core " + core.getName() + " at the re-sorted index"); + } + try { + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + // The core now points at the re-sorted index; drop the previous (unsorted) index directory. + core.cleanupOldIndexDirectories(false); + } catch (Exception e) { + log.warn("Could not switch to re-sorted index for core {}; rolling back", core.getName(), e); + rollbackIndexProps(core, df, lockType); + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + // Discard the orphaned re-sort directory we just wrote but never swapped in. + removeDirectory(df, resortIndexPath, lockType); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to swap in the re-sorted index for core " + core.getName(), + e); + } + } + + private void removeDirectory(DirectoryFactory df, String path, String lockType) { + try { + if (df.exists(path)) { + Directory dir = df.get(path, DirectoryFactory.DirContext.DEFAULT, lockType); + try { + df.remove(dir, true); + } finally { + df.release(dir); + } + } + } catch (Exception cleanupError) { + log.warn("Could not remove re-sort directory {}", path, cleanupError); + } + } + + private void rollbackIndexProps(SolrCore core, DirectoryFactory df, String lockType) { + Directory dataDir = null; + try { + dataDir = df.get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA, lockType); + dataDir.deleteFile(IndexFetcher.INDEX_PROPERTIES); + } catch (Exception rollbackError) { + log.error("Rollback of index.properties failed for core {}", core.getName(), rollbackError); + } finally { + if (dataDir != null) { + try { + df.release(dataDir); + } catch (IOException ignored) { + } + } + } + } + + private void openNewSearcher(SolrCore core) throws Exception { + final Future[] waitSearcher = new Future[1]; + core.getSearcher(true, false, waitSearcher, true); + if (waitSearcher[0] != null) { + waitSearcher[0].get(); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java b/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java index 3b648921a42d..7e8fa82101c0 100644 --- a/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java +++ b/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java @@ -20,9 +20,14 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.search.Sort; -// TODO: remove this and add indexSort specification directly to solrconfig.xml? But for BWC, also -// accept SortingMergePolicy specifiction? - +/** + * A {@link MergePolicy} that does no merging of its own and only carries a {@link Sort} to the + * index writer. + * + * @deprecated Configure the index sort directly with {@code } in {@code } + * instead of wrapping a merge policy to carry a sort. + */ +@Deprecated public final class SortingMergePolicy extends FilterMergePolicy { private final Sort sort; diff --git a/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java index cf9705049c21..009af32d9f46 100644 --- a/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java +++ b/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java @@ -24,7 +24,13 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SortSpecParsing; -/** A {@link MergePolicyFactory} for {@code SortingMergePolicy} objects. */ +/** + * A {@link MergePolicyFactory} for {@code SortingMergePolicy} objects. + * + * @deprecated Configure the index sort directly with {@code } in {@code } + * instead. This factory only carries a sort to the index writer and does no merging of its own. + */ +@Deprecated public class SortingMergePolicyFactory extends WrapperMergePolicyFactory { protected final Sort mergeSort; diff --git a/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java b/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java deleted file mode 100644 index 4153f1974321..000000000000 --- a/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.search; - -import java.io.IOException; -import java.util.Arrays; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.FilterLeafCollector; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TotalHitCountCollector; - -/** - * A {@link Collector} that early terminates collection of documents on a per-segment basis, if the - * segment was sorted according to the given {@link Sort}. - * - *

NOTE: the {@code Collector} detects segments sorted according to a an {@link - * IndexWriterConfig#setIndexSort}. Also, it collects up to a specified {@code numDocsToCollect} - * from each segment, and therefore is mostly suitable for use in conjunction with collectors such - * as {@link TopDocsCollector}, and not e.g. {@link TotalHitCountCollector}. - * - *

NOTE: If you wrap a {@code TopDocsCollector} that sorts in the same order as the index - * order, the returned {@link TopDocsCollector#topDocs() TopDocs} will be correct. However the total - * of {@link TopDocsCollector#getTotalHits() hit count} will be vastly underestimated since not all - * matching documents will have been collected. - * - * @deprecated Use {@link TopFieldCollector} and set trackTotalHits to false. - * @lucene.experimental - */ -@Deprecated -final class EarlyTerminatingSortingCollector extends FilterCollector { - - /** - * Returns whether collection can be early-terminated if it sorts with the provided {@link Sort} - * and if segments are merged with the provided {@link Sort}. - */ - public static boolean canEarlyTerminate(Sort searchSort, Sort mergePolicySort) { - final SortField[] fields1 = searchSort.getSort(); - final SortField[] fields2 = mergePolicySort.getSort(); - // early termination is possible if fields1 is a prefix of fields2 - if (fields1.length > fields2.length) { - return false; - } - return Arrays.asList(fields1).equals(Arrays.asList(fields2).subList(0, fields1.length)); - } - - /** Sort used to sort the search results */ - private final Sort sort; - - /** Number of documents to collect in each segment */ - private final int numDocsToCollect; - - private final AtomicBoolean terminatedEarly = new AtomicBoolean(false); - - /** - * Create a new {@link EarlyTerminatingSortingCollector} instance. - * - * @param in the collector to wrap - * @param sort the sort you are sorting the search results on - * @param numDocsToCollect the number of documents to collect on each segment. When wrapping a - * {@link TopDocsCollector}, this number should be the number of hits. - * @throws IllegalArgumentException if the sort order doesn't allow for early termination with the - * given merge policy. - */ - public EarlyTerminatingSortingCollector(Collector in, Sort sort, int numDocsToCollect) { - super(in); - if (numDocsToCollect <= 0) { - throw new IllegalArgumentException( - "numDocsToCollect must always be > 0, got " + numDocsToCollect); - } - this.sort = sort; - this.numDocsToCollect = numDocsToCollect; - } - - @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - Sort segmentSort = context.reader().getMetaData().sort(); - if (segmentSort != null && canEarlyTerminate(sort, segmentSort) == false) { - throw new IllegalStateException( - "Cannot early terminate with sort order " - + sort - + " if segments are sorted with " - + segmentSort); - } - - if (segmentSort != null) { - // segment is sorted, can early-terminate - return new FilterLeafCollector(super.getLeafCollector(context)) { - private int numCollected; - - @Override - public void collect(int doc) throws IOException { - super.collect(doc); - if (++numCollected >= numDocsToCollect) { - terminatedEarly.set(true); - throw new CollectionTerminatedException(); - } - } - }; - } else { - return super.getLeafCollector(context); - } - } - - public boolean terminatedEarly() { - return terminatedEarly.get(); - } -} diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 8098b39f4a19..30295d4bb2bc 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -284,25 +284,28 @@ private Collector buildAndRunCollectorChain( DelegatingCollector postFilter) throws IOException { - EarlyTerminatingSortingCollector earlyTerminatingSortingCollector = null; + // When segmentTerminateEarly is requested, TopFieldCollector will natively stop collecting a + // segment once it has enough hits, provided the search sort is a prefix of the index sort. We + // report whether that happened by inspecting the collector after the search (see the finally + // block below). The eligibility check mirrors what TopFieldCollector does internally so we can + // warn on an unsupported combination rather than silently ignoring the flag. + TopFieldCollector segmentTerminatingCollector = null; if (cmd.getSegmentTerminateEarly()) { final Sort cmdSort = cmd.getSort(); final int cmdLen = cmd.getLen(); - final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); + final Sort indexSort = configuredIndexSort(); if (cmdSort == null || cmdLen <= 0 - || mergeSort == null - || !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) { + || indexSort == null + || !canEarlyTerminate(cmdSort, indexSort)) { log.warn( - "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}", + "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} indexSort={}", cmdSort, cmdLen, - mergeSort); + indexSort); } else { - collector = - earlyTerminatingSortingCollector = - new EarlyTerminatingSortingCollector(collector, cmdSort, cmd.getLen()); + segmentTerminatingCollector = firstTopFieldCollector(collector); } } @@ -343,8 +346,8 @@ private Collector buildAndRunCollectorChain( qr.setPartialResultsDetails(etce.getMessage()); qr.setApproximateTotalHits(etce.getApproximateTotalHits(reader.maxDoc())); } finally { - if (earlyTerminatingSortingCollector != null) { - qr.setSegmentTerminatedEarly(earlyTerminatingSortingCollector.terminatedEarly()); + if (segmentTerminatingCollector != null) { + qr.setSegmentTerminatedEarly(segmentTerminatingCollector.isEarlyTerminated()); } if (cmd.isQueryCancellable()) { core.getCancellableQueryTracker().removeCancellableQuery(cmd.getQueryID()); @@ -354,6 +357,61 @@ private Collector buildAndRunCollectorChain( return collector; } + /** + * The sort the segments are indexed by, for deciding whether {@code segmentTerminateEarly} is + * possible. Prefers the directly configured {@code }, falling back to a {@link + * org.apache.solr.index.SortingMergePolicy}'s sort. Returns null if neither is configured. + */ + private Sort configuredIndexSort() throws IOException { + String indexSortSpec = core.getSolrConfig().indexConfig.indexSort; + if (indexSortSpec != null) { + // Parsed per request, but only on the opt-in segmentTerminateEarly path, so the cost is + // negligible; parsing here also avoids caching a sort across schema reloads. + return SortSpecParsing.parseSortSpec(indexSortSpec, core.getLatestSchema()).getSort(); + } + // TODO: the SortingMergePolicy fallback can be removed once SortingMergePolicy is (SOLR-12230) + // removed. + return core.getSolrCoreState().getMergePolicySort(); + } + + /** + * Whether a search sorted by {@code searchSort} can be early-terminated on segments sorted by + * {@code indexSort}, i.e. the search sort is a prefix of the index sort. Mirrors the (package + * private) logic in Lucene's {@link TopFieldCollector} so callers can decide up front whether the + * {@code segmentTerminateEarly} request is honorable. + */ + private static boolean canEarlyTerminate(Sort searchSort, Sort indexSort) { + final SortField[] searchFields = searchSort.getSort(); + final SortField[] indexFields = indexSort.getSort(); + if (searchFields.length > indexFields.length) { + return false; + } + return Arrays.asList(searchFields) + .equals(Arrays.asList(indexFields).subList(0, searchFields.length)); + } + + /** + * Finds the {@link TopFieldCollector} reachable from the given collector, unwrapping any {@link + * MultiCollector} that a caller may have layered around it (e.g. to also collect max score or a + * {@link DocSet}). Returns null if none is present. This is called on the collector passed to + * {@link #buildAndRunCollectorChain} before the chain wraps it further, so only the caller-side + * wrapping needs to be considered. + */ + private static TopFieldCollector firstTopFieldCollector(Collector collector) { + if (collector instanceof TopFieldCollector topFieldCollector) { + return topFieldCollector; + } + if (collector instanceof MultiCollector multiCollector) { + for (Collector child : multiCollector.getCollectors()) { + TopFieldCollector found = firstTopFieldCollector(child); + if (found != null) { + return found; + } + } + } + return null; + } + public SolrIndexSearcher( SolrCore core, String path, @@ -1908,7 +1966,14 @@ TopDocsCollector buildTopDocsCollector(int len, QueryCommand final CursorMark cursor = cmd.getCursorMark(); final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null); - return new TopFieldCollectorManager(weightedSort, len, searchAfter, minNumFound) + // When segmentTerminateEarly is requested and the search sort is a prefix of the index sort, + // TopFieldCollector can stop collecting a segment once it has gathered `len` hits. That + // requires a total-hits threshold no larger than `len` (the default minExactCount is + // Integer.MAX_VALUE, which would count all hits and never terminate early). Take the smaller + // of the two so an explicit minExactCount still wins when it is lower. + final int totalHitsThreshold = + cmd.getSegmentTerminateEarly() ? Math.min(minNumFound, len) : minNumFound; + return new TopFieldCollectorManager(weightedSort, len, searchAfter, totalHitsThreshold) .newCollector(); } } diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 9a2e7e96dd56..a527f4c2c716 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -269,6 +269,7 @@ private SolrIndexWriter createMainIndexWriter(SolrCore core, String name) throws } @Override + @SuppressWarnings("deprecation") // still reads the sort from a (deprecated) SortingMergePolicy public Sort getMergePolicySort() throws IOException { lock(iwLock.readLock()); try { diff --git a/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java new file mode 100644 index 000000000000..d9fcbc3abd1c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.NumberType; +import org.apache.solr.schema.SchemaField; + +/** + * Builds the {@link Comparator} for the {@code } configuration, ordering a {@link + * org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) at open time via {@link + * org.apache.lucene.index.IndexWriterConfig#setLeafSorter(Comparator)}. + * + *

For a {@link SegmentSort.Kind#TIME} sort, the key is the segment creation time (from Lucene's + * per-segment {@code timestamp} diagnostic). For a {@link SegmentSort.Kind#FIELD} sort, the key is + * the per-segment minimum ({@code asc}) or maximum ({@code desc}) of a numeric docValues field, + * with each value normalized to an order-preserving long (see {@code toSortableLong}) so the raw + * {@code long} comparison matches true numeric order for every numeric type. A leaf whose key + * cannot be determined (e.g. a reader that does not resolve to a {@link SegmentReader}, or a + * segment with no value for the field) sorts last, so an unexpected reader shape degrades ordering + * rather than failing the search. (A genuine I/O error while reading docValues still propagates.) + */ +final class SegmentLeafSorter { + + private static final String TIMESTAMP_DIAGNOSTIC_KEY = "timestamp"; + + private SegmentLeafSorter() {} + + /** + * Returns the leaf sorter for the given config, or null for {@link SegmentSort.Kind#NONE} (no + * leaf sorter installed). For a field sort the field must exist and be a single-valued numeric + * docValues field. + */ + static Comparator forConfig(SegmentSort segmentSort, IndexSchema schema) { + switch (segmentSort.kind()) { + case NONE: + return null; + case TIME: + return build(segmentSort.descending(), SegmentLeafSorter::segmentTimestamp); + case FIELD: + final SchemaField sf = validateFieldForSort(segmentSort.field(), schema); + final String field = sf.getName(); + final NumberType numberType = sf.getType().getNumberType(); + final boolean descending = segmentSort.descending(); + // asc orders by each segment's min value; desc by its max value. + return build(descending, leaf -> segmentFieldExtremum(leaf, field, numberType, descending)); + default: + return null; + } + } + + private static SchemaField validateFieldForSort(String field, IndexSchema schema) { + SchemaField sf = schema.getFieldOrNull(field); + if (sf == null) { + throw new IllegalArgumentException( + " field '" + field + "' does not exist in the schema"); + } + if (!sf.hasDocValues()) { + throw new IllegalArgumentException( + " field '" + field + "' must have docValues=\"true\""); + } + if (sf.getType().getNumberType() == null) { + throw new IllegalArgumentException( + " field '" + field + "' must be a numeric field"); + } + if (sf.multiValued()) { + throw new IllegalArgumentException( + " field '" + field + "' must be single-valued"); + } + return sf; + } + + /** + * Builds a comparator over the given per-leaf key, memoizing the key per segment (the comparator + * is invoked O(n log n) times while the reader is opened). Unknown keys sort last in both + * directions. + */ + private static Comparator build(boolean descending, KeyFn keyFn) { + final long missingValue = descending ? Long.MIN_VALUE : Long.MAX_VALUE; + final Map cache = new ConcurrentHashMap<>(); + Comparator byKey = + Comparator.comparingLong( + leaf -> { + Object cacheKey = coreCacheKey(leaf); + if (cacheKey == null) { + return safeKey(keyFn, leaf, missingValue); + } + return cache.computeIfAbsent(cacheKey, k -> safeKey(keyFn, leaf, missingValue)); + }); + return descending ? byKey.reversed() : byKey; + } + + private static long safeKey(KeyFn keyFn, LeafReader leaf, long missingValue) { + try { + Long key = keyFn.key(leaf); + return key == null ? missingValue : key; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Object coreCacheKey(LeafReader leaf) { + var helper = leaf.getCoreCacheHelper(); + return helper == null ? null : helper.getKey(); + } + + private static Long segmentTimestamp(LeafReader reader) { + LeafReader unwrapped = FilterLeafReader.unwrap(reader); + if (unwrapped instanceof SegmentReader segmentReader) { + String timestamp = + segmentReader.getSegmentInfo().info.getDiagnostics().get(TIMESTAMP_DIAGNOSTIC_KEY); + if (timestamp != null) { + try { + return Long.parseLong(timestamp); + } catch (NumberFormatException ignored) { + return null; + } + } + } + return null; + } + + /** + * Per-segment min (asc) or max (desc) of the field's numeric docValues, as sortable-bits long. + */ + private static Long segmentFieldExtremum( + LeafReader leaf, String field, NumberType numberType, boolean max) throws IOException { + // Solr writes single-valued numeric docValues as NumericDocValues (older indexes may use + // SortedNumericDocValues); read either. Values are normalized to an order-preserving + // ("sortable") + // long before comparison -- see toSortableLong. + NumericDocValues numeric = leaf.getNumericDocValues(field); + if (numeric != null) { + return scan(max, numeric::nextDoc, () -> toSortableLong(numeric.longValue(), numberType)); + } + SortedNumericDocValues sorted = leaf.getSortedNumericDocValues(field); + if (sorted != null) { + // Normally single-valued (one value per doc). A legacy segment written while the field was + // multi-valued may have more; values come in ascending order, so take the first for a min + // scan (asc) and the last for a max scan (desc). + return scan(max, sorted::nextDoc, () -> toSortableLong(docValue(sorted, max), numberType)); + } + return null; + } + + private static long docValue(SortedNumericDocValues sorted, boolean max) throws IOException { + int count = sorted.docValueCount(); + long value = sorted.nextValue(); + if (max) { + for (int i = 1; i < count; i++) { + value = sorted.nextValue(); + } + } + return value; + } + + /** + * Normalizes a raw docValues long into an order-preserving long so a raw {@code long} min/max + * matches the field's numeric order. Single-valued FLOAT/DOUBLE docValues are stored by Solr as + * raw {@link Float#floatToIntBits}/{@link Double#doubleToLongBits} (not sortable bits), which do + * not order correctly for negative values, so they are decoded and re-encoded to sortable bits. + * INTEGER/LONG/DATE are already order-correct as signed longs. + */ + private static long toSortableLong(long raw, NumberType numberType) { + switch (numberType) { + case FLOAT: + return NumericUtils.floatToSortableInt(Float.intBitsToFloat((int) raw)); + case DOUBLE: + return NumericUtils.doubleToSortableLong(Double.longBitsToDouble(raw)); + default: // INTEGER, LONG, DATE + return raw; + } + } + + private static Long scan(boolean max, DocIterator advance, ValueSupplier value) + throws IOException { + long extremum = max ? Long.MIN_VALUE : Long.MAX_VALUE; + boolean any = false; + for (int doc = advance.nextDoc(); doc != NO_MORE; doc = advance.nextDoc()) { + long v = value.get(); + extremum = max ? Math.max(extremum, v) : Math.min(extremum, v); + any = true; + } + return any ? extremum : null; + } + + private static final int NO_MORE = DocIdSetIterator.NO_MORE_DOCS; + + @FunctionalInterface + private interface KeyFn { + Long key(LeafReader leaf) throws IOException; + } + + @FunctionalInterface + private interface DocIterator { + int nextDoc() throws IOException; + } + + @FunctionalInterface + private interface ValueSupplier { + long get() throws IOException; + } +} diff --git a/solr/core/src/java/org/apache/solr/update/SegmentSort.java b/solr/core/src/java/org/apache/solr/update/SegmentSort.java new file mode 100644 index 000000000000..32f0f1599d4c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/SegmentSort.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Locale; + +/** + * Parsed value of the {@code } index configuration, which controls the order in which + * a {@link org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) are visited at search + * time (Lucene's "leaf sorter"). This is "between-segment" ordering, distinct from index sort (the + * "within-segment" order of documents), and has no effect on merging. + * + *

Two forms are accepted: + * + *

    + *
  • a time preset, {@code TIME_ASC} or {@code TIME_DESC}, ordering by each segment's creation + * time (a segment property; used to visit the most recently written NRT segments first); or + *
  • a field sort spec, {@code asc|desc}, ordering by each segment's minimum (for + * {@code asc}) or maximum (for {@code desc}) value of a numeric docValues field. + *
+ */ +public final class SegmentSort { + + /** No segment ordering is imposed (Lucene's default). */ + public static final SegmentSort NONE = new SegmentSort(Kind.NONE, false, null); + + /** How segments are ordered. */ + public enum Kind { + NONE, + TIME, + FIELD + } + + private final Kind kind; + private final boolean descending; + private final String field; // non-null only for Kind.FIELD + + private SegmentSort(Kind kind, boolean descending, String field) { + this.kind = kind; + this.descending = descending; + this.field = field; + } + + public Kind kind() { + return kind; + } + + public boolean descending() { + return descending; + } + + /** The field name for a {@link Kind#FIELD} sort; null otherwise. */ + public String field() { + return field; + } + + /** + * Parses a {@code } value. Returns {@link #NONE} for null/blank. Accepts the presets + * {@code TIME_ASC}/{@code TIME_DESC} (case-insensitive), or a {@code asc|desc} spec. + * + * @throws IllegalArgumentException if the value is neither a known preset nor a valid {@code + * asc|desc} spec + */ + public static SegmentSort parse(String value) { + if (value == null || value.isBlank()) { + return NONE; + } + final String trimmed = value.trim(); + final String upper = trimmed.toUpperCase(Locale.ROOT); + if (upper.equals("NONE")) { + return NONE; + } + if (upper.equals("TIME_ASC")) { + return new SegmentSort(Kind.TIME, false, null); + } + if (upper.equals("TIME_DESC")) { + return new SegmentSort(Kind.TIME, true, null); + } + // Otherwise expect " asc|desc". + final String[] parts = trimmed.split("\\s+"); + if (parts.length == 2) { + final String order = parts[1].toLowerCase(Locale.ROOT); + if (order.equals("asc") || order.equals("desc")) { + return new SegmentSort(Kind.FIELD, order.equals("desc"), parts[0]); + } + } + throw new IllegalArgumentException( + "Invalid value '" + + value + + "'; expected NONE, TIME_ASC, TIME_DESC, or ' asc|desc'"); + } + + @Override + public String toString() { + switch (kind) { + case NONE: + return "NONE"; + case TIME: + return descending ? "TIME_DESC" : "TIME_ASC"; + case FIELD: + return field + (descending ? " desc" : " asc"); + default: + return kind.name(); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index b35a9b3651fc..c9ce693f3822 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -21,12 +21,14 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; +import java.util.Comparator; import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.search.Sort; @@ -46,6 +48,7 @@ import org.apache.solr.index.MergePolicyFactoryArgs; import org.apache.solr.index.SortingMergePolicy; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SortSpecParsing; import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,12 +90,23 @@ public class SolrIndexConfig implements MapWriter { public final int writeLockTimeout; public final String lockType; + + /** + * The index sort, as a sort spec string (e.g. {@code "timestamp desc"}), configured directly via + * {@code } rather than indirectly through a {@link SortingMergePolicy}. Null when not + * configured. + */ + public final String indexSort; + public final PluginInfo mergePolicyFactoryInfo; public final PluginInfo mergeSchedulerInfo; public final PluginInfo metricsInfo; public final PluginInfo mergedSegmentWarmerInfo; + /** Order in which segments (leaf readers) are visited at search time; never null. */ + public final SegmentSort segmentSort; + public InfoStream infoStream = InfoStream.NO_OUTPUT; private ConfigNode node; @@ -105,6 +119,8 @@ private SolrIndexConfig() { maxCommitMergeWaitMillis = -1; writeLockTimeout = -1; lockType = DirectoryFactory.LOCK_TYPE_NATIVE; + indexSort = null; + segmentSort = SegmentSort.NONE; mergePolicyFactoryInfo = null; mergeSchedulerInfo = null; mergedSegmentWarmerInfo = null; @@ -160,6 +176,9 @@ public SolrIndexConfig(ConfigNode cfg, SolrIndexConfig def) { writeLockTimeout = get("writeLockTimeout").intVal(def.writeLockTimeout); lockType = get("lockType").txt(def.lockType); + indexSort = get("indexSort").txt(def.indexSort); + String segmentSortSpec = get("segmentSort").txt(null); + segmentSort = segmentSortSpec == null ? def.segmentSort : SegmentSort.parse(segmentSortSpec); metricsInfo = getPluginInfo(get("metrics"), def.metricsInfo); mergeSchedulerInfo = getPluginInfo(get("mergeScheduler"), def.mergeSchedulerInfo); @@ -205,6 +224,8 @@ public void writeMap(EntryWriter ew) throws IOException { .put("writeLockTimeout", writeLockTimeout) .put("lockType", lockType) .put("infoStreamEnabled", infoStream != InfoStream.NO_OUTPUT) + .putIfNotNull("indexSort", indexSort) + .put("segmentSort", segmentSort.toString()) .putIfNotNull("mergeScheduler", mergeSchedulerInfo) .putIfNotNull("metrics", metricsInfo) .putIfNotNull("mergePolicyFactory", mergePolicyFactoryInfo) @@ -231,6 +252,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { } } + @SuppressWarnings("deprecation") // reconciles with a (deprecated) SortingMergePolicy if present public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { IndexSchema schema = core.getLatestSchema(); IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core)); @@ -257,9 +279,26 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergeScheduler(mergeScheduler); iwc.setInfoStream(infoStream); + Sort configuredIndexSort = null; + if (indexSort != null) { + configuredIndexSort = SortSpecParsing.parseSortSpec(indexSort, schema).getSort(); + if (configuredIndexSort == null) { + throw new IllegalArgumentException( + "Could not parse a usable index sort from " + indexSort + ""); + } + iwc.setIndexSort(configuredIndexSort); + } + if (mergePolicy instanceof SortingMergePolicy) { - Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort(); - iwc.setIndexSort(indexSort); + Sort mergePolicySort = ((SortingMergePolicy) mergePolicy).getSort(); + if (configuredIndexSort == null) { + iwc.setIndexSort(mergePolicySort); + } else if (!configuredIndexSort.equals(mergePolicySort)) { + log.warn( + " {} differs from the SortingMergePolicy sort {}; using ", + configuredIndexSort, + mergePolicySort); + } } if (iwc.getIndexSort() != null @@ -285,6 +324,11 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergedSegmentWarmer(warmer); } + Comparator leafSorter = SegmentLeafSorter.forConfig(segmentSort, schema); + if (leafSorter != null) { + iwc.setLeafSorter(leafSorter); + } + return iwc; } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml new file mode 100644 index 000000000000..4909542666ca --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml @@ -0,0 +1,37 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + in + org.apache.solr.index.TieredMergePolicyFactory + timestamp_i_dvo asc + + timestamp_i_dvo desc + ${solr.tests.lockType:single} + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml new file mode 100644 index 000000000000..9defdb0ab85e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml @@ -0,0 +1,32 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + timestamp_i_dvo desc + TIME_DESC + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml new file mode 100644 index 000000000000..0aea805fc2cc --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml @@ -0,0 +1,30 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + score desc + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml new file mode 100644 index 000000000000..6474650fe575 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml @@ -0,0 +1,52 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${mergePolicySort:timestamp_i_dvo desc} + ${solr.tests.lockType:single} + + + + + + + ${solr.ulog.dir:} + + + + ${solr.autoCommit.maxTime:-1} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + text + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml new file mode 100644 index 000000000000..794a8ca05a59 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml @@ -0,0 +1,31 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + no_such_field asc + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml new file mode 100644 index 000000000000..484671bf17da --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml @@ -0,0 +1,29 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + NOT_A_VALID_ORDER + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml new file mode 100644 index 000000000000..7ee7f1927f6f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml @@ -0,0 +1,29 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + TIME_DESC + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java index 3ea8a91919a8..26753e9b4c2a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java @@ -68,9 +68,18 @@ public void createCollection() throws Exception { collectionName = testName.getMethodName(); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); + // Exercise both ways of configuring the index sort: indirectly via a SortingMergePolicy, and + // directly via the element (SOLR-13681). Both must produce the same behavior. + // testSegmentTerminateEarlyWithIndexSort pins the config deterministically. + final String solrConfigFileName = + testName.getMethodName().contains("WithIndexSort") + ? "solrconfig-indexsort.xml" + : (random().nextBoolean() + ? "solrconfig-sortingmergepolicyfactory.xml" + : "solrconfig-indexsort.xml"); + final Map collectionProperties = new HashMap<>(); - collectionProperties.put( - CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml"); + collectionProperties.put(CoreDescriptor.CORE_CONFIG, solrConfigFileName); CollectionAdminRequest.Create cmd = CollectionAdminRequest.createCollection( @@ -86,7 +95,22 @@ public void createCollection() throws Exception { cluster.waitForActiveCollection(collectionName, NUM_SHARDS, NUM_SHARDS * REPLICATION_FACTOR); } + /** Runs the full early-termination scenario against whichever index-sort config was chosen. */ public void testSegmentTerminateEarly() throws Exception { + doTestSegmentTerminateEarly(); + } + + /** + * Same scenario, but deterministically against a directly configured {@code } (no + * SortingMergePolicy), so a regression in the {@code } branch of early-termination + * eligibility cannot pass CI on a coin flip (SOLR-13681 / SOLR-15390). + */ + @Test + public void testSegmentTerminateEarlyWithIndexSort() throws Exception { + doTestSegmentTerminateEarly(); + } + + private void doTestSegmentTerminateEarly() throws Exception { final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState(random()); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java new file mode 100644 index 000000000000..198d24f199b1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import static org.hamcrest.CoreMatchers.containsString; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.util.RefCounted; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * End-to-end test of the {@code RESORTINDEX} core-admin action (SOLR-12239): re-sort an existing + * (unsorted) core index and swap it in, so the core serves a sorted index without reindexing. + */ +public class ResortIndexActionTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // Plain config, no index sort configured -> the initial index is UNSORTED. + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void clearIndexBefore() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortIndexAction() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Build an existing unsorted index (intDvoDefault values out of order). + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + } finally { + admin.shutdown(); + admin.close(); + } + + // All docs still present and queryable after the resort+swap. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + // The index is now physically sorted by intDvoDefault ascending (internal docid order). + RefCounted ref = core.getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + int maxDoc = ref.get().getIndexReader().maxDoc(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < maxDoc; i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue( + "internal docids must be ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals( + "smallest value sorts first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } + + @Test + public void testResortIndexRequiresSortWhenNoneConfigured() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + // No 'sort' param, and this core has no SortingMergePolicy configured -> clear error. + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp)); + assertThat(thrown.getMessage(), containsString("no configured index sort")); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsUnparseableSort() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "no_such_field asc"), + resp)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, thrown.code()); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsChildDocs() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index a parent+child block; re-sorting would break the block, so it must be rejected. + SolrInputDocument parent = new SolrInputDocument(); + parent.addField("id", "100"); + parent.addField("title", "parent"); + SolrInputDocument child = new SolrInputDocument(); + child.addField("id", "101"); + child.addField("title", "child"); + parent.addChildDocument(child); + try (SolrQueryRequestBase addReq = + new SolrQueryRequestBase(core, new ModifiableSolrParams()) {}) { + AddUpdateCommand cmd = new AddUpdateCommand(addReq); + cmd.solrDoc = parent; + core.getUpdateHandler().addDoc(cmd); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp)); + assertThat(thrown.getMessage(), containsString("child/nested documents")); + } finally { + admin.shutdown(); + admin.close(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java new file mode 100644 index 000000000000..4bf2be33be97 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies the SOLR-13681/SOLR-12239 migration workflow: with the index sort configured directly + * via {@code }, {@code RESORTINDEX} (no {@code sort} param) picks up that configured + * sort and physically re-sorts the existing (unsorted) segments. + */ +public class ResortIndexConfiguredIndexSortTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // This config declares timestamp_i_dvo desc and no SortingMergePolicy. + initCore("solrconfig-indexsort.xml", "schema.xml"); + } + + @Test + public void testResortUsesConfiguredIndexSortWhenNoSortParam() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index documents in an order that does NOT match the configured desc sort. id == timestamp so + // the stored id doubles as the (docValues-only) timestamp for verification below. + int[] vals = {2, 0, 4, 1, 3}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "timestamp_i_dvo", Integer.toString(v))); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + // No 'sort' param -> must fall back to the configured , not error out. + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + String applied = String.valueOf(resp.getValues().get("indexSort")); + assertTrue( + "expected the configured (on timestamp_i_dvo) in response, got: " + applied, + applied.contains("timestamp_i_dvo")); + } finally { + admin.shutdown(); + admin.close(); + } + + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='5']"); + + // The index is now physically sorted by timestamp_i_dvo descending (internal docid order), + // which is what lets a matching query terminate early per segment. + RefCounted ref = core.getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + int maxDoc = ref.get().getIndexReader().maxDoc(); + long prev = Long.MAX_VALUE; + for (int i = 0; i < maxDoc; i++) { + // id == timestamp_i_dvo in this data; id is stored, the timestamp is docValues-only. + long ts = Long.parseLong(storedFields.document(i).get("id")); + assertTrue( + "internal docids must be timestamp-descending after resort: " + prev + " then " + ts, + ts <= prev); + prev = ts; + } + } finally { + ref.decref(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java new file mode 100644 index 000000000000..d4e32c6ec6be --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies that {@code RESORTINDEX} falls back to the sort configured on the core (via a + * SortingMergePolicy) when no explicit {@code sort} parameter is given (SOLR-12239). + */ +public class ResortIndexConfiguredSortTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // This config declares a SortingMergePolicy with sort = "timestamp_i_dvo desc". + initCore("solrconfig-sortingmergepolicyfactory.xml", "schema.xml"); + } + + @Test + public void testResortUsesConfiguredSortWhenNoSortParam() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + for (int i = 0; i < 5; i++) { + assertU(adoc("id", Integer.toString(i), "timestamp_i_dvo", Integer.toString(i))); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + // No 'sort' param -> must fall back to the configured SortingMergePolicy sort. + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + // The applied sort should be the one configured on the core (on timestamp_i_dvo), proving the + // fallback used the configured SortingMergePolicy sort rather than requiring a param. + String applied = String.valueOf(resp.getValues().get("indexSort")); + assertTrue( + "expected configured sort (on timestamp_i_dvo) in response, got: " + applied, + applied.contains("timestamp_i_dvo")); + } finally { + admin.shutdown(); + admin.close(); + } + + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='5']"); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java new file mode 100644 index 000000000000..8407320b2758 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests the {@link ResortCoreIndex} V2 API directly (SOLR-12239). */ +public class ResortCoreIndexTest extends SolrTestCaseJ4 { + + private CoreContainer coreContainer; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void setup() { + coreContainer = h.getCoreContainer(); + assertU(delQ("*:*")); + assertU(commit()); + } + + @After + public void cleanup() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortViaV2Api() throws Exception { + final String coreName = h.getCore().getName(); + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + + final SolrQueryRequest req = req(); + try { + ResortCoreIndex api = + new ResortCoreIndex( + coreContainer, + new CoreAdminHandler.CoreAdminAsyncTracker(), + req, + new SolrQueryResponse()); + ResortCoreIndexRequestBody body = new ResortCoreIndexRequestBody(); + body.sort = "intDvoDefault asc"; + + ResortCoreIndexResponse response = api.resortCoreIndex(coreName, body); + assertEquals(coreName, response.core); + assertNotNull(response.indexSort); + } finally { + req.close(); + } + + // Index is now physically sorted ascending by intDvoDefault. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + var ref = h.getCore().getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < ref.get().getIndexReader().maxDoc(); i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue("ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals("smallest first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java b/solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java new file mode 100644 index 000000000000..03f3577ca014 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Comparator; +import java.util.List; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.ParallelLeafReader; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.Directory; +import org.apache.solr.SolrTestCase; + +/** + * Verifies the time-based leaf (segment) ordering produced by {@link SegmentLeafSorter}. The + * field-based ordering is exercised where a real schema/core is available (see {@code + * SegmentSortFieldTest}). + */ +public class SegmentLeafSorterTest extends SolrTestCase { + + public void testNoneReturnsNoSorter() { + assertNull(SegmentLeafSorter.forConfig(SegmentSort.NONE, null)); + } + + public void testParseRejectsGarbage() { + expectThrows(IllegalArgumentException.class, () -> SegmentSort.parse("nonsense value here")); + expectThrows(IllegalArgumentException.class, () -> SegmentSort.parse("field bogusorder")); + } + + /** + * A leaf that does not resolve to a SegmentReader must be tolerated (ordered last), not crash the + * comparator. A {@link ParallelLeafReader} extends {@link LeafReader} directly (it is not a + * FilterLeafReader), so it does not unwrap to a SegmentReader and has no segment timestamp. + */ + public void testNonSegmentReaderSortsLastWithoutFailing() throws Exception { + try (Directory dirA = newDirectory(); + Directory dirB = newDirectory()) { + try (IndexWriter writer = new IndexWriter(dirA, new IndexWriterConfig())) { + Document doc = new Document(); + doc.add(new StringField("id", "0", Store.YES)); + writer.addDocument(doc); + writer.commit(); + } + // A second index with a disjoint field, so the two leaves can be combined into a + // multi-reader ParallelLeafReader (which reports no core cache key). + try (IndexWriter writer = new IndexWriter(dirB, new IndexWriterConfig())) { + Document doc = new Document(); + doc.add(new StringField("other", "0", Store.YES)); + writer.addDocument(doc); + writer.commit(); + } + try (DirectoryReader readerA = DirectoryReader.open(dirA); + DirectoryReader readerB = DirectoryReader.open(dirB)) { + LeafReader segmentLeaf = readerA.leaves().get(0).reader(); + LeafReader otherSegmentLeaf = readerB.leaves().get(0).reader(); + // A multi-reader ParallelLeafReader does not resolve to a single SegmentReader (and reports + // no core cache key), so it has no segment timestamp and must sort last. + try (LeafReader nonSegmentLeaf = + new ParallelLeafReader(false, segmentLeaf, otherSegmentLeaf)) { + for (String spec : new String[] {"TIME_ASC", "TIME_DESC"}) { + Comparator sorter = + SegmentLeafSorter.forConfig(SegmentSort.parse(spec), null); + assertNotNull(sorter); + // Must not throw for either ordering, regardless of argument order. + sorter.compare(segmentLeaf, nonSegmentLeaf); + sorter.compare(nonSegmentLeaf, segmentLeaf); + // The known-timestamp segment sorts before the unknown one in both directions. + assertTrue( + "segment with a timestamp should sort before one without (" + spec + ")", + sorter.compare(segmentLeaf, nonSegmentLeaf) < 0); + } + } + } + } + } + + public void testTimeDescVisitsNewestSegmentsFirst() throws Exception { + assertLeafOrder("TIME_DESC", /* newestFirst= */ true); + } + + public void testTimeAscVisitsOldestSegmentsFirst() throws Exception { + assertLeafOrder("TIME_ASC", /* newestFirst= */ false); + } + + /** + * Writes three single-document segments (each commit creates its own segment), reopens the reader + * with the leaf sorter for the given order, and asserts the segments are visited by creation time + * as expected. + */ + private void assertLeafOrder(String spec, boolean newestFirst) throws Exception { + Comparator leafSorter = SegmentLeafSorter.forConfig(SegmentSort.parse(spec), null); + assertNotNull(leafSorter); + + try (Directory dir = newDirectory()) { + // Disable merging so each commit stays its own segment with a distinct timestamp. + IndexWriterConfig iwc = new IndexWriterConfig().setLeafSorter(leafSorter); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter writer = new IndexWriter(dir, iwc)) { + for (int i = 0; i < 3; i++) { + Document doc = new Document(); + doc.add(new StringField("id", Integer.toString(i), Store.YES)); + writer.addDocument(doc); + writer.commit(); + // Ensure a strictly increasing per-segment timestamp (diagnostics are millis-resolution). + Thread.sleep(5); + } + } + + try (DirectoryReader reader = DirectoryReader.open(dir, leafSorter)) { + List leaves = reader.leaves(); + assertEquals(3, leaves.size()); + long previous = -1; + for (LeafReaderContext leaf : leaves) { + long timestamp = segmentTimestamp(leaf.reader()); + if (previous >= 0) { + if (newestFirst) { + assertTrue( + "expected descending segment timestamps, got " + previous + " then " + timestamp, + timestamp <= previous); + } else { + assertTrue( + "expected ascending segment timestamps, got " + previous + " then " + timestamp, + timestamp >= previous); + } + } + previous = timestamp; + } + } + } + } + + private static long segmentTimestamp(LeafReader reader) { + SegmentReader segmentReader = (SegmentReader) reader; + return Long.parseLong(segmentReader.getSegmentInfo().info.getDiagnostics().get("timestamp")); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java new file mode 100644 index 000000000000..3241feccc0ba --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Comparator; +import java.util.List; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Verifies field-based {@code } ordering and its validation (SOLR-17310). */ +public class SegmentSortFieldTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + private IndexSchema schema() { + return h.getCore().getLatestSchema(); + } + + @Test + public void testIntFieldOrdersBothDirections() throws Exception { + // Two segments: A holds 3, B holds 7 (single-valued int docValues). + assertU(adoc("id", "1", "timestamp_i_dvo", "3")); + assertU(commit()); // segment A + assertU(adoc("id", "2", "timestamp_i_dvo", "7")); + assertU(commit()); // segment B + + RefCounted ref = h.getCore().getSearcher(); + try { + List leaves = ref.get().getIndexReader().leaves(); + assumeTrue("test needs two segments", leaves.size() == 2); + LeafReader a = leafContaining(leaves, "1"); // 3 + LeafReader b = leafContaining(leaves, "2"); // 7 + + Comparator asc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo asc"), schema()); + Comparator desc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo desc"), schema()); + // asc: smaller (3) first -> A before B. desc: larger (7) first -> B before A. + assertTrue("asc: 3 before 7", asc.compare(a, b) < 0); + assertTrue("desc: 7 before 3", desc.compare(b, a) < 0); + } finally { + ref.decref(); + } + } + + @Test + public void testFieldSortRoundTripsSpec() { + SegmentSort s = SegmentSort.parse("timestamp_i_dvo desc"); + assertEquals(SegmentSort.Kind.FIELD, s.kind()); + assertTrue(s.descending()); + assertEquals("timestamp_i_dvo", s.field()); + assertEquals("timestamp_i_dvo desc", s.toString()); + } + + @Test + public void testUnknownFieldRejected() { + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("no_such_field asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("does not exist")); + } + + @Test + public void testNonNumericFieldRejected() { + // *_s_dvo is a string docValues field (has docValues, but is not numeric). + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("category_s_dvo asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("numeric")); + } + + @Test + public void testNonDocValuesFieldRejected() { + // 'name' is indexed/stored text without docValues. + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("name asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("docValues")); + } + + @Test + public void testMultiValuedFieldRejected() { + // *_ii_dvo is a multi-valued numeric docValues field. + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("nums_ii_dvo asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("single-valued")); + } + + /** + * A single-valued float field with negative values orders correctly. This is a regression guard: + * Solr stores single-valued float docValues as raw {@link Float#floatToIntBits} (not sortable + * bits), so a naive raw-long comparison would order negatives incorrectly. + */ + @Test + public void testFloatFieldWithNegativesOrdersByValue() throws Exception { + // *_f_p is a single-valued pfloat with docValues always enabled. + // Two all-negative segments chosen so raw floatToIntBits order DIVERGES from true numeric + // order: + // floatToIntBits(-8.0) > floatToIntBits(-2.0), so a naive raw-long compare would rank the + // segment holding -8.0 as "larger" and sort it AFTER the -2.0 segment, which is wrong. + assertU(adoc("id", "1", "price_f_p", "-2.0")); + assertU(commit()); // segment A (min = -2.0) + assertU(adoc("id", "2", "price_f_p", "-8.0")); + assertU(commit()); // segment B (min = -8.0) + + Comparator asc = + SegmentLeafSorter.forConfig(SegmentSort.parse("price_f_p asc"), schema()); + assertNotNull(asc); + + RefCounted ref = h.getCore().getSearcher(); + try { + List leaves = ref.get().getIndexReader().leaves(); + assumeTrue("test needs two segments", leaves.size() == 2); + LeafReader a = leafContaining(leaves, "1"); // -2.0 + LeafReader b = leafContaining(leaves, "2"); // -8.0 + // asc orders by each segment's min: B(-8.0) < A(-2.0), so B must sort before A. + assertTrue( + "segment with the smaller (more negative) value must sort first (asc)", + asc.compare(b, a) < 0); + assertTrue(asc.compare(a, b) > 0); + } finally { + ref.decref(); + } + } + + private static LeafReader leafContaining(List leaves, String id) + throws Exception { + for (LeafReaderContext ctx : leaves) { + var terms = ctx.reader().terms("id"); + if (terms != null && terms.iterator().seekExact(new BytesRef(id))) { + return ctx.reader(); + } + } + throw new AssertionError("no leaf contains id=" + id); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index e207d213af0c..a2d18b02acd6 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -61,6 +61,14 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { "solrconfig-bpreorderingmergepolicyfactory.xml"; private static final String solrConfigFileNameMergeOnFlushMergePolicyFactory = "solrconfig-mergeonflushmergepolicyfactory.xml"; + private static final String solrConfigFileNameIndexSort = "solrconfig-indexsort.xml"; + private static final String solrConfigFileNameIndexSortInvalid = + "solrconfig-indexsort-invalid.xml"; + private static final String solrConfigFileNameIndexSortAndMergePolicy = + "solrconfig-indexsort-and-mergepolicy.xml"; + private static final String solrConfigFileNameSegmentSort = "solrconfig-segmentsort.xml"; + private static final String solrConfigFileNameSegmentSortInvalid = + "solrconfig-segmentsort-invalid.xml"; private static final String schemaFileName = "schema.xml"; private static boolean compoundMergePolicySort = false; @@ -136,6 +144,7 @@ public void testConcurrentMergeSchedularSolrIndexConfigCreation() throws Excepti assertTrue("ms.isAutoIOThrottle", ms.getAutoIOThrottle()); } + @SuppressWarnings("deprecation") // exercises the deprecated SortingMergePolicy path public void testSortingMPSolrIndexConfigCreation() throws Exception { final SortField sortField1 = new SortField("timestamp_i_dvo", SortField.Type.INT, true); final SortField sortField2 = new SortField("id", SortField.Type.STRING, false); @@ -297,6 +306,9 @@ public void testToMap() throws Exception { assertFalse(Boolean.valueOf(m.get("infoStreamEnabled").toString())); } + ++mSizeExpected; + assertEquals("NONE", m.get("segmentSort")); + ++mSizeExpected; assertTrue(m.get("mergeScheduler") instanceof MapWriter); ++mSizeExpected; @@ -313,6 +325,36 @@ public void testToMap() throws Exception { assertEquals(mSizeExpected, m.size()); } + @Test + public void testSegmentSortDefaultsToNone() throws Exception { + SolrConfig solrConfig = + new SolrConfig(instanceDir, solrConfigFileNameSortingMergePolicyFactory); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertEquals(SegmentSort.Kind.NONE, solrIndexConfig.segmentSort.kind()); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + assertNull("no leaf sorter should be installed by default", iwc.getLeafSorter()); + } + + @Test + public void testSegmentSortConfigured() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSegmentSort); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertEquals("TIME_DESC", solrIndexConfig.segmentSort.toString()); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + assertNotNull("configured segmentSort should install a leaf sorter", iwc.getLeafSorter()); + } + + @Test + public void testSegmentSortInvalidValueFailsFast() { + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> + new SolrIndexConfig( + new SolrConfig(instanceDir, solrConfigFileNameSegmentSortInvalid), null)); + assertTrue(e.getMessage(), e.getMessage().contains("segmentSort")); + } + public void testMaxCommitMergeWaitTime() throws Exception { SolrConfig sc = new SolrConfig(TEST_PATH().resolve("collection1"), "solrconfig-test-misc.xml"); assertEquals(-1, sc.indexConfig.maxCommitMergeWaitMillis); @@ -325,4 +367,96 @@ public void testMaxCommitMergeWaitTime() throws Exception { assertEquals( 10, sc.indexConfig.toIndexWriterConfig(h.getCore()).getMaxFullFlushMergeWaitMillis()); } + + @Test + @SuppressWarnings( + "deprecation") // asserts the merge policy is NOT a (deprecated) SortingMergePolicy + public void testDirectIndexSortConfig() throws Exception { + // The compound-sort randomization in @BeforeClass may add a secondary "id asc" sort field. + final SortField primary = new SortField("timestamp_i_dvo", SortField.Type.INT, true); + final SortField secondary = new SortField("id", SortField.Type.STRING, false); + secondary.setMissingValue(SortField.STRING_LAST); + final Sort expected = + compoundMergePolicySort ? new Sort(primary, secondary) : new Sort(primary); + + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSort); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNotNull("indexSort should be parsed from ", solrIndexConfig.indexSort); + + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + + assertEquals("index sort should come from ", expected, iwc.getIndexSort()); + // the index sort was configured directly, without a SortingMergePolicy + assertFalse(iwc.getMergePolicy() instanceof SortingMergePolicy); + } + + @Test + public void testNoIndexSortByDefault() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameTieredMergePolicyFactory); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNull(solrIndexConfig.indexSort); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + assertNull(solrIndexConfig.toIndexWriterConfig(h.getCore()).getIndexSort()); + } + + @Test + public void testInvalidIndexSortFailsWithClearError() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSortInvalid); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, () -> solrIndexConfig.toIndexWriterConfig(h.getCore())); + assertTrue(e.getMessage(), e.getMessage().contains("")); + } + + @Test + public void testSegmentSortBadFieldFailsWhenBuildingWriterConfig() throws Exception { + // Syntax is valid so parse() succeeds, but the field does not exist; the field is validated + // when the IndexWriterConfig is built. + SolrConfig solrConfig = new SolrConfig(instanceDir, "solrconfig-segmentsort-badfield.xml"); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, () -> solrIndexConfig.toIndexWriterConfig(h.getCore())); + assertTrue(e.getMessage(), e.getMessage().contains("does not exist")); + } + + @Test + public void testIndexSortAndSegmentSortTogether() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, "solrconfig-indexsort-and-segmentsort.xml"); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNotNull("indexSort should be configured", solrIndexConfig.indexSort); + assertEquals("TIME_DESC", solrIndexConfig.segmentSort.toString()); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + // Both knobs take effect independently: index sort (within-segment) and leaf sorter (between). + assertNotNull("index sort should be set", iwc.getIndexSort()); + assertNotNull("leaf sorter should be set", iwc.getLeafSorter()); + } + + @Test + @SuppressWarnings( + "deprecation") // configures a (deprecated) SortingMergePolicy alongside + public void testIndexSortWinsOverSortingMergePolicy() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSortAndMergePolicy); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + + // is "desc" and the SortingMergePolicy sort is "asc"; must win. + final Sort expected = new Sort(new SortField("timestamp_i_dvo", SortField.Type.INT, true)); + assertEquals( + " should win over the SortingMergePolicy sort", expected, iwc.getIndexSort()); + // both were configured, so the merge policy is still a SortingMergePolicy + assertTrue(iwc.getMergePolicy() instanceof SortingMergePolicy); + } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc index a7dff995ee38..cee12d66b28b 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc @@ -898,6 +898,97 @@ Reusing the same `` value across restarts ensures continuity: D http://localhost:8983/solr/_collection_/update?commit=true ---- +[[coreadmin-resortindex]] +== RESORTINDEX + +The `RESORTINDEX` action re-sorts an existing core's index into a given xref:query-guide:common-query-parameters.adoc#sort-parameter[sort] order. +This makes it possible to enable xref:index-segments-merging.adoc[index sorting] on a collection that was created without it, without having to delete the index and reindex from source. + +Enabling an index sort on an existing (unsorted) index otherwise fails on core reload with an `IndexWriter` `validateIndexSort` error, because Lucene rejects the transition from unsorted segments to a configured sort. +`RESORTINDEX` produces a freshly sorted copy of the index (by merging each segment through the target sort) and swaps it in. + +This action is expensive and can take a while to complete on large indexes. +Consider running with the `async` option in such cases. + +Note: + +* Not supported in SolrCloud mode. +* Indexes containing child/nested documents are not supported (re-sorting would break the parent-child document blocks). +* The fields used in the sort must have docValues enabled. + +A typical workflow is to configure the sort in `solrconfig.xml` with xref:index-segments-merging.adoc#indexsort[``] (so subsequently added documents keep the index sorted), then run `RESORTINDEX` once to re-sort the pre-existing segments. +When `` is configured, `RESORTINDEX` may be called without a `sort` parameter and will use the configured sort. + +It is recommended to have a backup before running on production data. + +[tabs#coreadmin-resortindex-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=RESORTINDEX&core=techproducts&sort=timestamp+desc +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl -X POST http://localhost:8983/api/cores/techproducts/resort -H 'Content-Type: application/json' -d ' +{ + "sort": "timestamp desc" +} +' +---- +==== +====== + +=== RESORTINDEX Parameters + +`core`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the core whose index should be re-sorted. + +`sort`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The target index sort, in the usual Solr xref:query-guide:common-query-parameters.adoc#sort-parameter[sort syntax] (for example `timestamp desc`). +If omitted, the sort configured for the core is used -- the xref:index-segments-merging.adoc#indexsort[``] element if present, otherwise a (deprecated) `SortingMergePolicy` sort; if the core has no configured sort, the request fails. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. +Use <> with the provided `requestid` to poll for completion. + +=== RESORTINDEX Response + +On success, the response includes: + +`core`:: +The core name. + +`indexSort`:: +The index sort that was applied. + +On failure, an exception is thrown with error details. + [[coreadmin-requeststatus]] == REQUESTSTATUS diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index aa6a4985323e..332b2ea5bcec 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -253,6 +253,45 @@ This is not required for near real-time search, but will reduce search latency o ---- +=== indexSort + +Sorts the documents within each segment by the given fields, at flush and merge time (Lucene index sorting). +When a query's xref:query-guide:common-query-parameters.adoc#sort-parameter[sort] matches (or is a prefix of) this index sort, Solr can terminate collection on each segment early, which can speed up such queries. + +The value is a xref:query-guide:common-query-parameters.adoc#sort-parameter[sort spec] over docValues fields. +This is the direct replacement for configuring the sort indirectly through a `SortingMergePolicyFactory`; if both are configured, `` takes precedence. + +[source,xml] +---- +timestamp_dt desc +---- + +NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the xref:coreadmin-api.adoc#coreadmin-resortindex[`RESORTINDEX`] core admin action. It is not applied to existing segments automatically on reload. + +TIP: Do not confuse `` with the similarly named <>. `` sets the _within-segment_ document order (it changes how documents are physically stored, at flush/merge); `` only sets the _between-segment_ visitation order at search time and does not affect what is written. + +=== segmentSort + +Controls the order in which a segment's documents are visited at search time, by sorting the leaf readers of a newly opened reader. +This is _between-segment_ ordering; it is independent of index sorting (the _within-segment_ order of documents) and does not affect merging. + +The main use case is early termination: visiting the most promising segments first, so a query that only needs the first page of results can stop sooner. + +By default no ordering is imposed. Two forms are supported: + +* A *time preset* -- `TIME_ASC` or `TIME_DESC` -- orders segments by their creation time (a segment property). `TIME_DESC` visits the most recently written segments first, which suits time-based / append-mostly indexes. +* A *field spec* -- ` asc|desc` -- orders segments by the minimum (`asc`) or maximum (`desc`) value of a single-valued numeric docValues field. This is useful when the useful documents cluster by a field value rather than by write time. + +[source,xml] +---- +TIME_DESC +---- + +[source,xml] +---- +price_i desc +---- + == Compound File Segments Each Lucene segment is typically comprised of a dozen or so files. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc new file mode 100644 index 000000000000..f63b9dfe430d --- /dev/null +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc @@ -0,0 +1,43 @@ += Major Changes in Solr 11 +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr 11.0 is a major new release of Solr. + +This page highlights the most important changes including new features and changes in default behavior as well as previously deprecated features that have now been removed. + +== Solr 11 Upgrade Planning + +Before starting an upgrade to this version of Solr, please be sure to review all information about changes from the version you are currently on up to this one, to include the minor version number changes as well. + +== Index Sorting Configuration + +Index sorting can now be configured directly with the xref:configuration-guide:index-segments-merging.adoc#indexsort[``] element in the `` section of `solrconfig.xml`, instead of indirectly through a `SortingMergePolicyFactory`. + +As a result, `SortingMergePolicyFactory` (and the underlying `SortingMergePolicy`) are now *deprecated*. +Configurations that use `SortingMergePolicyFactory` continue to work, but should migrate to ``: + +[source,xml] +---- + + timestamp_dt desc + +---- + +To enable (or change) index sorting on a collection that already has data, use the xref:configuration-guide:coreadmin-api.adoc#coreadmin-resortindex[`RESORTINDEX`] core admin action to re-sort the existing segments; index sorting is not applied to already-written segments automatically on reload. + +The related between-segment ordering can be configured with the new xref:configuration-guide:index-segments-merging.adoc#segmentsort[``] element. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc index 061aea238e4a..2d51b6c54859 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc @@ -1,5 +1,6 @@ = Solr Upgrade Notes -:page-children: major-changes-in-solr-10, \ +:page-children: major-changes-in-solr-11, \ + major-changes-in-solr-10, \ major-changes-in-solr-9, \ major-changes-in-solr-8, \ major-changes-in-solr-7, \ diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 0401de380f1b..b3f035c8ac7b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -179,7 +179,8 @@ public enum CoreAdminAction { CREATESNAPSHOT, DELETESNAPSHOT, LISTSNAPSHOTS, - UPGRADEINDEX; + UPGRADEINDEX, + RESORTINDEX; public final boolean isRead;