From e790b22939ec98da34a1a617dcc3476c67bbc48f Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 11:28:37 +0300 Subject: [PATCH 1/9] SOLR-15390: use native TopFieldCollector early termination for segmentTerminateEarly Solr carried its own @Deprecated copy of EarlyTerminatingSortingCollector to implement the segmentTerminateEarly query parameter. Lucene's TopFieldCollector now performs the same per-segment early termination natively when the search sort is a prefix of the index sort, and exposes it via isEarlyTerminated(). buildAndRunCollectorChain now inspects the TopFieldCollector after the search instead of wrapping it, and buildTopDocsCollector lowers the total-hits threshold to the page size on the segmentTerminateEarly path so termination can fire. The segmentTerminatedEarly response header is reported exactly as before (absent when not requested, false when the sort/merge-policy combination is not eligible, true when a segment terminated early). The deprecated collector is removed. --- .../SOLR-15390-native-early-termination.yml | 11 ++ .../EarlyTerminatingSortingCollector.java | 130 ------------------ .../apache/solr/search/SolrIndexSearcher.java | 64 +++++++-- 3 files changed, 67 insertions(+), 138 deletions(-) create mode 100644 changelog/unreleased/SOLR-15390-native-early-termination.yml delete mode 100644 solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java diff --git a/changelog/unreleased/SOLR-15390-native-early-termination.yml b/changelog/unreleased/SOLR-15390-native-early-termination.yml new file mode 100644 index 000000000000..64779af6a13f --- /dev/null +++ b/changelog/unreleased/SOLR-15390-native-early-termination.yml @@ -0,0 +1,11 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Use Lucene's native TopFieldCollector early termination for the segmentTerminateEarly query + parameter, replacing the deprecated EarlyTerminatingSortingCollector +type: changed +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-15390 + url: https://issues.apache.org/jira/browse/SOLR-15390 diff --git a/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java b/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java deleted file mode 100644 index 4153f1974321..000000000000 --- a/solr/core/src/java/org/apache/solr/search/EarlyTerminatingSortingCollector.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.search; - -import java.io.IOException; -import java.util.Arrays; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.FilterLeafCollector; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TotalHitCountCollector; - -/** - * A {@link Collector} that early terminates collection of documents on a per-segment basis, if the - * segment was sorted according to the given {@link Sort}. - * - *

NOTE: the {@code Collector} detects segments sorted according to a an {@link - * IndexWriterConfig#setIndexSort}. Also, it collects up to a specified {@code numDocsToCollect} - * from each segment, and therefore is mostly suitable for use in conjunction with collectors such - * as {@link TopDocsCollector}, and not e.g. {@link TotalHitCountCollector}. - * - *

NOTE: If you wrap a {@code TopDocsCollector} that sorts in the same order as the index - * order, the returned {@link TopDocsCollector#topDocs() TopDocs} will be correct. However the total - * of {@link TopDocsCollector#getTotalHits() hit count} will be vastly underestimated since not all - * matching documents will have been collected. - * - * @deprecated Use {@link TopFieldCollector} and set trackTotalHits to false. - * @lucene.experimental - */ -@Deprecated -final class EarlyTerminatingSortingCollector extends FilterCollector { - - /** - * Returns whether collection can be early-terminated if it sorts with the provided {@link Sort} - * and if segments are merged with the provided {@link Sort}. - */ - public static boolean canEarlyTerminate(Sort searchSort, Sort mergePolicySort) { - final SortField[] fields1 = searchSort.getSort(); - final SortField[] fields2 = mergePolicySort.getSort(); - // early termination is possible if fields1 is a prefix of fields2 - if (fields1.length > fields2.length) { - return false; - } - return Arrays.asList(fields1).equals(Arrays.asList(fields2).subList(0, fields1.length)); - } - - /** Sort used to sort the search results */ - private final Sort sort; - - /** Number of documents to collect in each segment */ - private final int numDocsToCollect; - - private final AtomicBoolean terminatedEarly = new AtomicBoolean(false); - - /** - * Create a new {@link EarlyTerminatingSortingCollector} instance. - * - * @param in the collector to wrap - * @param sort the sort you are sorting the search results on - * @param numDocsToCollect the number of documents to collect on each segment. When wrapping a - * {@link TopDocsCollector}, this number should be the number of hits. - * @throws IllegalArgumentException if the sort order doesn't allow for early termination with the - * given merge policy. - */ - public EarlyTerminatingSortingCollector(Collector in, Sort sort, int numDocsToCollect) { - super(in); - if (numDocsToCollect <= 0) { - throw new IllegalArgumentException( - "numDocsToCollect must always be > 0, got " + numDocsToCollect); - } - this.sort = sort; - this.numDocsToCollect = numDocsToCollect; - } - - @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - Sort segmentSort = context.reader().getMetaData().sort(); - if (segmentSort != null && canEarlyTerminate(sort, segmentSort) == false) { - throw new IllegalStateException( - "Cannot early terminate with sort order " - + sort - + " if segments are sorted with " - + segmentSort); - } - - if (segmentSort != null) { - // segment is sorted, can early-terminate - return new FilterLeafCollector(super.getLeafCollector(context)) { - private int numCollected; - - @Override - public void collect(int doc) throws IOException { - super.collect(doc); - if (++numCollected >= numDocsToCollect) { - terminatedEarly.set(true); - throw new CollectionTerminatedException(); - } - } - }; - } else { - return super.getLeafCollector(context); - } - } - - public boolean terminatedEarly() { - return terminatedEarly.get(); - } -} diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 8098b39f4a19..9d252d03ba6e 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -284,7 +284,12 @@ private Collector buildAndRunCollectorChain( DelegatingCollector postFilter) throws IOException { - EarlyTerminatingSortingCollector earlyTerminatingSortingCollector = null; + // When segmentTerminateEarly is requested, TopFieldCollector will natively stop collecting a + // segment once it has enough hits, provided the search sort is a prefix of the index sort. We + // report whether that happened by inspecting the collector after the search (see the finally + // block below). The eligibility check mirrors what TopFieldCollector does internally so we can + // warn on an unsupported combination rather than silently ignoring the flag. + TopFieldCollector segmentTerminatingCollector = null; if (cmd.getSegmentTerminateEarly()) { final Sort cmdSort = cmd.getSort(); final int cmdLen = cmd.getLen(); @@ -293,16 +298,14 @@ private Collector buildAndRunCollectorChain( if (cmdSort == null || cmdLen <= 0 || mergeSort == null - || !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) { + || !canEarlyTerminate(cmdSort, mergeSort)) { log.warn( "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}", cmdSort, cmdLen, mergeSort); } else { - collector = - earlyTerminatingSortingCollector = - new EarlyTerminatingSortingCollector(collector, cmdSort, cmd.getLen()); + segmentTerminatingCollector = firstTopFieldCollector(collector); } } @@ -343,8 +346,8 @@ private Collector buildAndRunCollectorChain( qr.setPartialResultsDetails(etce.getMessage()); qr.setApproximateTotalHits(etce.getApproximateTotalHits(reader.maxDoc())); } finally { - if (earlyTerminatingSortingCollector != null) { - qr.setSegmentTerminatedEarly(earlyTerminatingSortingCollector.terminatedEarly()); + if (segmentTerminatingCollector != null) { + qr.setSegmentTerminatedEarly(segmentTerminatingCollector.isEarlyTerminated()); } if (cmd.isQueryCancellable()) { core.getCancellableQueryTracker().removeCancellableQuery(cmd.getQueryID()); @@ -354,6 +357,44 @@ private Collector buildAndRunCollectorChain( return collector; } + /** + * Whether a search sorted by {@code searchSort} can be early-terminated on segments sorted by + * {@code indexSort}, i.e. the search sort is a prefix of the index sort. Mirrors the (package + * private) logic in Lucene's {@link TopFieldCollector} so callers can decide up front whether the + * {@code segmentTerminateEarly} request is honorable. + */ + private static boolean canEarlyTerminate(Sort searchSort, Sort indexSort) { + final SortField[] searchFields = searchSort.getSort(); + final SortField[] indexFields = indexSort.getSort(); + if (searchFields.length > indexFields.length) { + return false; + } + return Arrays.asList(searchFields) + .equals(Arrays.asList(indexFields).subList(0, searchFields.length)); + } + + /** + * Finds the {@link TopFieldCollector} reachable from the given collector, unwrapping any {@link + * MultiCollector} that a caller may have layered around it (e.g. to also collect max score or a + * {@link DocSet}). Returns null if none is present. This is called on the collector passed to + * {@link #buildAndRunCollectorChain} before the chain wraps it further, so only the caller-side + * wrapping needs to be considered. + */ + private static TopFieldCollector firstTopFieldCollector(Collector collector) { + if (collector instanceof TopFieldCollector topFieldCollector) { + return topFieldCollector; + } + if (collector instanceof MultiCollector multiCollector) { + for (Collector child : multiCollector.getCollectors()) { + TopFieldCollector found = firstTopFieldCollector(child); + if (found != null) { + return found; + } + } + } + return null; + } + public SolrIndexSearcher( SolrCore core, String path, @@ -1908,7 +1949,14 @@ TopDocsCollector buildTopDocsCollector(int len, QueryCommand final CursorMark cursor = cmd.getCursorMark(); final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null); - return new TopFieldCollectorManager(weightedSort, len, searchAfter, minNumFound) + // When segmentTerminateEarly is requested and the search sort is a prefix of the index sort, + // TopFieldCollector can stop collecting a segment once it has gathered `len` hits. That + // requires a total-hits threshold no larger than `len` (the default minExactCount is + // Integer.MAX_VALUE, which would count all hits and never terminate early). Take the smaller + // of the two so an explicit minExactCount still wins when it is lower. + final int totalHitsThreshold = + cmd.getSegmentTerminateEarly() ? Math.min(minNumFound, len) : minNumFound; + return new TopFieldCollectorManager(weightedSort, len, searchAfter, totalHitsThreshold) .newCollector(); } } From 7a2962722d1216dcb6b80e01ab3838c6fc04dd30 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 14:26:35 +0300 Subject: [PATCH 2/9] SOLR-13681: configure Lucene's index sort directly via Adds an element to that sets Lucene's index sort directly, instead of the indirect route through a SortingMergePolicyFactory (a merge policy that does no merging and only carries a Sort). The value is a standard sort spec over docValues fields. When both an and a SortingMergePolicy sort are configured, the takes precedence; a mismatch is logged. The segmentTerminateEarly eligibility check now consults the configured index sort, falling back to the merge policy sort. Existing nested-document handling (the parent field for child-doc schemas) applies to the directly configured sort as well. Enabling index sorting on a collection with existing unsorted segments still requires re-sorting them (see the RESORTINDEX core admin action); it is not applied on reload automatically. --- .../SOLR-13681-direct-indexsort-config.yml | 12 +++++ .../apache/solr/search/SolrIndexSearcher.java | 23 ++++++-- .../apache/solr/update/SolrIndexConfig.java | 29 ++++++++++- .../collection1/conf/solrconfig-indexsort.xml | 52 +++++++++++++++++++ .../apache/solr/cloud/TestSegmentSorting.java | 10 +++- .../solr/update/SolrIndexConfigTest.java | 33 ++++++++++++ .../pages/index-segments-merging.adoc | 15 ++++++ 7 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 changelog/unreleased/SOLR-13681-direct-indexsort-config.yml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml diff --git a/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml b/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml new file mode 100644 index 000000000000..4f15ec70f8f4 --- /dev/null +++ b/changelog/unreleased/SOLR-13681-direct-indexsort-config.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Allow configuring Lucene's index sort directly with an element in , + instead of indirectly through a SortingMergePolicyFactory +type: added +authors: + - name: Serhiy Bzhezytskyy + - name: Christine Poerschke +links: + - name: SOLR-13681 + url: https://issues.apache.org/jira/browse/SOLR-13681 diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 9d252d03ba6e..e05f34bf6f76 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -293,17 +293,17 @@ private Collector buildAndRunCollectorChain( if (cmd.getSegmentTerminateEarly()) { final Sort cmdSort = cmd.getSort(); final int cmdLen = cmd.getLen(); - final Sort mergeSort = core.getSolrCoreState().getMergePolicySort(); + final Sort indexSort = configuredIndexSort(); if (cmdSort == null || cmdLen <= 0 - || mergeSort == null - || !canEarlyTerminate(cmdSort, mergeSort)) { + || indexSort == null + || !canEarlyTerminate(cmdSort, indexSort)) { log.warn( - "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}", + "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} indexSort={}", cmdSort, cmdLen, - mergeSort); + indexSort); } else { segmentTerminatingCollector = firstTopFieldCollector(collector); } @@ -357,6 +357,19 @@ private Collector buildAndRunCollectorChain( return collector; } + /** + * The sort the segments are indexed by, for deciding whether {@code segmentTerminateEarly} is + * possible. Prefers the directly configured {@code }, falling back to a {@link + * org.apache.solr.index.SortingMergePolicy}'s sort. Returns null if neither is configured. + */ + private Sort configuredIndexSort() throws IOException { + String indexSortSpec = core.getSolrConfig().indexConfig.indexSort; + if (indexSortSpec != null) { + return SortSpecParsing.parseSortSpec(indexSortSpec, core.getLatestSchema()).getSort(); + } + return core.getSolrCoreState().getMergePolicySort(); + } + /** * Whether a search sorted by {@code searchSort} can be early-terminated on segments sorted by * {@code indexSort}, i.e. the search sort is a prefix of the index sort. Mirrors the (package diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index b35a9b3651fc..d6f0415edc4a 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -46,6 +46,7 @@ import org.apache.solr.index.MergePolicyFactoryArgs; import org.apache.solr.index.SortingMergePolicy; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SortSpecParsing; import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +88,14 @@ public class SolrIndexConfig implements MapWriter { public final int writeLockTimeout; public final String lockType; + + /** + * The index sort, as a sort spec string (e.g. {@code "timestamp desc"}), configured directly via + * {@code } rather than indirectly through a {@link SortingMergePolicy}. Null when not + * configured. + */ + public final String indexSort; + public final PluginInfo mergePolicyFactoryInfo; public final PluginInfo mergeSchedulerInfo; public final PluginInfo metricsInfo; @@ -105,6 +114,7 @@ private SolrIndexConfig() { maxCommitMergeWaitMillis = -1; writeLockTimeout = -1; lockType = DirectoryFactory.LOCK_TYPE_NATIVE; + indexSort = null; mergePolicyFactoryInfo = null; mergeSchedulerInfo = null; mergedSegmentWarmerInfo = null; @@ -160,6 +170,7 @@ public SolrIndexConfig(ConfigNode cfg, SolrIndexConfig def) { writeLockTimeout = get("writeLockTimeout").intVal(def.writeLockTimeout); lockType = get("lockType").txt(def.lockType); + indexSort = get("indexSort").txt(def.indexSort); metricsInfo = getPluginInfo(get("metrics"), def.metricsInfo); mergeSchedulerInfo = getPluginInfo(get("mergeScheduler"), def.mergeSchedulerInfo); @@ -205,6 +216,7 @@ public void writeMap(EntryWriter ew) throws IOException { .put("writeLockTimeout", writeLockTimeout) .put("lockType", lockType) .put("infoStreamEnabled", infoStream != InfoStream.NO_OUTPUT) + .putIfNotNull("indexSort", indexSort) .putIfNotNull("mergeScheduler", mergeSchedulerInfo) .putIfNotNull("metrics", metricsInfo) .putIfNotNull("mergePolicyFactory", mergePolicyFactoryInfo) @@ -257,9 +269,22 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergeScheduler(mergeScheduler); iwc.setInfoStream(infoStream); + Sort configuredIndexSort = null; + if (indexSort != null) { + configuredIndexSort = SortSpecParsing.parseSortSpec(indexSort, schema).getSort(); + iwc.setIndexSort(configuredIndexSort); + } + if (mergePolicy instanceof SortingMergePolicy) { - Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort(); - iwc.setIndexSort(indexSort); + Sort mergePolicySort = ((SortingMergePolicy) mergePolicy).getSort(); + if (configuredIndexSort == null) { + iwc.setIndexSort(mergePolicySort); + } else if (!configuredIndexSort.equals(mergePolicySort)) { + log.warn( + " {} differs from the SortingMergePolicy sort {}; using ", + configuredIndexSort, + mergePolicySort); + } } if (iwc.getIndexSort() != null diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml new file mode 100644 index 000000000000..6474650fe575 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort.xml @@ -0,0 +1,52 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${mergePolicySort:timestamp_i_dvo desc} + ${solr.tests.lockType:single} + + + + + + + ${solr.ulog.dir:} + + + + ${solr.autoCommit.maxTime:-1} + false + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + text + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java index 3ea8a91919a8..beceb7f075c0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java @@ -68,9 +68,15 @@ public void createCollection() throws Exception { collectionName = testName.getMethodName(); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); + // Exercise both ways of configuring the index sort: indirectly via a SortingMergePolicy, and + // directly via the element (SOLR-13681). Both must produce the same behavior. + final String solrConfigFileName = + random().nextBoolean() + ? "solrconfig-sortingmergepolicyfactory.xml" + : "solrconfig-indexsort.xml"; + final Map collectionProperties = new HashMap<>(); - collectionProperties.put( - CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml"); + collectionProperties.put(CoreDescriptor.CORE_CONFIG, solrConfigFileName); CollectionAdminRequest.Create cmd = CollectionAdminRequest.createCollection( diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index e207d213af0c..e4c330bb49be 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -61,6 +61,7 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { "solrconfig-bpreorderingmergepolicyfactory.xml"; private static final String solrConfigFileNameMergeOnFlushMergePolicyFactory = "solrconfig-mergeonflushmergepolicyfactory.xml"; + private static final String solrConfigFileNameIndexSort = "solrconfig-indexsort.xml"; private static final String schemaFileName = "schema.xml"; private static boolean compoundMergePolicySort = false; @@ -325,4 +326,36 @@ public void testMaxCommitMergeWaitTime() throws Exception { assertEquals( 10, sc.indexConfig.toIndexWriterConfig(h.getCore()).getMaxFullFlushMergeWaitMillis()); } + + @Test + public void testDirectIndexSortConfig() throws Exception { + // The compound-sort randomization in @BeforeClass may add a secondary "id asc" sort field. + final SortField primary = new SortField("timestamp_i_dvo", SortField.Type.INT, true); + final SortField secondary = new SortField("id", SortField.Type.STRING, false); + secondary.setMissingValue(SortField.STRING_LAST); + final Sort expected = + compoundMergePolicySort ? new Sort(primary, secondary) : new Sort(primary); + + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSort); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNotNull("indexSort should be parsed from ", solrIndexConfig.indexSort); + + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + + assertEquals("index sort should come from ", expected, iwc.getIndexSort()); + // the index sort was configured directly, without a SortingMergePolicy + assertFalse(iwc.getMergePolicy() instanceof SortingMergePolicy); + } + + @Test + public void testNoIndexSortByDefault() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameTieredMergePolicyFactory); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNull(solrIndexConfig.indexSort); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + assertNull(solrIndexConfig.toIndexWriterConfig(h.getCore()).getIndexSort()); + } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index aa6a4985323e..8d5785c309f0 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -253,6 +253,21 @@ This is not required for near real-time search, but will reduce search latency o ---- +=== indexSort + +Sorts the documents within each segment by the given fields, at flush and merge time (Lucene index sorting). +When a query's xref:query-guide:common-query-parameters.adoc#sort-parameter[sort] matches (or is a prefix of) this index sort, Solr can terminate collection on each segment early, which can speed up such queries. + +The value is a xref:query-guide:common-query-parameters.adoc#sort-parameter[sort spec] over docValues fields. +This is the direct replacement for configuring the sort indirectly through a `SortingMergePolicyFactory`; if both are configured, `` takes precedence. + +[source,xml] +---- +timestamp_dt desc +---- + +NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the `RESORTINDEX` core admin action. It is not applied to existing segments automatically on reload. + == Compound File Segments Each Lucene segment is typically comprised of a dozen or so files. From 4d39d73cff1cb92cb2fceb60072a209d6c56beae Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Thu, 16 Jul 2026 16:50:04 +0300 Subject: [PATCH 3/9] SOLR-12239: add RESORTINDEX core-admin action to re-sort an existing index Enabling index sorting on a collection created without it currently fails on core reload with an IndexWriter.validateIndexSort error, forcing a delete + reindex from source. This adds a RESORTINDEX core-admin action that re-sorts the existing index into the requested sort order in place, using the LUCENE-9484 mechanism: each segment reader is wrapped in a SortingCodecReader and merged into a fresh sort-configured IndexWriter via addIndexes(CodecReader...), and the result is swapped in with modifyIndexProps (as RestoreCore/replication do), then the writer and searcher are reopened. Notes: - addIndexes does not auto-sort unsorted readers (LUCENE-8505 removed that), so the explicit SortingCodecReader wrap is what performs the merge-based re-sort. - Not supported in SolrCloud mode. - Indexes containing child/nested documents are rejected (re-sorting would break the parent-child blocks), matching UPGRADEINDEX's restriction. - On a failed swap the original index.properties is restored (RestoreCore's rollback). The target sort is given by the sort request parameter using the usual Solr sort syntax; the fields must have docValues. --- ...R-12239-index-sort-existing-collection.yml | 10 + .../api/endpoint/ResortCoreIndexApi.java | 43 +++ .../api/model/ResortCoreIndexRequestBody.java | 35 +++ .../api/model/ResortCoreIndexResponse.java | 31 +++ .../solr/handler/admin/CoreAdminHandler.java | 2 + .../handler/admin/CoreAdminOperation.java | 4 +- .../solr/handler/admin/ResortIndexOp.java | 53 ++++ .../handler/admin/api/ResortCoreIndex.java | 246 ++++++++++++++++++ .../handler/admin/ResortIndexActionTest.java | 211 +++++++++++++++ .../admin/ResortIndexConfiguredSortTest.java | 73 ++++++ .../admin/api/ResortCoreIndexTest.java | 97 +++++++ .../pages/coreadmin-api.adoc | 90 +++++++ .../solr/common/params/CoreAdminParams.java | 3 +- 13 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml create mode 100644 solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java create mode 100644 solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java diff --git a/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml new file mode 100644 index 000000000000..fde7d60839bd --- /dev/null +++ b/changelog/unreleased/SOLR-12239-index-sort-existing-collection.yml @@ -0,0 +1,10 @@ +title: > + New RESORTINDEX core-admin action re-sorts an existing core's index into a configured sort order, + so index sorting can be enabled on a collection created without it, without deleting and reindexing + from source. Not supported in SolrCloud mode; indexes with child/nested documents are rejected. +type: added +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-12239 + url: https://issues.apache.org/jira/browse/SOLR-12239 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java new file mode 100644 index 000000000000..6a2521f993a8 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ResortCoreIndexApi.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; + +/** V2 API definition for re-sorting an existing core index (SOLR-12239). */ +@Path("/cores/{coreName}/resort") +public interface ResortCoreIndexApi { + + @POST + @Operation( + summary = + "Re-sort the existing index of the specified core into the target sort order, so index " + + "sorting can be enabled without reindexing from source.", + tags = {"cores"}) + ResortCoreIndexResponse resortCoreIndex( + @Parameter(description = "The name of the core whose index should be re-sorted") + @PathParam("coreName") + String coreName, + ResortCoreIndexRequestBody requestBody) + throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java new file mode 100644 index 000000000000..4dfb2fa180b4 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ResortCoreIndexRequestBody { + + @Schema( + description = + "The target index sort, in Solr sort syntax (e.g. 'timestamp desc'). If omitted, the " + + "sort configured for the core (via a SortingMergePolicy) is used. Fields must have " + + "docValues.") + @JsonProperty + public String sort; + + @Schema(description = "Request ID to track this action which will be processed asynchronously.") + @JsonProperty + public String async; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java new file mode 100644 index 000000000000..dce2fd337f3e --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ResortCoreIndexResponse extends SolrJerseyResponse { + + @Schema(description = "The name of the core whose index was re-sorted.") + @JsonProperty + public String core; + + @Schema(description = "The index sort that was applied.") + @JsonProperty + public String indexSort; +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 494e3f898414..de2d6e7c7019 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -69,6 +69,7 @@ import org.apache.solr.handler.admin.api.RequestBufferUpdatesAPI; import org.apache.solr.handler.admin.api.RequestCoreRecoveryAPI; import org.apache.solr.handler.admin.api.RequestSyncShardAPI; +import org.apache.solr.handler.admin.api.ResortCoreIndex; import org.apache.solr.handler.admin.api.RestoreCore; import org.apache.solr.handler.admin.api.SplitCoreAPI; import org.apache.solr.handler.admin.api.SwapCores; @@ -339,6 +340,7 @@ public Collection> getJerseyResources() { SwapCores.class, RenameCore.class, MergeIndexes.class, + ResortCoreIndex.class, GetNodeCommandStatus.class); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 2edc70aef30b..5213e3027898 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -33,6 +33,7 @@ import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTRECOVERY; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSTATUS; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.REQUESTSYNCSHARD; +import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESORTINDEX; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.RESTORECORE; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.SPLIT; import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.STATUS; @@ -258,7 +259,8 @@ public enum CoreAdminOperation implements CoreAdminOp { V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); }), - UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()); + UPGRADEINDEX_OP(UPGRADEINDEX, new UpgradeCoreIndexOp()), + RESORTINDEX_OP(RESORTINDEX, new ResortIndexOp()); final CoreAdminParams.CoreAdminAction action; final CoreAdminOp fun; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java new file mode 100644 index 000000000000..3a4c0efc2651 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/ResortIndexOp.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.api.ResortCoreIndex; +import org.apache.solr.handler.api.V2ApiUtils; + +/** + * v1 ({@code action=RESORTINDEX}) wrapper delegating to the {@link ResortCoreIndex} V2 API + * (SOLR-12239). + */ +class ResortIndexOp implements CoreAdminHandler.CoreAdminOp { + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public void execute(CoreAdminHandler.CallInfo it) throws Exception { + final SolrParams params = it.req.getParams(); + final String cname = params.required().get(CoreAdminParams.CORE); + final var requestBody = new ResortCoreIndexRequestBody(); + // "async" is intentionally omitted because CoreAdminHandler has already processed it. + requestBody.sort = params.get(CommonParams.SORT); + + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final ResortCoreIndex api = + new ResortCoreIndex(coreContainer, it.handler.getCoreAdminAsyncTracker(), it.req, it.rsp); + final ResortCoreIndexResponse response = api.resortCoreIndex(cname, requestBody); + V2ApiUtils.squashIntoSolrResponseWithoutHeader(it.rsp, response); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java new file mode 100644 index 000000000000..2e350072ebef --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SlowCodecReaderWrapper; +import org.apache.lucene.index.SortingCodecReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.Directory; +import org.apache.solr.client.api.endpoint.ResortCoreIndexApi; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.IndexFetcher; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortSpecParsing; +import org.apache.solr.util.RefCounted; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * V2 API implementation for re-sorting an existing core index (SOLR-12239). + * + *

Re-sorts a (possibly unsorted) core index into the target {@link Sort} using the LUCENE-9484 + * mechanism — each segment reader is wrapped in a {@link SortingCodecReader} and merged into a + * fresh sort-configured {@link IndexWriter} via {@link IndexWriter#addIndexes(CodecReader...)} — + * then swaps it in via {@code modifyIndexProps} and reopens the writer/searcher (mirroring + * RestoreCore). + * + *

Not supported in SolrCloud mode. Indexes with child/nested documents are rejected. + */ +public class ResortCoreIndex extends CoreAdminAPIBase implements ResortCoreIndexApi { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ResortCoreIndex( + CoreContainer coreContainer, + CoreAdminHandler.CoreAdminAsyncTracker coreAdminAsyncTracker, + SolrQueryRequest req, + SolrQueryResponse rsp) { + super(coreContainer, coreAdminAsyncTracker, req, rsp); + } + + @Override + public boolean isExpensive() { + return true; + } + + @Override + public ResortCoreIndexResponse resortCoreIndex( + String coreName, ResortCoreIndexRequestBody requestBody) throws Exception { + ensureRequiredParameterProvided("coreName", coreName); + if (coreContainer.isZooKeeperAware()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "resort is not supported in SolrCloud mode"); + } + final ResortCoreIndexResponse response = + instantiateJerseyResponse(ResortCoreIndexResponse.class); + final String sortParam = requestBody == null ? null : requestBody.sort; + final String async = requestBody == null ? null : requestBody.async; + + return handlePotentiallyAsynchronousTask( + response, + coreName, + async, + "resort-index", + () -> { + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Core not found: " + coreName); + } + final Sort indexSort = resolveTargetSort(core, sortParam); + assertNoChildDocs(core); + resortAndSwap(core, indexSort); + response.core = coreName; + response.indexSort = indexSort.toString(); + } catch (Exception e) { + throw new CoreAdminAPIBaseException(e); + } + return response; + }); + } + + private Sort resolveTargetSort(SolrCore core, String sortParam) throws IOException { + if (sortParam != null && !sortParam.isBlank()) { + final Sort sort = SortSpecParsing.parseSortSpec(sortParam, core.getLatestSchema()).getSort(); + if (sort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not parse a usable index sort from sort=" + sortParam); + } + return sort; + } + final Sort configured = core.getSolrCoreState().getMergePolicySort(); + if (configured == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No 'sort' parameter given and the core has no configured index sort " + + "(SortingMergePolicy) to fall back to"); + } + return configured; + } + + private void assertNoChildDocs(SolrCore core) throws IOException { + RefCounted ref = core.getSearcher(); + try { + SolrIndexSearcher searcher = ref.get(); + if (!searcher.getSchema().isUsableForChildDocs()) { + return; + } + String uniqueKeyField = searcher.getSchema().getUniqueKeyField().getName(); + for (LeafReaderContext leaf : searcher.getIndexReader().leaves()) { + Terms rootTerms = leaf.reader().terms(IndexSchema.ROOT_FIELD_NAME); + if (rootTerms == null) { + continue; + } + long uniqueRootValues = rootTerms.size(); + Terms idTerms = leaf.reader().terms(uniqueKeyField); + long uniqueIdValues = (idTerms != null) ? idTerms.size() : -1; + if (uniqueRootValues == -1 || uniqueIdValues == -1 || uniqueRootValues < uniqueIdValues) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "resort does not support indexes containing child/nested documents. " + + "Consider reindexing your data from the original source."); + } + } + } finally { + ref.decref(); + } + } + + private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { + final String resortIndexName = "resort." + System.nanoTime(); + final String resortIndexPath = core.getDataDir() + resortIndexName; + final String currentIndexPath = core.getIndexDir(); + final DirectoryFactory df = core.getDirectoryFactory(); + final String lockType = core.getSolrConfig().indexConfig.lockType; + + Directory currentDir = null; + Directory resortDir = null; + try { + currentDir = df.get(currentIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + resortDir = df.get(resortIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); + + final IndexWriterConfig iwc = new IndexWriterConfig().setIndexSort(indexSort); + // Mirror SolrIndexConfig: a child-doc-capable schema records a parent field, which the source + // segments carry, so the re-sort writer must declare the same parent field or addIndexes + // fails. + if (core.getLatestSchema().isUsableForChildDocs()) { + iwc.setParentField(IndexSchema.IS_ROOT_FIELD_NAME); + } + + try (DirectoryReader reader = DirectoryReader.open(currentDir); + IndexWriter resortWriter = new IndexWriter(resortDir, iwc)) { + final List wrapped = new ArrayList<>(reader.leaves().size()); + for (LeafReaderContext ctx : reader.leaves()) { + // addIndexes does NOT auto-sort (LUCENE-8505); the SortingCodecReader wrap performs the + // (merge-based) re-sort of each segment. + wrapped.add( + SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(ctx.reader()), indexSort)); + } + resortWriter.addIndexes(wrapped.toArray(new CodecReader[0])); + resortWriter.commit(); + } + } finally { + if (currentDir != null) df.release(currentDir); + if (resortDir != null) df.release(resortDir); + } + + if (!core.modifyIndexProps(resortIndexName)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to point core " + core.getName() + " at the re-sorted index"); + } + try { + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + } catch (Exception e) { + log.warn("Could not switch to re-sorted index for core {}; rolling back", core.getName(), e); + rollbackIndexProps(core, df, lockType); + core.getUpdateHandler().newIndexWriter(false); + openNewSearcher(core); + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to swap in the re-sorted index for core " + core.getName(), + e); + } + } + + private void rollbackIndexProps(SolrCore core, DirectoryFactory df, String lockType) { + Directory dataDir = null; + try { + dataDir = df.get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA, lockType); + dataDir.deleteFile(IndexFetcher.INDEX_PROPERTIES); + } catch (Exception rollbackError) { + log.error("Rollback of index.properties failed for core {}", core.getName(), rollbackError); + } finally { + if (dataDir != null) { + try { + df.release(dataDir); + } catch (IOException ignored) { + } + } + } + } + + private void openNewSearcher(SolrCore core) throws Exception { + final Future[] waitSearcher = new Future[1]; + core.getSearcher(true, false, waitSearcher, true); + if (waitSearcher[0] != null) { + waitSearcher[0].get(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java new file mode 100644 index 000000000000..198d24f199b1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexActionTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import static org.hamcrest.CoreMatchers.containsString; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.util.RefCounted; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * End-to-end test of the {@code RESORTINDEX} core-admin action (SOLR-12239): re-sort an existing + * (unsorted) core index and swap it in, so the core serves a sorted index without reindexing. + */ +public class ResortIndexActionTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // Plain config, no index sort configured -> the initial index is UNSORTED. + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void clearIndexBefore() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortIndexAction() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Build an existing unsorted index (intDvoDefault values out of order). + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + } finally { + admin.shutdown(); + admin.close(); + } + + // All docs still present and queryable after the resort+swap. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + + // The index is now physically sorted by intDvoDefault ascending (internal docid order). + RefCounted ref = core.getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + int maxDoc = ref.get().getIndexReader().maxDoc(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < maxDoc; i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue( + "internal docids must be ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals( + "smallest value sorts first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } + + @Test + public void testResortIndexRequiresSortWhenNoneConfigured() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + // No 'sort' param, and this core has no SortingMergePolicy configured -> clear error. + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp)); + assertThat(thrown.getMessage(), containsString("no configured index sort")); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsUnparseableSort() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + assertU(adoc("id", "1", "intDvoDefault", "1")); + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "no_such_field asc"), + resp)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, thrown.code()); + } finally { + admin.shutdown(); + admin.close(); + } + } + + @Test + public void testResortIndexRejectsChildDocs() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index a parent+child block; re-sorting would break the block, so it must be rejected. + SolrInputDocument parent = new SolrInputDocument(); + parent.addField("id", "100"); + parent.addField("title", "parent"); + SolrInputDocument child = new SolrInputDocument(); + child.addField("id", "101"); + child.addField("title", "child"); + parent.addChildDocument(child); + try (SolrQueryRequestBase addReq = + new SolrQueryRequestBase(core, new ModifiableSolrParams()) {}) { + AddUpdateCommand cmd = new AddUpdateCommand(addReq); + cmd.solrDoc = parent; + core.getUpdateHandler().addDoc(cmd); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + SolrException thrown = + assertThrows( + SolrException.class, + () -> + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName, + CommonParams.SORT, + "intDvoDefault asc"), + resp)); + assertThat(thrown.getMessage(), containsString("child/nested documents")); + } finally { + admin.shutdown(); + admin.close(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java new file mode 100644 index 000000000000..d4e32c6ec6be --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredSortTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies that {@code RESORTINDEX} falls back to the sort configured on the core (via a + * SortingMergePolicy) when no explicit {@code sort} parameter is given (SOLR-12239). + */ +public class ResortIndexConfiguredSortTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // This config declares a SortingMergePolicy with sort = "timestamp_i_dvo desc". + initCore("solrconfig-sortingmergepolicyfactory.xml", "schema.xml"); + } + + @Test + public void testResortUsesConfiguredSortWhenNoSortParam() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + for (int i = 0; i < 5; i++) { + assertU(adoc("id", Integer.toString(i), "timestamp_i_dvo", Integer.toString(i))); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + // No 'sort' param -> must fall back to the configured SortingMergePolicy sort. + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + assertEquals(coreName, resp.getValues().get("core")); + // The applied sort should be the one configured on the core (on timestamp_i_dvo), proving the + // fallback used the configured SortingMergePolicy sort rather than requiring a param. + String applied = String.valueOf(resp.getValues().get("indexSort")); + assertTrue( + "expected configured sort (on timestamp_i_dvo) in response, got: " + applied, + applied.contains("timestamp_i_dvo")); + } finally { + admin.shutdown(); + admin.close(); + } + + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='5']"); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java new file mode 100644 index 000000000000..8407320b2758 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ResortCoreIndexTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.ResortCoreIndexRequestBody; +import org.apache.solr.client.api.model.ResortCoreIndexResponse; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests the {@link ResortCoreIndex} V2 API directly (SOLR-12239). */ +public class ResortCoreIndexTest extends SolrTestCaseJ4 { + + private CoreContainer coreContainer; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Before + public void setup() { + coreContainer = h.getCoreContainer(); + assertU(delQ("*:*")); + assertU(commit()); + } + + @After + public void cleanup() { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testResortViaV2Api() throws Exception { + final String coreName = h.getCore().getName(); + int[] vals = {5, 1, 4, 2, 3, 0}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "intDvoDefault", Integer.toString(v))); + } + assertU(commit()); + + final SolrQueryRequest req = req(); + try { + ResortCoreIndex api = + new ResortCoreIndex( + coreContainer, + new CoreAdminHandler.CoreAdminAsyncTracker(), + req, + new SolrQueryResponse()); + ResortCoreIndexRequestBody body = new ResortCoreIndexRequestBody(); + body.sort = "intDvoDefault asc"; + + ResortCoreIndexResponse response = api.resortCoreIndex(coreName, body); + assertEquals(coreName, response.core); + assertNotNull(response.indexSort); + } finally { + req.close(); + } + + // Index is now physically sorted ascending by intDvoDefault. + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='6']"); + var ref = h.getCore().getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + long prev = Long.MIN_VALUE; + for (int i = 0; i < ref.get().getIndexReader().maxDoc(); i++) { + long id = Long.parseLong(storedFields.document(i).get("id")); + assertTrue("ascending after resort: " + prev + " then " + id, id >= prev); + prev = id; + } + assertEquals("smallest first", 0L, Long.parseLong(storedFields.document(0).get("id"))); + } finally { + ref.decref(); + } + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc index a7dff995ee38..9cb6dc69a6eb 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc @@ -898,6 +898,96 @@ Reusing the same `` value across restarts ensures continuity: D http://localhost:8983/solr/_collection_/update?commit=true ---- +[[coreadmin-resortindex]] +== RESORTINDEX + +The `RESORTINDEX` action re-sorts an existing core's index into a given xref:query-guide:common-query-parameters.adoc#sort-parameter[sort] order. +This makes it possible to enable xref:index-segments-merging.adoc[index sorting] on a collection that was created without it, without having to delete the index and reindex from source. + +Enabling an index sort on an existing (unsorted) index otherwise fails on core reload with an `IndexWriter` `validateIndexSort` error, because Lucene rejects the transition from unsorted segments to a configured sort. +`RESORTINDEX` produces a freshly sorted copy of the index (by merging each segment through the target sort) and swaps it in. + +This action is expensive and can take a while to complete on large indexes. +Consider running with the `async` option in such cases. + +Note: + +* Not supported in SolrCloud mode. +* Indexes containing child/nested documents are not supported (re-sorting would break the parent-child document blocks). +* The fields used in the sort must have docValues enabled. + +A typical workflow is to run `RESORTINDEX` on the existing index, then configure the same sort in `solrconfig.xml` (via a xref:index-segments-merging.adoc#mergepolicyfactory[`SortingMergePolicyFactory`]) so that subsequently added documents keep the index sorted. + +It is recommended to have a backup before running on production data. + +[tabs#coreadmin-resortindex-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +http://localhost:8983/solr/admin/cores?action=RESORTINDEX&core=techproducts&sort=timestamp+desc +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl -X POST http://localhost:8983/api/cores/techproducts/resort -H 'Content-Type: application/json' -d ' +{ + "sort": "timestamp desc" +} +' +---- +==== +====== + +=== RESORTINDEX Parameters + +`core`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the core whose index should be re-sorted. + +`sort`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The target index sort, in the usual Solr xref:query-guide:common-query-parameters.adoc#sort-parameter[sort syntax] (for example `timestamp desc`). +If omitted, the sort configured for the core (via a `SortingMergePolicy`) is used; if the core has no configured sort, the request fails. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. +Use <> with the provided `requestid` to poll for completion. + +=== RESORTINDEX Response + +On success, the response includes: + +`core`:: +The core name. + +`indexSort`:: +The index sort that was applied. + +On failure, an exception is thrown with error details. + [[coreadmin-requeststatus]] == REQUESTSTATUS diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java index 0401de380f1b..b3f035c8ac7b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java @@ -179,7 +179,8 @@ public enum CoreAdminAction { CREATESNAPSHOT, DELETESNAPSHOT, LISTSNAPSHOTS, - UPGRADEINDEX; + UPGRADEINDEX, + RESORTINDEX; public final boolean isRead; From 4afa02d3b840500a93a9561737728ad097d54fee Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 13:57:23 +0300 Subject: [PATCH 4/9] SOLR-17310: add to configure Lucene's leaf sorter Exposes Lucene IndexWriterConfig.setLeafSorter via a new element in , so a collection can visit its segments (leaf readers) in creation-time order at search time. This is between-segment ordering, separate from index sorting (within-segment document order) and independent of merging; the main use is early termination on time-based indexes by searching the most recently created segments first. Supported values are TIME_ASC and TIME_DESC; the default (NONE) installs no leaf sorter. The value is parsed and validated when the config is loaded. Segment creation time is read from Lucene's per-segment timestamp diagnostic; a leaf that does not resolve to a SegmentReader is ordered last rather than failing the search. Builds on the approach in the earlier PR #2477 by Wei Wang. --- .../SOLR-17310-configurable-leafsorter.yml | 12 +++ .../org/apache/solr/update/SegmentSort.java | 33 ++++++ .../solr/update/SegmentTimeLeafSorter.java | 76 +++++++++++++ .../apache/solr/update/SolrIndexConfig.java | 29 +++++ .../conf/solrconfig-segmentsort-invalid.xml | 29 +++++ .../conf/solrconfig-segmentsort.xml | 29 +++++ .../update/SegmentTimeLeafSorterTest.java | 101 ++++++++++++++++++ .../solr/update/SolrIndexConfigTest.java | 36 +++++++ .../pages/index-segments-merging.adoc | 18 ++++ 9 files changed, 363 insertions(+) create mode 100644 changelog/unreleased/SOLR-17310-configurable-leafsorter.yml create mode 100644 solr/core/src/java/org/apache/solr/update/SegmentSort.java create mode 100644 solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml create mode 100644 solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java diff --git a/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml new file mode 100644 index 000000000000..7b17fe5f0e3c --- /dev/null +++ b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Add a index configuration option to order segments (leaf readers) by creation time + at search time, exposing Lucene's leaf sorter for early termination on time-based indexes +type: added +authors: + - name: Serhiy Bzhezytskyy + - name: Wei Wang +links: + - name: SOLR-17310 + url: https://issues.apache.org/jira/browse/SOLR-17310 diff --git a/solr/core/src/java/org/apache/solr/update/SegmentSort.java b/solr/core/src/java/org/apache/solr/update/SegmentSort.java new file mode 100644 index 000000000000..d7df14dc5c18 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/SegmentSort.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +/** + * The supported orderings for the {@code } index configuration, which controls the + * order in which a {@link org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) are + * visited at search time. This is "between-segment" ordering, distinct from Lucene's index sort + * (the "within-segment" order of documents), and has no effect on merging. + */ +public enum SegmentSort { + /** Do not impose any ordering on segments (Lucene's default). */ + NONE, + /** Visit older segments (by creation timestamp) first. */ + TIME_ASC, + /** Visit more recently created segments (by creation timestamp) first. */ + TIME_DESC +} diff --git a/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java b/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java new file mode 100644 index 000000000000..34e8dfef9afd --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Comparator; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SegmentReader; + +/** + * Builds a {@link Comparator} that orders a {@link org.apache.lucene.index.DirectoryReader}'s leaf + * readers by the creation timestamp of their underlying segment, for use with {@link + * org.apache.lucene.index.IndexWriterConfig#setLeafSorter(Comparator)}. + * + *

The timestamp is read from the segment's {@link + * org.apache.lucene.index.SegmentInfo#getDiagnostics() diagnostics}, where Lucene records it under + * the {@code timestamp} key at flush and merge time. A leaf whose timestamp cannot be determined + * (for example a reader that does not wrap a {@link SegmentReader}) is ordered last, so an + * unexpected reader shape degrades the ordering rather than failing the search. + */ +final class SegmentTimeLeafSorter { + + /** + * Diagnostics key under which Lucene's IndexWriter records the segment creation time (millis). + */ + private static final String TIMESTAMP_DIAGNOSTIC_KEY = "timestamp"; + + private SegmentTimeLeafSorter() {} + + /** + * Returns the leaf sorter for the given ordering, or null for {@link SegmentSort#NONE} (meaning + * no leaf sorter should be installed). + */ + static Comparator forOrder(SegmentSort order) { + if (order == SegmentSort.NONE) { + return null; + } + final boolean ascending = order == SegmentSort.TIME_ASC; + // Unknown timestamps sort last regardless of direction, so they never displace ordered leaves. + final long missingValue = ascending ? Long.MAX_VALUE : Long.MIN_VALUE; + Comparator byTimestamp = + Comparator.comparingLong(reader -> segmentTimestamp(reader, missingValue)); + return ascending ? byTimestamp : byTimestamp.reversed(); + } + + private static long segmentTimestamp(LeafReader reader, long missingValue) { + LeafReader unwrapped = FilterLeafReader.unwrap(reader); + if (unwrapped instanceof SegmentReader segmentReader) { + String timestamp = + segmentReader.getSegmentInfo().info.getDiagnostics().get(TIMESTAMP_DIAGNOSTIC_KEY); + if (timestamp != null) { + try { + return Long.parseLong(timestamp); + } catch (NumberFormatException e) { + return missingValue; + } + } + } + return missingValue; + } +} diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index d6f0415edc4a..e1d727abb4ca 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -21,12 +21,15 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; +import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.search.Sort; @@ -102,6 +105,9 @@ public class SolrIndexConfig implements MapWriter { public final PluginInfo mergedSegmentWarmerInfo; + /** Order in which segments (leaf readers) are visited at search time; never null. */ + public final SegmentSort segmentSort; + public InfoStream infoStream = InfoStream.NO_OUTPUT; private ConfigNode node; @@ -115,6 +121,7 @@ private SolrIndexConfig() { writeLockTimeout = -1; lockType = DirectoryFactory.LOCK_TYPE_NATIVE; indexSort = null; + segmentSort = SegmentSort.NONE; mergePolicyFactoryInfo = null; mergeSchedulerInfo = null; mergedSegmentWarmerInfo = null; @@ -171,6 +178,7 @@ public SolrIndexConfig(ConfigNode cfg, SolrIndexConfig def) { writeLockTimeout = get("writeLockTimeout").intVal(def.writeLockTimeout); lockType = get("lockType").txt(def.lockType); indexSort = get("indexSort").txt(def.indexSort); + segmentSort = parseSegmentSort(get("segmentSort").txt(null), def.segmentSort); metricsInfo = getPluginInfo(get("metrics"), def.metricsInfo); mergeSchedulerInfo = getPluginInfo(get("mergeScheduler"), def.mergeSchedulerInfo); @@ -217,6 +225,7 @@ public void writeMap(EntryWriter ew) throws IOException { .put("lockType", lockType) .put("infoStreamEnabled", infoStream != InfoStream.NO_OUTPUT) .putIfNotNull("indexSort", indexSort) + .put("segmentSort", segmentSort.name()) .putIfNotNull("mergeScheduler", mergeSchedulerInfo) .putIfNotNull("metrics", metricsInfo) .putIfNotNull("mergePolicyFactory", mergePolicyFactoryInfo) @@ -229,6 +238,21 @@ private PluginInfo getPluginInfo(ConfigNode node, PluginInfo def) { : def; } + private static SegmentSort parseSegmentSort(String value, SegmentSort def) { + if (value == null || value.isBlank()) { + return def; + } + try { + return SegmentSort.valueOf(value.trim()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Invalid value '" + + value + + "'; expected one of " + + Arrays.toString(SegmentSort.values())); + } + } + private static class DelayedSchemaAnalyzer extends DelegatingAnalyzerWrapper { private final SolrCore core; @@ -310,6 +334,11 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergedSegmentWarmer(warmer); } + Comparator leafSorter = SegmentTimeLeafSorter.forOrder(segmentSort); + if (leafSorter != null) { + iwc.setLeafSorter(leafSorter); + } + return iwc; } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml new file mode 100644 index 000000000000..484671bf17da --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-invalid.xml @@ -0,0 +1,29 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + NOT_A_VALID_ORDER + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml new file mode 100644 index 000000000000..7ee7f1927f6f --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort.xml @@ -0,0 +1,29 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + TIME_DESC + + + diff --git a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java new file mode 100644 index 000000000000..1f9a662f0a79 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Comparator; +import java.util.List; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.Directory; +import org.apache.solr.SolrTestCase; + +/** Verifies the leaf (segment) ordering produced by {@link SegmentTimeLeafSorter}. */ +public class SegmentTimeLeafSorterTest extends SolrTestCase { + + public void testNoneReturnsNoSorter() { + assertNull(SegmentTimeLeafSorter.forOrder(SegmentSort.NONE)); + } + + public void testTimeDescVisitsNewestSegmentsFirst() throws Exception { + assertLeafOrder(SegmentSort.TIME_DESC, /* newestFirst= */ true); + } + + public void testTimeAscVisitsOldestSegmentsFirst() throws Exception { + assertLeafOrder(SegmentSort.TIME_ASC, /* newestFirst= */ false); + } + + /** + * Writes three single-document segments (each commit creates its own segment), reopens the reader + * with the leaf sorter for the given order, and asserts the segments are visited by creation time + * as expected. Segment creation time comes from Lucene's per-segment {@code timestamp} + * diagnostic. + */ + private void assertLeafOrder(SegmentSort order, boolean newestFirst) throws Exception { + Comparator leafSorter = SegmentTimeLeafSorter.forOrder(order); + assertNotNull(leafSorter); + + try (Directory dir = newDirectory()) { + // Disable merging so each commit stays its own segment with a distinct timestamp. + IndexWriterConfig iwc = new IndexWriterConfig().setLeafSorter(leafSorter); + iwc.setMergePolicy(org.apache.lucene.index.NoMergePolicy.INSTANCE); + try (IndexWriter writer = new IndexWriter(dir, iwc)) { + for (int i = 0; i < 3; i++) { + Document doc = new Document(); + doc.add(new StringField("id", Integer.toString(i), Store.YES)); + writer.addDocument(doc); + writer.commit(); + // Ensure a strictly increasing per-segment timestamp (diagnostics are millis-resolution). + Thread.sleep(5); + } + } + + try (DirectoryReader reader = DirectoryReader.open(dir, leafSorter)) { + List leaves = reader.leaves(); + assertEquals(3, leaves.size()); + long previous = -1; + for (LeafReaderContext leaf : leaves) { + long timestamp = segmentTimestamp(leaf.reader()); + if (previous >= 0) { + if (newestFirst) { + assertTrue( + "expected descending segment timestamps, got " + previous + " then " + timestamp, + timestamp <= previous); + } else { + assertTrue( + "expected ascending segment timestamps, got " + previous + " then " + timestamp, + timestamp >= previous); + } + } + previous = timestamp; + } + } + } + } + + private static long segmentTimestamp(LeafReader reader) { + SegmentReader segmentReader = (SegmentReader) reader; + return Long.parseLong(segmentReader.getSegmentInfo().info.getDiagnostics().get("timestamp")); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index e4c330bb49be..d2814a3dd139 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -62,6 +62,9 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { private static final String solrConfigFileNameMergeOnFlushMergePolicyFactory = "solrconfig-mergeonflushmergepolicyfactory.xml"; private static final String solrConfigFileNameIndexSort = "solrconfig-indexsort.xml"; + private static final String solrConfigFileNameSegmentSort = "solrconfig-segmentsort.xml"; + private static final String solrConfigFileNameSegmentSortInvalid = + "solrconfig-segmentsort-invalid.xml"; private static final String schemaFileName = "schema.xml"; private static boolean compoundMergePolicySort = false; @@ -298,6 +301,9 @@ public void testToMap() throws Exception { assertFalse(Boolean.valueOf(m.get("infoStreamEnabled").toString())); } + ++mSizeExpected; + assertEquals(SegmentSort.NONE.name(), m.get("segmentSort")); + ++mSizeExpected; assertTrue(m.get("mergeScheduler") instanceof MapWriter); ++mSizeExpected; @@ -314,6 +320,36 @@ public void testToMap() throws Exception { assertEquals(mSizeExpected, m.size()); } + @Test + public void testSegmentSortDefaultsToNone() throws Exception { + SolrConfig solrConfig = + new SolrConfig(instanceDir, solrConfigFileNameSortingMergePolicyFactory); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertEquals(SegmentSort.NONE, solrIndexConfig.segmentSort); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + assertNull("no leaf sorter should be installed by default", iwc.getLeafSorter()); + } + + @Test + public void testSegmentSortConfigured() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSegmentSort); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertEquals(SegmentSort.TIME_DESC, solrIndexConfig.segmentSort); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + assertNotNull("configured segmentSort should install a leaf sorter", iwc.getLeafSorter()); + } + + @Test + public void testSegmentSortInvalidValueFailsFast() { + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> + new SolrIndexConfig( + new SolrConfig(instanceDir, solrConfigFileNameSegmentSortInvalid), null)); + assertTrue(e.getMessage(), e.getMessage().contains("segmentSort")); + } + public void testMaxCommitMergeWaitTime() throws Exception { SolrConfig sc = new SolrConfig(TEST_PATH().resolve("collection1"), "solrconfig-test-misc.xml"); assertEquals(-1, sc.indexConfig.maxCommitMergeWaitMillis); diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index 8d5785c309f0..7164db136d8e 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -268,6 +268,24 @@ This is the direct replacement for configuring the sort indirectly through a `So NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the `RESORTINDEX` core admin action. It is not applied to existing segments automatically on reload. +=== segmentSort + +Controls the order in which a segment's documents are visited at search time, by sorting the leaf readers of a newly opened reader. +This is _between-segment_ ordering; it is independent of index sorting (the _within-segment_ order of documents) and does not affect merging. + +The main use case is early termination: visiting the most promising segments first. +For a time-based index, ordering by segment creation time so that the most recently created segments are searched first lets a query that only needs recent documents stop sooner. + +By default no ordering is imposed. The supported values are: + +* `TIME_ASC`: visit older segments (by creation time) first. +* `TIME_DESC`: visit more recently created segments first. + +[source,xml] +---- +TIME_DESC +---- + == Compound File Segments Each Lucene segment is typically comprised of a dozen or so files. From c4df2284d495cf81da7a578dfb4cda84c024b0de Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 14:48:55 +0300 Subject: [PATCH 5/9] SOLR-12230: deprecate SortingMergePolicy in favor of With index sorting now configurable directly via in (SOLR-13681), wrapping a merge policy solely to carry a Sort to the index writer is no longer needed. Deprecate both SortingMergePolicy and its SortingMergePolicyFactory, pointing users to . The classes still function (and are still used internally by index re-sorting, so remaining references are marked @SuppressWarnings("deprecation")); this only signals the configuration approach that should replace them. --- .../SOLR-12230-deprecate-sortingmergepolicy.yml | 11 +++++++++++ .../org/apache/solr/index/SortingMergePolicy.java | 11 ++++++++--- .../apache/solr/index/SortingMergePolicyFactory.java | 8 +++++++- .../org/apache/solr/update/DefaultSolrCoreState.java | 1 + .../java/org/apache/solr/update/SolrIndexConfig.java | 1 + .../apache/solr/update/SegmentTimeLeafSorterTest.java | 3 ++- .../org/apache/solr/update/SolrIndexConfigTest.java | 3 +++ 7 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml diff --git a/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml b/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml new file mode 100644 index 000000000000..33c359b108e1 --- /dev/null +++ b/changelog/unreleased/SOLR-12230-deprecate-sortingmergepolicy.yml @@ -0,0 +1,11 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Deprecate SortingMergePolicyFactory in favor of configuring the index sort directly with + in +type: changed +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-12230 + url: https://issues.apache.org/jira/browse/SOLR-12230 diff --git a/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java b/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java index 3b648921a42d..7e8fa82101c0 100644 --- a/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java +++ b/solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java @@ -20,9 +20,14 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.search.Sort; -// TODO: remove this and add indexSort specification directly to solrconfig.xml? But for BWC, also -// accept SortingMergePolicy specifiction? - +/** + * A {@link MergePolicy} that does no merging of its own and only carries a {@link Sort} to the + * index writer. + * + * @deprecated Configure the index sort directly with {@code } in {@code } + * instead of wrapping a merge policy to carry a sort. + */ +@Deprecated public final class SortingMergePolicy extends FilterMergePolicy { private final Sort sort; diff --git a/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java index cf9705049c21..009af32d9f46 100644 --- a/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java +++ b/solr/core/src/java/org/apache/solr/index/SortingMergePolicyFactory.java @@ -24,7 +24,13 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SortSpecParsing; -/** A {@link MergePolicyFactory} for {@code SortingMergePolicy} objects. */ +/** + * A {@link MergePolicyFactory} for {@code SortingMergePolicy} objects. + * + * @deprecated Configure the index sort directly with {@code } in {@code } + * instead. This factory only carries a sort to the index writer and does no merging of its own. + */ +@Deprecated public class SortingMergePolicyFactory extends WrapperMergePolicyFactory { protected final Sort mergeSort; diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 9a2e7e96dd56..a527f4c2c716 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -269,6 +269,7 @@ private SolrIndexWriter createMainIndexWriter(SolrCore core, String name) throws } @Override + @SuppressWarnings("deprecation") // still reads the sort from a (deprecated) SortingMergePolicy public Sort getMergePolicySort() throws IOException { lock(iwLock.readLock()); try { diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index e1d727abb4ca..07d37103f7ad 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -267,6 +267,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { } } + @SuppressWarnings("deprecation") // reconciles with a (deprecated) SortingMergePolicy if present public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { IndexSchema schema = core.getLatestSchema(); IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core)); diff --git a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java index 1f9a662f0a79..03a3f6cd4b7f 100644 --- a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java +++ b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.solr.SolrTestCase; @@ -59,7 +60,7 @@ private void assertLeafOrder(SegmentSort order, boolean newestFirst) throws Exce try (Directory dir = newDirectory()) { // Disable merging so each commit stays its own segment with a distinct timestamp. IndexWriterConfig iwc = new IndexWriterConfig().setLeafSorter(leafSorter); - iwc.setMergePolicy(org.apache.lucene.index.NoMergePolicy.INSTANCE); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); try (IndexWriter writer = new IndexWriter(dir, iwc)) { for (int i = 0; i < 3; i++) { Document doc = new Document(); diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index d2814a3dd139..3d32406edef4 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -140,6 +140,7 @@ public void testConcurrentMergeSchedularSolrIndexConfigCreation() throws Excepti assertTrue("ms.isAutoIOThrottle", ms.getAutoIOThrottle()); } + @SuppressWarnings("deprecation") // exercises the deprecated SortingMergePolicy path public void testSortingMPSolrIndexConfigCreation() throws Exception { final SortField sortField1 = new SortField("timestamp_i_dvo", SortField.Type.INT, true); final SortField sortField2 = new SortField("id", SortField.Type.STRING, false); @@ -364,6 +365,8 @@ public void testMaxCommitMergeWaitTime() throws Exception { } @Test + @SuppressWarnings( + "deprecation") // asserts the merge policy is NOT a (deprecated) SortingMergePolicy public void testDirectIndexSortConfig() throws Exception { // The compound-sort randomization in @BeforeClass may add a secondary "id asc" sort field. final SortField primary = new SortField("timestamp_i_dvo", SortField.Type.INT, true); From 939b8cad02e68f7179b16d08ee16a28759822ee7 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 15:33:45 +0300 Subject: [PATCH 6/9] Index sorting: address review findings across the feature - RESORTINDEX now falls back to the configured (SOLR-13681), not only a SortingMergePolicy, so the migration workflow (configure , then RESORTINDEX with no sort param) works end to end. - RESORTINDEX sets the core codec and schema similarity on the re-sort writer, so addIndexes no longer rewrites a per-field index with Lucene defaults; and it cleans up the old index dir on success / the orphaned resort dir on rollback. - fails fast with a clear error naming the element when the configured spec is not a usable index sort. - Tests: invalid ; wins over a SortingMergePolicy sort; RESORTINDEX using a configured ; a deterministic segmentTerminateEarly run against ; SegmentTimeLeafSorter tolerates a non-SegmentReader leaf (sorts last). - Docs: RESORTINDEX points at instead of the deprecated SortingMergePolicyFactory; a Major Changes in Solr 11 upgrade note for the deprecation; and a note distinguishing from . --- .../handler/admin/api/ResortCoreIndex.java | 47 ++++++++- .../apache/solr/search/SolrIndexSearcher.java | 3 + .../apache/solr/update/SolrIndexConfig.java | 4 + .../solrconfig-indexsort-and-mergepolicy.xml | 37 +++++++ .../conf/solrconfig-indexsort-invalid.xml | 30 ++++++ .../apache/solr/cloud/TestSegmentSorting.java | 24 ++++- .../ResortIndexConfiguredIndexSortTest.java | 96 +++++++++++++++++++ .../update/SegmentTimeLeafSorterTest.java | 34 +++++++ .../solr/update/SolrIndexConfigTest.java | 34 +++++++ .../pages/coreadmin-api.adoc | 5 +- .../pages/index-segments-merging.adoc | 4 +- .../pages/major-changes-in-solr-11.adoc | 43 +++++++++ .../pages/solr-upgrade-notes.adoc | 3 +- 13 files changed, 353 insertions(+), 11 deletions(-) create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java create mode 100644 solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java index 2e350072ebef..3d3babf8b6a4 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ResortCoreIndex.java @@ -123,14 +123,29 @@ private Sort resolveTargetSort(SolrCore core, String sortParam) throws IOExcepti } return sort; } - final Sort configured = core.getSolrCoreState().getMergePolicySort(); - if (configured == null) { + // Fall back to the sort configured for the core: the directly configured + // (preferred), or a SortingMergePolicy's sort (deprecated). This mirrors how the searcher + // resolves the index sort, so the migration workflow "set , then RESORTINDEX" works. + final String configuredSpec = core.getSolrConfig().indexConfig.indexSort; + if (configuredSpec != null && !configuredSpec.isBlank()) { + final Sort sort = + SortSpecParsing.parseSortSpec(configuredSpec, core.getLatestSchema()).getSort(); + if (sort == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Could not parse a usable index sort from the configured =" + + configuredSpec); + } + return sort; + } + final Sort mergePolicySort = core.getSolrCoreState().getMergePolicySort(); + if (mergePolicySort == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No 'sort' parameter given and the core has no configured index sort " - + "(SortingMergePolicy) to fall back to"); + + "(via or a SortingMergePolicy) to fall back to"); } - return configured; + return mergePolicySort; } private void assertNoChildDocs(SolrCore core) throws IOException { @@ -175,6 +190,11 @@ private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { resortDir = df.get(resortIndexPath, DirectoryFactory.DirContext.DEFAULT, lockType); final IndexWriterConfig iwc = new IndexWriterConfig().setIndexSort(indexSort); + // addIndexes rewrites every segment through this writer, so it must use the core's codec and + // similarity; otherwise a per-field (e.g. SchemaCodecFactory) index would be silently + // rewritten with Lucene defaults. + iwc.setCodec(core.getCodec()); + iwc.setSimilarity(core.getLatestSchema().getSimilarity()); // Mirror SolrIndexConfig: a child-doc-capable schema records a parent field, which the source // segments carry, so the re-sort writer must declare the same parent field or addIndexes // fails. @@ -207,11 +227,15 @@ private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { try { core.getUpdateHandler().newIndexWriter(false); openNewSearcher(core); + // The core now points at the re-sorted index; drop the previous (unsorted) index directory. + core.cleanupOldIndexDirectories(false); } catch (Exception e) { log.warn("Could not switch to re-sorted index for core {}; rolling back", core.getName(), e); rollbackIndexProps(core, df, lockType); core.getUpdateHandler().newIndexWriter(false); openNewSearcher(core); + // Discard the orphaned re-sort directory we just wrote but never swapped in. + removeDirectory(df, resortIndexPath, lockType); throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Failed to swap in the re-sorted index for core " + core.getName(), @@ -219,6 +243,21 @@ private void resortAndSwap(SolrCore core, Sort indexSort) throws Exception { } } + private void removeDirectory(DirectoryFactory df, String path, String lockType) { + try { + if (df.exists(path)) { + Directory dir = df.get(path, DirectoryFactory.DirContext.DEFAULT, lockType); + try { + df.remove(dir, true); + } finally { + df.release(dir); + } + } + } catch (Exception cleanupError) { + log.warn("Could not remove re-sort directory {}", path, cleanupError); + } + } + private void rollbackIndexProps(SolrCore core, DirectoryFactory df, String lockType) { Directory dataDir = null; try { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index e05f34bf6f76..da402ef5f9d3 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -365,8 +365,11 @@ private Collector buildAndRunCollectorChain( private Sort configuredIndexSort() throws IOException { String indexSortSpec = core.getSolrConfig().indexConfig.indexSort; if (indexSortSpec != null) { + // Parsed per request, but only on the opt-in segmentTerminateEarly path, so the cost is + // negligible; parsing here also avoids caching a sort across schema reloads. return SortSpecParsing.parseSortSpec(indexSortSpec, core.getLatestSchema()).getSort(); } + // TODO: the SortingMergePolicy fallback can be removed once SortingMergePolicy is (SOLR-12230). return core.getSolrCoreState().getMergePolicySort(); } diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 07d37103f7ad..490a41eda7cf 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -297,6 +297,10 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { Sort configuredIndexSort = null; if (indexSort != null) { configuredIndexSort = SortSpecParsing.parseSortSpec(indexSort, schema).getSort(); + if (configuredIndexSort == null) { + throw new IllegalArgumentException( + "Could not parse a usable index sort from " + indexSort + ""); + } iwc.setIndexSort(configuredIndexSort); } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml new file mode 100644 index 000000000000..4909542666ca --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-mergepolicy.xml @@ -0,0 +1,37 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + in + org.apache.solr.index.TieredMergePolicyFactory + timestamp_i_dvo asc + + timestamp_i_dvo desc + ${solr.tests.lockType:single} + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml new file mode 100644 index 000000000000..0aea805fc2cc --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-invalid.xml @@ -0,0 +1,30 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + score desc + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java index beceb7f075c0..26753e9b4c2a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java @@ -70,10 +70,13 @@ public void createCollection() throws Exception { // Exercise both ways of configuring the index sort: indirectly via a SortingMergePolicy, and // directly via the element (SOLR-13681). Both must produce the same behavior. + // testSegmentTerminateEarlyWithIndexSort pins the config deterministically. final String solrConfigFileName = - random().nextBoolean() - ? "solrconfig-sortingmergepolicyfactory.xml" - : "solrconfig-indexsort.xml"; + testName.getMethodName().contains("WithIndexSort") + ? "solrconfig-indexsort.xml" + : (random().nextBoolean() + ? "solrconfig-sortingmergepolicyfactory.xml" + : "solrconfig-indexsort.xml"); final Map collectionProperties = new HashMap<>(); collectionProperties.put(CoreDescriptor.CORE_CONFIG, solrConfigFileName); @@ -92,7 +95,22 @@ public void createCollection() throws Exception { cluster.waitForActiveCollection(collectionName, NUM_SHARDS, NUM_SHARDS * REPLICATION_FACTOR); } + /** Runs the full early-termination scenario against whichever index-sort config was chosen. */ public void testSegmentTerminateEarly() throws Exception { + doTestSegmentTerminateEarly(); + } + + /** + * Same scenario, but deterministically against a directly configured {@code } (no + * SortingMergePolicy), so a regression in the {@code } branch of early-termination + * eligibility cannot pass CI on a coin flip (SOLR-13681 / SOLR-15390). + */ + @Test + public void testSegmentTerminateEarlyWithIndexSort() throws Exception { + doTestSegmentTerminateEarly(); + } + + private void doTestSegmentTerminateEarly() throws Exception { final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState(random()); final CloudSolrClient cloudSolrClient = cluster.getSolrClient(); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java new file mode 100644 index 000000000000..4bf2be33be97 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/ResortIndexConfiguredIndexSortTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Verifies the SOLR-13681/SOLR-12239 migration workflow: with the index sort configured directly + * via {@code }, {@code RESORTINDEX} (no {@code sort} param) picks up that configured + * sort and physically re-sorts the existing (unsorted) segments. + */ +public class ResortIndexConfiguredIndexSortTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + // This config declares timestamp_i_dvo desc and no SortingMergePolicy. + initCore("solrconfig-indexsort.xml", "schema.xml"); + } + + @Test + public void testResortUsesConfiguredIndexSortWhenNoSortParam() throws Exception { + final SolrCore core = h.getCore(); + final String coreName = core.getName(); + + // Index documents in an order that does NOT match the configured desc sort. id == timestamp so + // the stored id doubles as the (docValues-only) timestamp for verification below. + int[] vals = {2, 0, 4, 1, 3}; + for (int v : vals) { + assertU(adoc("id", Integer.toString(v), "timestamp_i_dvo", Integer.toString(v))); + } + assertU(commit()); + + CoreAdminHandler admin = new CoreAdminHandler(h.getCoreContainer()); + try { + final SolrQueryResponse resp = new SolrQueryResponse(); + // No 'sort' param -> must fall back to the configured , not error out. + admin.handleRequestBody( + req( + CoreAdminParams.ACTION, + CoreAdminParams.CoreAdminAction.RESORTINDEX.toString(), + CoreAdminParams.CORE, + coreName), + resp); + assertNull("Unexpected exception: " + resp.getException(), resp.getException()); + String applied = String.valueOf(resp.getValues().get("indexSort")); + assertTrue( + "expected the configured (on timestamp_i_dvo) in response, got: " + applied, + applied.contains("timestamp_i_dvo")); + } finally { + admin.shutdown(); + admin.close(); + } + + assertQ(req("q", "*:*", "rows", "0"), "//result[@numFound='5']"); + + // The index is now physically sorted by timestamp_i_dvo descending (internal docid order), + // which is what lets a matching query terminate early per segment. + RefCounted ref = core.getSearcher(); + try { + var storedFields = ref.get().getIndexReader().storedFields(); + int maxDoc = ref.get().getIndexReader().maxDoc(); + long prev = Long.MAX_VALUE; + for (int i = 0; i < maxDoc; i++) { + // id == timestamp_i_dvo in this data; id is stored, the timestamp is docValues-only. + long ts = Long.parseLong(storedFields.document(i).get("id")); + assertTrue( + "internal docids must be timestamp-descending after resort: " + prev + " then " + ts, + ts <= prev); + prev = ts; + } + } finally { + ref.decref(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java index 03a3f6cd4b7f..c40d849338cd 100644 --- a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java +++ b/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java @@ -28,6 +28,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.ParallelLeafReader; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; import org.apache.solr.SolrTestCase; @@ -39,6 +40,39 @@ public void testNoneReturnsNoSorter() { assertNull(SegmentTimeLeafSorter.forOrder(SegmentSort.NONE)); } + /** + * A leaf that does not resolve to a SegmentReader must be tolerated (ordered last), not crash the + * comparator. A {@link ParallelLeafReader} extends {@link LeafReader} directly (it is not a + * FilterLeafReader), so it does not unwrap to a SegmentReader and has no segment timestamp. + */ + public void testNonSegmentReaderSortsLastWithoutFailing() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) { + Document doc = new Document(); + doc.add(new StringField("id", "0", Store.YES)); + writer.addDocument(doc); + writer.commit(); + } + try (DirectoryReader reader = DirectoryReader.open(dir)) { + LeafReader segmentLeaf = reader.leaves().get(0).reader(); + try (LeafReader nonSegmentLeaf = new ParallelLeafReader(false, segmentLeaf)) { + for (SegmentSort order : + new SegmentSort[] {SegmentSort.TIME_ASC, SegmentSort.TIME_DESC}) { + Comparator sorter = SegmentTimeLeafSorter.forOrder(order); + assertNotNull(sorter); + // Must not throw for either ordering, regardless of argument order. + sorter.compare(segmentLeaf, nonSegmentLeaf); + sorter.compare(nonSegmentLeaf, segmentLeaf); + // The known-timestamp segment sorts before the unknown one in both directions. + assertTrue( + "segment with a timestamp should sort before one without (" + order + ")", + sorter.compare(segmentLeaf, nonSegmentLeaf) < 0); + } + } + } + } + } + public void testTimeDescVisitsNewestSegmentsFirst() throws Exception { assertLeafOrder(SegmentSort.TIME_DESC, /* newestFirst= */ true); } diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index 3d32406edef4..6b5f2f87efef 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -62,6 +62,10 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 { private static final String solrConfigFileNameMergeOnFlushMergePolicyFactory = "solrconfig-mergeonflushmergepolicyfactory.xml"; private static final String solrConfigFileNameIndexSort = "solrconfig-indexsort.xml"; + private static final String solrConfigFileNameIndexSortInvalid = + "solrconfig-indexsort-invalid.xml"; + private static final String solrConfigFileNameIndexSortAndMergePolicy = + "solrconfig-indexsort-and-mergepolicy.xml"; private static final String solrConfigFileNameSegmentSort = "solrconfig-segmentsort.xml"; private static final String solrConfigFileNameSegmentSortInvalid = "solrconfig-segmentsort-invalid.xml"; @@ -397,4 +401,34 @@ public void testNoIndexSortByDefault() throws Exception { h.getCore().setLatestSchema(indexSchema); assertNull(solrIndexConfig.toIndexWriterConfig(h.getCore()).getIndexSort()); } + + @Test + public void testInvalidIndexSortFailsWithClearError() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSortInvalid); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, () -> solrIndexConfig.toIndexWriterConfig(h.getCore())); + assertTrue(e.getMessage(), e.getMessage().contains("")); + } + + @Test + @SuppressWarnings( + "deprecation") // configures a (deprecated) SortingMergePolicy alongside + public void testIndexSortWinsOverSortingMergePolicy() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameIndexSortAndMergePolicy); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + + // is "desc" and the SortingMergePolicy sort is "asc"; must win. + final Sort expected = new Sort(new SortField("timestamp_i_dvo", SortField.Type.INT, true)); + assertEquals( + " should win over the SortingMergePolicy sort", expected, iwc.getIndexSort()); + // both were configured, so the merge policy is still a SortingMergePolicy + assertTrue(iwc.getMergePolicy() instanceof SortingMergePolicy); + } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc index 9cb6dc69a6eb..cee12d66b28b 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/coreadmin-api.adoc @@ -916,7 +916,8 @@ Note: * Indexes containing child/nested documents are not supported (re-sorting would break the parent-child document blocks). * The fields used in the sort must have docValues enabled. -A typical workflow is to run `RESORTINDEX` on the existing index, then configure the same sort in `solrconfig.xml` (via a xref:index-segments-merging.adoc#mergepolicyfactory[`SortingMergePolicyFactory`]) so that subsequently added documents keep the index sorted. +A typical workflow is to configure the sort in `solrconfig.xml` with xref:index-segments-merging.adoc#indexsort[``] (so subsequently added documents keep the index sorted), then run `RESORTINDEX` once to re-sort the pre-existing segments. +When `` is configured, `RESORTINDEX` may be called without a `sort` parameter and will use the configured sort. It is recommended to have a backup before running on production data. @@ -964,7 +965,7 @@ The name of the core whose index should be re-sorted. |=== + The target index sort, in the usual Solr xref:query-guide:common-query-parameters.adoc#sort-parameter[sort syntax] (for example `timestamp desc`). -If omitted, the sort configured for the core (via a `SortingMergePolicy`) is used; if the core has no configured sort, the request fails. +If omitted, the sort configured for the core is used -- the xref:index-segments-merging.adoc#indexsort[``] element if present, otherwise a (deprecated) `SortingMergePolicy` sort; if the core has no configured sort, the request fails. `async`:: + diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index 7164db136d8e..09c9e1a7a190 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -266,7 +266,9 @@ This is the direct replacement for configuring the sort indirectly through a `So timestamp_dt desc ---- -NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the `RESORTINDEX` core admin action. It is not applied to existing segments automatically on reload. +NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the xref:coreadmin-api.adoc#coreadmin-resortindex[`RESORTINDEX`] core admin action. It is not applied to existing segments automatically on reload. + +TIP: Do not confuse `` with the similarly named <>. `` is _within-segment_ document order and takes a sort spec (e.g. `timestamp_dt desc`); `` is _between-segment_ visitation order and takes an enum value. === segmentSort diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc new file mode 100644 index 000000000000..f63b9dfe430d --- /dev/null +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-11.adoc @@ -0,0 +1,43 @@ += Major Changes in Solr 11 +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr 11.0 is a major new release of Solr. + +This page highlights the most important changes including new features and changes in default behavior as well as previously deprecated features that have now been removed. + +== Solr 11 Upgrade Planning + +Before starting an upgrade to this version of Solr, please be sure to review all information about changes from the version you are currently on up to this one, to include the minor version number changes as well. + +== Index Sorting Configuration + +Index sorting can now be configured directly with the xref:configuration-guide:index-segments-merging.adoc#indexsort[``] element in the `` section of `solrconfig.xml`, instead of indirectly through a `SortingMergePolicyFactory`. + +As a result, `SortingMergePolicyFactory` (and the underlying `SortingMergePolicy`) are now *deprecated*. +Configurations that use `SortingMergePolicyFactory` continue to work, but should migrate to ``: + +[source,xml] +---- + + timestamp_dt desc + +---- + +To enable (or change) index sorting on a collection that already has data, use the xref:configuration-guide:coreadmin-api.adoc#coreadmin-resortindex[`RESORTINDEX`] core admin action to re-sort the existing segments; index sorting is not applied to already-written segments automatically on reload. + +The related between-segment ordering can be configured with the new xref:configuration-guide:index-segments-merging.adoc#segmentsort[``] element. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc index 061aea238e4a..2d51b6c54859 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc @@ -1,5 +1,6 @@ = Solr Upgrade Notes -:page-children: major-changes-in-solr-10, \ +:page-children: major-changes-in-solr-11, \ + major-changes-in-solr-10, \ major-changes-in-solr-9, \ major-changes-in-solr-8, \ major-changes-in-solr-7, \ From e26945bf90009a8b04745faeafba1a47954d4896 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 16:12:52 +0300 Subject: [PATCH 7/9] SOLR-17310: generalize to also order by a numeric docValues field now accepts either a time preset (TIME_ASC/TIME_DESC, by segment creation time) or a field spec (" asc|desc") that orders segments by the per-segment min (asc) or max (desc) of a single-valued numeric docValues field. Lucene's setLeafSorter takes an arbitrary Comparator, so this is not a Lucene limitation -- it broadens what the config exposes. SegmentSort becomes a small parsed spec (NONE/TIME/FIELD); SegmentTimeLeafSorter is renamed to SegmentLeafSorter and builds the comparator for both forms, with the per-segment key memoized on the segment core cache key (the comparator runs O(n log n) times per reader open) and unknown keys sorting last. Field sorts are validated at config load (must be an existing single-valued numeric docValues field). Comparing the raw docValues long is order-correct for all numeric types because Solr stores float/double docValues in sortable-bits form. Tests: field-sort comparator build + spec round-trip; rejection of unknown, non-numeric, non-docValues, and multi-valued fields; garbage spec parsing. Docs updated for both forms. --- .../SOLR-17310-configurable-leafsorter.yml | 5 +- .../apache/solr/update/SegmentLeafSorter.java | 191 ++++++++++++++++++ .../org/apache/solr/update/SegmentSort.java | 110 +++++++++- .../solr/update/SegmentTimeLeafSorter.java | 76 ------- .../apache/solr/update/SolrIndexConfig.java | 23 +-- ...erTest.java => SegmentLeafSorterTest.java} | 58 ++++-- .../solr/update/SegmentSortFieldTest.java | 97 +++++++++ .../solr/update/SolrIndexConfigTest.java | 6 +- .../pages/index-segments-merging.adoc | 16 +- 9 files changed, 447 insertions(+), 135 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java delete mode 100644 solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java rename solr/core/src/test/org/apache/solr/update/{SegmentTimeLeafSorterTest.java => SegmentLeafSorterTest.java} (67%) create mode 100644 solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java diff --git a/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml index 7b17fe5f0e3c..a04533eb6a83 100644 --- a/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml +++ b/changelog/unreleased/SOLR-17310-configurable-leafsorter.yml @@ -1,8 +1,9 @@ # See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc title: > - Add a index configuration option to order segments (leaf readers) by creation time - at search time, exposing Lucene's leaf sorter for early termination on time-based indexes + Add a index configuration option that orders segments (leaf readers) at search + time -- by creation time (TIME_ASC/TIME_DESC) or by a numeric docValues field (" asc|desc") + -- exposing Lucene's leaf sorter for early termination type: added authors: - name: Serhiy Bzhezytskyy diff --git a/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java new file mode 100644 index 000000000000..a08810c9a38a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; + +/** + * Builds the {@link Comparator} for the {@code } configuration, ordering a {@link + * org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) at open time via {@link + * org.apache.lucene.index.IndexWriterConfig#setLeafSorter(Comparator)}. + * + *

For a {@link SegmentSort.Kind#TIME} sort, the key is the segment creation time (from Lucene's + * per-segment {@code timestamp} diagnostic). For a {@link SegmentSort.Kind#FIELD} sort, the key is + * the per-segment minimum ({@code asc}) or maximum ({@code desc}) of a numeric docValues field; + * Solr stores numeric docValues in sortable-bits form, so comparing the raw {@code long} preserves + * true numeric order for all numeric types. A leaf whose key cannot be determined (e.g. a reader + * that does not resolve to a {@link SegmentReader}, or a segment with no value for the field) sorts + * last, so an unexpected reader shape degrades ordering rather than failing the search. + */ +final class SegmentLeafSorter { + + private static final String TIMESTAMP_DIAGNOSTIC_KEY = "timestamp"; + + private SegmentLeafSorter() {} + + /** + * Returns the leaf sorter for the given config, or null for {@link SegmentSort.Kind#NONE} (no + * leaf sorter installed). For a field sort the field must exist and be a single-valued numeric + * docValues field. + */ + static Comparator forConfig(SegmentSort segmentSort, IndexSchema schema) { + switch (segmentSort.kind()) { + case NONE: + return null; + case TIME: + return build(segmentSort.descending(), SegmentLeafSorter::segmentTimestamp); + case FIELD: + final String field = validateFieldForSort(segmentSort.field(), schema); + final boolean descending = segmentSort.descending(); + // asc orders by each segment's min value; desc by its max value. + return build(descending, leaf -> segmentFieldExtremum(leaf, field, descending)); + default: + return null; + } + } + + private static String validateFieldForSort(String field, IndexSchema schema) { + SchemaField sf = schema.getFieldOrNull(field); + if (sf == null) { + throw new IllegalArgumentException( + " field '" + field + "' does not exist in the schema"); + } + if (!sf.hasDocValues()) { + throw new IllegalArgumentException( + " field '" + field + "' must have docValues=\"true\""); + } + if (sf.getType().getNumberType() == null) { + throw new IllegalArgumentException( + " field '" + field + "' must be a numeric field"); + } + if (sf.multiValued()) { + throw new IllegalArgumentException( + " field '" + field + "' must be single-valued"); + } + return field; + } + + /** + * Builds a comparator over the given per-leaf key, memoizing the key per segment (the comparator + * is invoked O(n log n) times while the reader is opened). Unknown keys sort last in both + * directions. + */ + private static Comparator build(boolean descending, KeyFn keyFn) { + final long missingValue = descending ? Long.MIN_VALUE : Long.MAX_VALUE; + final Map cache = new ConcurrentHashMap<>(); + Comparator byKey = + Comparator.comparingLong( + leaf -> { + Object cacheKey = coreCacheKey(leaf); + if (cacheKey == null) { + return safeKey(keyFn, leaf, missingValue); + } + return cache.computeIfAbsent(cacheKey, k -> safeKey(keyFn, leaf, missingValue)); + }); + return descending ? byKey.reversed() : byKey; + } + + private static long safeKey(KeyFn keyFn, LeafReader leaf, long missingValue) { + try { + Long key = keyFn.key(leaf); + return key == null ? missingValue : key; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static Object coreCacheKey(LeafReader leaf) { + var helper = leaf.getCoreCacheHelper(); + return helper == null ? null : helper.getKey(); + } + + private static Long segmentTimestamp(LeafReader reader) { + LeafReader unwrapped = FilterLeafReader.unwrap(reader); + if (unwrapped instanceof SegmentReader segmentReader) { + String timestamp = + segmentReader.getSegmentInfo().info.getDiagnostics().get(TIMESTAMP_DIAGNOSTIC_KEY); + if (timestamp != null) { + try { + return Long.parseLong(timestamp); + } catch (NumberFormatException ignored) { + return null; + } + } + } + return null; + } + + /** + * Per-segment min (asc) or max (desc) of the field's numeric docValues, as sortable-bits long. + */ + private static Long segmentFieldExtremum(LeafReader leaf, String field, boolean max) + throws IOException { + // Solr writes single-valued numeric docValues; read either representation defensively. + NumericDocValues numeric = leaf.getNumericDocValues(field); + if (numeric != null) { + return scan(max, numeric::nextDoc, numeric::longValue); + } + SortedNumericDocValues sorted = leaf.getSortedNumericDocValues(field); + if (sorted != null) { + // Single-valued, so exactly one value per doc via nextValue(). + return scan(max, sorted::nextDoc, () -> sorted.nextValue()); + } + return null; + } + + private static Long scan(boolean max, DocIterator advance, ValueSupplier value) + throws IOException { + long extremum = max ? Long.MIN_VALUE : Long.MAX_VALUE; + boolean any = false; + for (int doc = advance.nextDoc(); doc != NO_MORE; doc = advance.nextDoc()) { + long v = value.get(); + extremum = max ? Math.max(extremum, v) : Math.min(extremum, v); + any = true; + } + return any ? extremum : null; + } + + private static final int NO_MORE = DocIdSetIterator.NO_MORE_DOCS; + + @FunctionalInterface + private interface KeyFn { + Long key(LeafReader leaf) throws IOException; + } + + @FunctionalInterface + private interface DocIterator { + int nextDoc() throws IOException; + } + + @FunctionalInterface + private interface ValueSupplier { + long get() throws IOException; + } +} diff --git a/solr/core/src/java/org/apache/solr/update/SegmentSort.java b/solr/core/src/java/org/apache/solr/update/SegmentSort.java index d7df14dc5c18..32f0f1599d4c 100644 --- a/solr/core/src/java/org/apache/solr/update/SegmentSort.java +++ b/solr/core/src/java/org/apache/solr/update/SegmentSort.java @@ -17,17 +17,105 @@ package org.apache.solr.update; +import java.util.Locale; + /** - * The supported orderings for the {@code } index configuration, which controls the - * order in which a {@link org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) are - * visited at search time. This is "between-segment" ordering, distinct from Lucene's index sort - * (the "within-segment" order of documents), and has no effect on merging. + * Parsed value of the {@code } index configuration, which controls the order in which + * a {@link org.apache.lucene.index.DirectoryReader}'s leaf readers (segments) are visited at search + * time (Lucene's "leaf sorter"). This is "between-segment" ordering, distinct from index sort (the + * "within-segment" order of documents), and has no effect on merging. + * + *

Two forms are accepted: + * + *

*/ -public enum SegmentSort { - /** Do not impose any ordering on segments (Lucene's default). */ - NONE, - /** Visit older segments (by creation timestamp) first. */ - TIME_ASC, - /** Visit more recently created segments (by creation timestamp) first. */ - TIME_DESC +public final class SegmentSort { + + /** No segment ordering is imposed (Lucene's default). */ + public static final SegmentSort NONE = new SegmentSort(Kind.NONE, false, null); + + /** How segments are ordered. */ + public enum Kind { + NONE, + TIME, + FIELD + } + + private final Kind kind; + private final boolean descending; + private final String field; // non-null only for Kind.FIELD + + private SegmentSort(Kind kind, boolean descending, String field) { + this.kind = kind; + this.descending = descending; + this.field = field; + } + + public Kind kind() { + return kind; + } + + public boolean descending() { + return descending; + } + + /** The field name for a {@link Kind#FIELD} sort; null otherwise. */ + public String field() { + return field; + } + + /** + * Parses a {@code } value. Returns {@link #NONE} for null/blank. Accepts the presets + * {@code TIME_ASC}/{@code TIME_DESC} (case-insensitive), or a {@code asc|desc} spec. + * + * @throws IllegalArgumentException if the value is neither a known preset nor a valid {@code + * asc|desc} spec + */ + public static SegmentSort parse(String value) { + if (value == null || value.isBlank()) { + return NONE; + } + final String trimmed = value.trim(); + final String upper = trimmed.toUpperCase(Locale.ROOT); + if (upper.equals("NONE")) { + return NONE; + } + if (upper.equals("TIME_ASC")) { + return new SegmentSort(Kind.TIME, false, null); + } + if (upper.equals("TIME_DESC")) { + return new SegmentSort(Kind.TIME, true, null); + } + // Otherwise expect " asc|desc". + final String[] parts = trimmed.split("\\s+"); + if (parts.length == 2) { + final String order = parts[1].toLowerCase(Locale.ROOT); + if (order.equals("asc") || order.equals("desc")) { + return new SegmentSort(Kind.FIELD, order.equals("desc"), parts[0]); + } + } + throw new IllegalArgumentException( + "Invalid value '" + + value + + "'; expected NONE, TIME_ASC, TIME_DESC, or ' asc|desc'"); + } + + @Override + public String toString() { + switch (kind) { + case NONE: + return "NONE"; + case TIME: + return descending ? "TIME_DESC" : "TIME_ASC"; + case FIELD: + return field + (descending ? " desc" : " asc"); + default: + return kind.name(); + } + } } diff --git a/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java b/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java deleted file mode 100644 index 34e8dfef9afd..000000000000 --- a/solr/core/src/java/org/apache/solr/update/SegmentTimeLeafSorter.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.update; - -import java.util.Comparator; -import org.apache.lucene.index.FilterLeafReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.SegmentReader; - -/** - * Builds a {@link Comparator} that orders a {@link org.apache.lucene.index.DirectoryReader}'s leaf - * readers by the creation timestamp of their underlying segment, for use with {@link - * org.apache.lucene.index.IndexWriterConfig#setLeafSorter(Comparator)}. - * - *

The timestamp is read from the segment's {@link - * org.apache.lucene.index.SegmentInfo#getDiagnostics() diagnostics}, where Lucene records it under - * the {@code timestamp} key at flush and merge time. A leaf whose timestamp cannot be determined - * (for example a reader that does not wrap a {@link SegmentReader}) is ordered last, so an - * unexpected reader shape degrades the ordering rather than failing the search. - */ -final class SegmentTimeLeafSorter { - - /** - * Diagnostics key under which Lucene's IndexWriter records the segment creation time (millis). - */ - private static final String TIMESTAMP_DIAGNOSTIC_KEY = "timestamp"; - - private SegmentTimeLeafSorter() {} - - /** - * Returns the leaf sorter for the given ordering, or null for {@link SegmentSort#NONE} (meaning - * no leaf sorter should be installed). - */ - static Comparator forOrder(SegmentSort order) { - if (order == SegmentSort.NONE) { - return null; - } - final boolean ascending = order == SegmentSort.TIME_ASC; - // Unknown timestamps sort last regardless of direction, so they never displace ordered leaves. - final long missingValue = ascending ? Long.MAX_VALUE : Long.MIN_VALUE; - Comparator byTimestamp = - Comparator.comparingLong(reader -> segmentTimestamp(reader, missingValue)); - return ascending ? byTimestamp : byTimestamp.reversed(); - } - - private static long segmentTimestamp(LeafReader reader, long missingValue) { - LeafReader unwrapped = FilterLeafReader.unwrap(reader); - if (unwrapped instanceof SegmentReader segmentReader) { - String timestamp = - segmentReader.getSegmentInfo().info.getDiagnostics().get(TIMESTAMP_DIAGNOSTIC_KEY); - if (timestamp != null) { - try { - return Long.parseLong(timestamp); - } catch (NumberFormatException e) { - return missingValue; - } - } - } - return missingValue; - } -} diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 490a41eda7cf..c9ce693f3822 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; -import java.util.Arrays; import java.util.Comparator; import java.util.Map; import org.apache.lucene.analysis.Analyzer; @@ -178,7 +177,8 @@ public SolrIndexConfig(ConfigNode cfg, SolrIndexConfig def) { writeLockTimeout = get("writeLockTimeout").intVal(def.writeLockTimeout); lockType = get("lockType").txt(def.lockType); indexSort = get("indexSort").txt(def.indexSort); - segmentSort = parseSegmentSort(get("segmentSort").txt(null), def.segmentSort); + String segmentSortSpec = get("segmentSort").txt(null); + segmentSort = segmentSortSpec == null ? def.segmentSort : SegmentSort.parse(segmentSortSpec); metricsInfo = getPluginInfo(get("metrics"), def.metricsInfo); mergeSchedulerInfo = getPluginInfo(get("mergeScheduler"), def.mergeSchedulerInfo); @@ -225,7 +225,7 @@ public void writeMap(EntryWriter ew) throws IOException { .put("lockType", lockType) .put("infoStreamEnabled", infoStream != InfoStream.NO_OUTPUT) .putIfNotNull("indexSort", indexSort) - .put("segmentSort", segmentSort.name()) + .put("segmentSort", segmentSort.toString()) .putIfNotNull("mergeScheduler", mergeSchedulerInfo) .putIfNotNull("metrics", metricsInfo) .putIfNotNull("mergePolicyFactory", mergePolicyFactoryInfo) @@ -238,21 +238,6 @@ private PluginInfo getPluginInfo(ConfigNode node, PluginInfo def) { : def; } - private static SegmentSort parseSegmentSort(String value, SegmentSort def) { - if (value == null || value.isBlank()) { - return def; - } - try { - return SegmentSort.valueOf(value.trim()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid value '" - + value - + "'; expected one of " - + Arrays.toString(SegmentSort.values())); - } - } - private static class DelayedSchemaAnalyzer extends DelegatingAnalyzerWrapper { private final SolrCore core; @@ -339,7 +324,7 @@ public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { iwc.setMergedSegmentWarmer(warmer); } - Comparator leafSorter = SegmentTimeLeafSorter.forOrder(segmentSort); + Comparator leafSorter = SegmentLeafSorter.forConfig(segmentSort, schema); if (leafSorter != null) { iwc.setLeafSorter(leafSorter); } diff --git a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java b/solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java similarity index 67% rename from solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java rename to solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java index c40d849338cd..03f3577ca014 100644 --- a/solr/core/src/test/org/apache/solr/update/SegmentTimeLeafSorterTest.java +++ b/solr/core/src/test/org/apache/solr/update/SegmentLeafSorterTest.java @@ -33,11 +33,20 @@ import org.apache.lucene.store.Directory; import org.apache.solr.SolrTestCase; -/** Verifies the leaf (segment) ordering produced by {@link SegmentTimeLeafSorter}. */ -public class SegmentTimeLeafSorterTest extends SolrTestCase { +/** + * Verifies the time-based leaf (segment) ordering produced by {@link SegmentLeafSorter}. The + * field-based ordering is exercised where a real schema/core is available (see {@code + * SegmentSortFieldTest}). + */ +public class SegmentLeafSorterTest extends SolrTestCase { public void testNoneReturnsNoSorter() { - assertNull(SegmentTimeLeafSorter.forOrder(SegmentSort.NONE)); + assertNull(SegmentLeafSorter.forConfig(SegmentSort.NONE, null)); + } + + public void testParseRejectsGarbage() { + expectThrows(IllegalArgumentException.class, () -> SegmentSort.parse("nonsense value here")); + expectThrows(IllegalArgumentException.class, () -> SegmentSort.parse("field bogusorder")); } /** @@ -46,26 +55,40 @@ public void testNoneReturnsNoSorter() { * FilterLeafReader), so it does not unwrap to a SegmentReader and has no segment timestamp. */ public void testNonSegmentReaderSortsLastWithoutFailing() throws Exception { - try (Directory dir = newDirectory()) { - try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) { + try (Directory dirA = newDirectory(); + Directory dirB = newDirectory()) { + try (IndexWriter writer = new IndexWriter(dirA, new IndexWriterConfig())) { Document doc = new Document(); doc.add(new StringField("id", "0", Store.YES)); writer.addDocument(doc); writer.commit(); } - try (DirectoryReader reader = DirectoryReader.open(dir)) { - LeafReader segmentLeaf = reader.leaves().get(0).reader(); - try (LeafReader nonSegmentLeaf = new ParallelLeafReader(false, segmentLeaf)) { - for (SegmentSort order : - new SegmentSort[] {SegmentSort.TIME_ASC, SegmentSort.TIME_DESC}) { - Comparator sorter = SegmentTimeLeafSorter.forOrder(order); + // A second index with a disjoint field, so the two leaves can be combined into a + // multi-reader ParallelLeafReader (which reports no core cache key). + try (IndexWriter writer = new IndexWriter(dirB, new IndexWriterConfig())) { + Document doc = new Document(); + doc.add(new StringField("other", "0", Store.YES)); + writer.addDocument(doc); + writer.commit(); + } + try (DirectoryReader readerA = DirectoryReader.open(dirA); + DirectoryReader readerB = DirectoryReader.open(dirB)) { + LeafReader segmentLeaf = readerA.leaves().get(0).reader(); + LeafReader otherSegmentLeaf = readerB.leaves().get(0).reader(); + // A multi-reader ParallelLeafReader does not resolve to a single SegmentReader (and reports + // no core cache key), so it has no segment timestamp and must sort last. + try (LeafReader nonSegmentLeaf = + new ParallelLeafReader(false, segmentLeaf, otherSegmentLeaf)) { + for (String spec : new String[] {"TIME_ASC", "TIME_DESC"}) { + Comparator sorter = + SegmentLeafSorter.forConfig(SegmentSort.parse(spec), null); assertNotNull(sorter); // Must not throw for either ordering, regardless of argument order. sorter.compare(segmentLeaf, nonSegmentLeaf); sorter.compare(nonSegmentLeaf, segmentLeaf); // The known-timestamp segment sorts before the unknown one in both directions. assertTrue( - "segment with a timestamp should sort before one without (" + order + ")", + "segment with a timestamp should sort before one without (" + spec + ")", sorter.compare(segmentLeaf, nonSegmentLeaf) < 0); } } @@ -74,21 +97,20 @@ public void testNonSegmentReaderSortsLastWithoutFailing() throws Exception { } public void testTimeDescVisitsNewestSegmentsFirst() throws Exception { - assertLeafOrder(SegmentSort.TIME_DESC, /* newestFirst= */ true); + assertLeafOrder("TIME_DESC", /* newestFirst= */ true); } public void testTimeAscVisitsOldestSegmentsFirst() throws Exception { - assertLeafOrder(SegmentSort.TIME_ASC, /* newestFirst= */ false); + assertLeafOrder("TIME_ASC", /* newestFirst= */ false); } /** * Writes three single-document segments (each commit creates its own segment), reopens the reader * with the leaf sorter for the given order, and asserts the segments are visited by creation time - * as expected. Segment creation time comes from Lucene's per-segment {@code timestamp} - * diagnostic. + * as expected. */ - private void assertLeafOrder(SegmentSort order, boolean newestFirst) throws Exception { - Comparator leafSorter = SegmentTimeLeafSorter.forOrder(order); + private void assertLeafOrder(String spec, boolean newestFirst) throws Exception { + Comparator leafSorter = SegmentLeafSorter.forConfig(SegmentSort.parse(spec), null); assertNotNull(leafSorter); try (Directory dir = newDirectory()) { diff --git a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java new file mode 100644 index 000000000000..02740a36c1eb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.Comparator; +import org.apache.lucene.index.LeafReader; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.schema.IndexSchema; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Verifies field-based {@code } ordering and its validation (SOLR-17310). */ +public class SegmentSortFieldTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + private IndexSchema schema() { + return h.getCore().getLatestSchema(); + } + + @Test + public void testFieldSortBuildsAComparator() { + // timestamp_i_dvo is a single-valued numeric docValues field. + Comparator asc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo asc"), schema()); + Comparator desc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo desc"), schema()); + assertNotNull(asc); + assertNotNull(desc); + } + + @Test + public void testFieldSortRoundTripsSpec() { + SegmentSort s = SegmentSort.parse("timestamp_i_dvo desc"); + assertEquals(SegmentSort.Kind.FIELD, s.kind()); + assertTrue(s.descending()); + assertEquals("timestamp_i_dvo", s.field()); + assertEquals("timestamp_i_dvo desc", s.toString()); + } + + @Test + public void testUnknownFieldRejected() { + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("no_such_field asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("does not exist")); + } + + @Test + public void testNonNumericFieldRejected() { + // *_s_dvo is a string docValues field (has docValues, but is not numeric). + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("category_s_dvo asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("numeric")); + } + + @Test + public void testNonDocValuesFieldRejected() { + // 'name' is indexed/stored text without docValues. + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("name asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("docValues")); + } + + @Test + public void testMultiValuedFieldRejected() { + // *_ii_dvo is a multi-valued numeric docValues field. + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> SegmentLeafSorter.forConfig(SegmentSort.parse("nums_ii_dvo asc"), schema())); + assertTrue(e.getMessage(), e.getMessage().contains("single-valued")); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index 6b5f2f87efef..defbf889b4ec 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -307,7 +307,7 @@ public void testToMap() throws Exception { } ++mSizeExpected; - assertEquals(SegmentSort.NONE.name(), m.get("segmentSort")); + assertEquals("NONE", m.get("segmentSort")); ++mSizeExpected; assertTrue(m.get("mergeScheduler") instanceof MapWriter); @@ -330,7 +330,7 @@ public void testSegmentSortDefaultsToNone() throws Exception { SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSortingMergePolicyFactory); SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); - assertEquals(SegmentSort.NONE, solrIndexConfig.segmentSort); + assertEquals(SegmentSort.Kind.NONE, solrIndexConfig.segmentSort.kind()); IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); assertNull("no leaf sorter should be installed by default", iwc.getLeafSorter()); } @@ -339,7 +339,7 @@ public void testSegmentSortDefaultsToNone() throws Exception { public void testSegmentSortConfigured() throws Exception { SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSegmentSort); SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); - assertEquals(SegmentSort.TIME_DESC, solrIndexConfig.segmentSort); + assertEquals("TIME_DESC", solrIndexConfig.segmentSort.toString()); IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); assertNotNull("configured segmentSort should install a leaf sorter", iwc.getLeafSorter()); } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc index 09c9e1a7a190..332b2ea5bcec 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc @@ -268,26 +268,30 @@ This is the direct replacement for configuring the sort indirectly through a `So NOTE: Enabling index sorting on a collection that already has unsorted segments requires re-sorting those segments; see the xref:coreadmin-api.adoc#coreadmin-resortindex[`RESORTINDEX`] core admin action. It is not applied to existing segments automatically on reload. -TIP: Do not confuse `` with the similarly named <>. `` is _within-segment_ document order and takes a sort spec (e.g. `timestamp_dt desc`); `` is _between-segment_ visitation order and takes an enum value. +TIP: Do not confuse `` with the similarly named <>. `` sets the _within-segment_ document order (it changes how documents are physically stored, at flush/merge); `` only sets the _between-segment_ visitation order at search time and does not affect what is written. === segmentSort Controls the order in which a segment's documents are visited at search time, by sorting the leaf readers of a newly opened reader. This is _between-segment_ ordering; it is independent of index sorting (the _within-segment_ order of documents) and does not affect merging. -The main use case is early termination: visiting the most promising segments first. -For a time-based index, ordering by segment creation time so that the most recently created segments are searched first lets a query that only needs recent documents stop sooner. +The main use case is early termination: visiting the most promising segments first, so a query that only needs the first page of results can stop sooner. -By default no ordering is imposed. The supported values are: +By default no ordering is imposed. Two forms are supported: -* `TIME_ASC`: visit older segments (by creation time) first. -* `TIME_DESC`: visit more recently created segments first. +* A *time preset* -- `TIME_ASC` or `TIME_DESC` -- orders segments by their creation time (a segment property). `TIME_DESC` visits the most recently written segments first, which suits time-based / append-mostly indexes. +* A *field spec* -- ` asc|desc` -- orders segments by the minimum (`asc`) or maximum (`desc`) value of a single-valued numeric docValues field. This is useful when the useful documents cluster by a field value rather than by write time. [source,xml] ---- TIME_DESC ---- +[source,xml] +---- +price_i desc +---- + == Compound File Segments Each Lucene segment is typically comprised of a dozen or so files. From d47d905b324bb47c6be373fb54712068f0d8df67 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 17:04:25 +0300 Subject: [PATCH 8/9] SOLR-17310: fix field-based segmentSort ordering for float/double and multi-value legacy segments Single-valued FLOAT/DOUBLE docValues are stored by Solr as raw Float.floatToIntBits/Double.doubleToLongBits (NOT sortable bits), so the previous raw-long min/max ordered negative values incorrectly. Normalize each value to an order-preserving long via toSortableLong (int/long/date pass through; float/double are decoded then re-encoded to sortable bits). Also, for a legacy segment where the field was multi-valued, take the last (largest) value for a desc/max scan rather than only the first. Adds a regression test (two all-negative segments where raw floatToIntBits order diverges from numeric order) proven to fail without the fix. --- .../apache/solr/update/SegmentLeafSorter.java | 69 +++++++++++++++---- .../solr/update/SegmentSortFieldTest.java | 53 ++++++++++++++ 2 files changed, 107 insertions(+), 15 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java index a08810c9a38a..d9fcbc3abd1c 100644 --- a/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java +++ b/solr/core/src/java/org/apache/solr/update/SegmentLeafSorter.java @@ -28,7 +28,9 @@ import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.NumericUtils; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.NumberType; import org.apache.solr.schema.SchemaField; /** @@ -38,11 +40,12 @@ * *

For a {@link SegmentSort.Kind#TIME} sort, the key is the segment creation time (from Lucene's * per-segment {@code timestamp} diagnostic). For a {@link SegmentSort.Kind#FIELD} sort, the key is - * the per-segment minimum ({@code asc}) or maximum ({@code desc}) of a numeric docValues field; - * Solr stores numeric docValues in sortable-bits form, so comparing the raw {@code long} preserves - * true numeric order for all numeric types. A leaf whose key cannot be determined (e.g. a reader - * that does not resolve to a {@link SegmentReader}, or a segment with no value for the field) sorts - * last, so an unexpected reader shape degrades ordering rather than failing the search. + * the per-segment minimum ({@code asc}) or maximum ({@code desc}) of a numeric docValues field, + * with each value normalized to an order-preserving long (see {@code toSortableLong}) so the raw + * {@code long} comparison matches true numeric order for every numeric type. A leaf whose key + * cannot be determined (e.g. a reader that does not resolve to a {@link SegmentReader}, or a + * segment with no value for the field) sorts last, so an unexpected reader shape degrades ordering + * rather than failing the search. (A genuine I/O error while reading docValues still propagates.) */ final class SegmentLeafSorter { @@ -62,16 +65,18 @@ static Comparator forConfig(SegmentSort segmentSort, IndexSchema sch case TIME: return build(segmentSort.descending(), SegmentLeafSorter::segmentTimestamp); case FIELD: - final String field = validateFieldForSort(segmentSort.field(), schema); + final SchemaField sf = validateFieldForSort(segmentSort.field(), schema); + final String field = sf.getName(); + final NumberType numberType = sf.getType().getNumberType(); final boolean descending = segmentSort.descending(); // asc orders by each segment's min value; desc by its max value. - return build(descending, leaf -> segmentFieldExtremum(leaf, field, descending)); + return build(descending, leaf -> segmentFieldExtremum(leaf, field, numberType, descending)); default: return null; } } - private static String validateFieldForSort(String field, IndexSchema schema) { + private static SchemaField validateFieldForSort(String field, IndexSchema schema) { SchemaField sf = schema.getFieldOrNull(field); if (sf == null) { throw new IllegalArgumentException( @@ -89,7 +94,7 @@ private static String validateFieldForSort(String field, IndexSchema schema) { throw new IllegalArgumentException( " field '" + field + "' must be single-valued"); } - return field; + return sf; } /** @@ -145,21 +150,55 @@ private static Long segmentTimestamp(LeafReader reader) { /** * Per-segment min (asc) or max (desc) of the field's numeric docValues, as sortable-bits long. */ - private static Long segmentFieldExtremum(LeafReader leaf, String field, boolean max) - throws IOException { - // Solr writes single-valued numeric docValues; read either representation defensively. + private static Long segmentFieldExtremum( + LeafReader leaf, String field, NumberType numberType, boolean max) throws IOException { + // Solr writes single-valued numeric docValues as NumericDocValues (older indexes may use + // SortedNumericDocValues); read either. Values are normalized to an order-preserving + // ("sortable") + // long before comparison -- see toSortableLong. NumericDocValues numeric = leaf.getNumericDocValues(field); if (numeric != null) { - return scan(max, numeric::nextDoc, numeric::longValue); + return scan(max, numeric::nextDoc, () -> toSortableLong(numeric.longValue(), numberType)); } SortedNumericDocValues sorted = leaf.getSortedNumericDocValues(field); if (sorted != null) { - // Single-valued, so exactly one value per doc via nextValue(). - return scan(max, sorted::nextDoc, () -> sorted.nextValue()); + // Normally single-valued (one value per doc). A legacy segment written while the field was + // multi-valued may have more; values come in ascending order, so take the first for a min + // scan (asc) and the last for a max scan (desc). + return scan(max, sorted::nextDoc, () -> toSortableLong(docValue(sorted, max), numberType)); } return null; } + private static long docValue(SortedNumericDocValues sorted, boolean max) throws IOException { + int count = sorted.docValueCount(); + long value = sorted.nextValue(); + if (max) { + for (int i = 1; i < count; i++) { + value = sorted.nextValue(); + } + } + return value; + } + + /** + * Normalizes a raw docValues long into an order-preserving long so a raw {@code long} min/max + * matches the field's numeric order. Single-valued FLOAT/DOUBLE docValues are stored by Solr as + * raw {@link Float#floatToIntBits}/{@link Double#doubleToLongBits} (not sortable bits), which do + * not order correctly for negative values, so they are decoded and re-encoded to sortable bits. + * INTEGER/LONG/DATE are already order-correct as signed longs. + */ + private static long toSortableLong(long raw, NumberType numberType) { + switch (numberType) { + case FLOAT: + return NumericUtils.floatToSortableInt(Float.intBitsToFloat((int) raw)); + case DOUBLE: + return NumericUtils.doubleToSortableLong(Double.longBitsToDouble(raw)); + default: // INTEGER, LONG, DATE + return raw; + } + } + private static Long scan(boolean max, DocIterator advance, ValueSupplier value) throws IOException { long extremum = max ? Long.MIN_VALUE : Long.MAX_VALUE; diff --git a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java index 02740a36c1eb..789228033dce 100644 --- a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java +++ b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java @@ -18,9 +18,14 @@ package org.apache.solr.update; import java.util.Comparator; +import java.util.List; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; import org.junit.BeforeClass; import org.junit.Test; @@ -94,4 +99,52 @@ public void testMultiValuedFieldRejected() { () -> SegmentLeafSorter.forConfig(SegmentSort.parse("nums_ii_dvo asc"), schema())); assertTrue(e.getMessage(), e.getMessage().contains("single-valued")); } + + /** + * A single-valued float field with negative values orders correctly. This is a regression guard: + * Solr stores single-valued float docValues as raw {@link Float#floatToIntBits} (not sortable + * bits), so a naive raw-long comparison would order negatives incorrectly. + */ + @Test + public void testFloatFieldWithNegativesOrdersByValue() throws Exception { + // *_f_p is a single-valued pfloat with docValues always enabled. + // Two all-negative segments chosen so raw floatToIntBits order DIVERGES from true numeric + // order: + // floatToIntBits(-8.0) > floatToIntBits(-2.0), so a naive raw-long compare would rank the + // segment holding -8.0 as "larger" and sort it AFTER the -2.0 segment, which is wrong. + assertU(adoc("id", "1", "price_f_p", "-2.0")); + assertU(commit()); // segment A (min = -2.0) + assertU(adoc("id", "2", "price_f_p", "-8.0")); + assertU(commit()); // segment B (min = -8.0) + + Comparator asc = + SegmentLeafSorter.forConfig(SegmentSort.parse("price_f_p asc"), schema()); + assertNotNull(asc); + + RefCounted ref = h.getCore().getSearcher(); + try { + List leaves = ref.get().getIndexReader().leaves(); + assumeTrue("test needs two segments", leaves.size() == 2); + LeafReader a = leafContaining(leaves, "1"); // -2.0 + LeafReader b = leafContaining(leaves, "2"); // -8.0 + // asc orders by each segment's min: B(-8.0) < A(-2.0), so B must sort before A. + assertTrue( + "segment with the smaller (more negative) value must sort first (asc)", + asc.compare(b, a) < 0); + assertTrue(asc.compare(a, b) > 0); + } finally { + ref.decref(); + } + } + + private static LeafReader leafContaining(List leaves, String id) + throws Exception { + for (LeafReaderContext ctx : leaves) { + var terms = ctx.reader().terms("id"); + if (terms != null && terms.iterator().seekExact(new BytesRef(id))) { + return ctx.reader(); + } + } + throw new AssertionError("no leaf contains id=" + id); + } } From db0aed12fe4c71c840b0cf6e49988837e5d1b1fe Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Fri, 17 Jul 2026 17:24:09 +0300 Subject: [PATCH 9/9] Index sorting: consistency-review polish - RESORTINDEX request-body schema now documents the -first fallback (matching resolveTargetSort and the ref-guide), and the two API model classes get class javadoc. - Fix a half-updated TODO wording in SolrIndexSearcher. - Tests: naming a nonexistent field fails when the writer config is built; int field ordering both directions against a live core searcher; and configured together. --- .../api/model/ResortCoreIndexRequestBody.java | 5 +-- .../api/model/ResortCoreIndexResponse.java | 1 + .../apache/solr/search/SolrIndexSearcher.java | 3 +- .../solrconfig-indexsort-and-segmentsort.xml | 32 +++++++++++++++++++ .../conf/solrconfig-segmentsort-badfield.xml | 31 ++++++++++++++++++ .../solr/update/SegmentSortFieldTest.java | 32 ++++++++++++++----- .../solr/update/SolrIndexConfigTest.java | 28 ++++++++++++++++ 7 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java index 4dfb2fa180b4..6df9919f2526 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexRequestBody.java @@ -19,13 +19,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; +/** Request body for the {@code RESORTINDEX} core-admin action. */ public class ResortCoreIndexRequestBody { @Schema( description = "The target index sort, in Solr sort syntax (e.g. 'timestamp desc'). If omitted, the " - + "sort configured for the core (via a SortingMergePolicy) is used. Fields must have " - + "docValues.") + + "sort configured for the core is used: the element if present, " + + "otherwise a (deprecated) SortingMergePolicy sort. Fields must have docValues.") @JsonProperty public String sort; diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java index dce2fd337f3e..a978210f05f4 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/ResortCoreIndexResponse.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; +/** Response for the {@code RESORTINDEX} core-admin action. */ public class ResortCoreIndexResponse extends SolrJerseyResponse { @Schema(description = "The name of the core whose index was re-sorted.") diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index da402ef5f9d3..30295d4bb2bc 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -369,7 +369,8 @@ private Sort configuredIndexSort() throws IOException { // negligible; parsing here also avoids caching a sort across schema reloads. return SortSpecParsing.parseSortSpec(indexSortSpec, core.getLatestSchema()).getSort(); } - // TODO: the SortingMergePolicy fallback can be removed once SortingMergePolicy is (SOLR-12230). + // TODO: the SortingMergePolicy fallback can be removed once SortingMergePolicy is (SOLR-12230) + // removed. return core.getSolrCoreState().getMergePolicySort(); } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml new file mode 100644 index 000000000000..9defdb0ab85e --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-indexsort-and-segmentsort.xml @@ -0,0 +1,32 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + timestamp_i_dvo desc + TIME_DESC + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml new file mode 100644 index 000000000000..794a8ca05a59 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-segmentsort-badfield.xml @@ -0,0 +1,31 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + no_such_field asc + + + diff --git a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java index 789228033dce..3241feccc0ba 100644 --- a/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java +++ b/solr/core/src/test/org/apache/solr/update/SegmentSortFieldTest.java @@ -42,14 +42,30 @@ private IndexSchema schema() { } @Test - public void testFieldSortBuildsAComparator() { - // timestamp_i_dvo is a single-valued numeric docValues field. - Comparator asc = - SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo asc"), schema()); - Comparator desc = - SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo desc"), schema()); - assertNotNull(asc); - assertNotNull(desc); + public void testIntFieldOrdersBothDirections() throws Exception { + // Two segments: A holds 3, B holds 7 (single-valued int docValues). + assertU(adoc("id", "1", "timestamp_i_dvo", "3")); + assertU(commit()); // segment A + assertU(adoc("id", "2", "timestamp_i_dvo", "7")); + assertU(commit()); // segment B + + RefCounted ref = h.getCore().getSearcher(); + try { + List leaves = ref.get().getIndexReader().leaves(); + assumeTrue("test needs two segments", leaves.size() == 2); + LeafReader a = leafContaining(leaves, "1"); // 3 + LeafReader b = leafContaining(leaves, "2"); // 7 + + Comparator asc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo asc"), schema()); + Comparator desc = + SegmentLeafSorter.forConfig(SegmentSort.parse("timestamp_i_dvo desc"), schema()); + // asc: smaller (3) first -> A before B. desc: larger (7) first -> B before A. + assertTrue("asc: 3 before 7", asc.compare(a, b) < 0); + assertTrue("desc: 7 before 3", desc.compare(b, a) < 0); + } finally { + ref.decref(); + } } @Test diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java index defbf889b4ec..a2d18b02acd6 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java @@ -414,6 +414,34 @@ public void testInvalidIndexSortFailsWithClearError() throws Exception { assertTrue(e.getMessage(), e.getMessage().contains("")); } + @Test + public void testSegmentSortBadFieldFailsWhenBuildingWriterConfig() throws Exception { + // Syntax is valid so parse() succeeds, but the field does not exist; the field is validated + // when the IndexWriterConfig is built. + SolrConfig solrConfig = new SolrConfig(instanceDir, "solrconfig-segmentsort-badfield.xml"); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, () -> solrIndexConfig.toIndexWriterConfig(h.getCore())); + assertTrue(e.getMessage(), e.getMessage().contains("does not exist")); + } + + @Test + public void testIndexSortAndSegmentSortTogether() throws Exception { + SolrConfig solrConfig = new SolrConfig(instanceDir, "solrconfig-indexsort-and-segmentsort.xml"); + SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null); + assertNotNull("indexSort should be configured", solrIndexConfig.indexSort); + assertEquals("TIME_DESC", solrIndexConfig.segmentSort.toString()); + IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig); + h.getCore().setLatestSchema(indexSchema); + IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore()); + // Both knobs take effect independently: index sort (within-segment) and leaf sorter (between). + assertNotNull("index sort should be set", iwc.getIndexSort()); + assertNotNull("leaf sorter should be set", iwc.getLeafSorter()); + } + @Test @SuppressWarnings( "deprecation") // configures a (deprecated) SortingMergePolicy alongside