diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 51e38d4dfd7..7b969f90990 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -238,6 +238,9 @@ Improvements * SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown (hossman) +* SOLR-17187: The polling interval for PULL and TLOG replicas can now be overridden using the `commitPollInterval` setting, which takes a String + formatted as "HH:mm:ss" (Torsten Koster via Christine Poerschke, Jason Gerlowski) + Optimizations --------------------- * SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac) diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java index 06bbbefedbe..7cc932365e1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java @@ -75,16 +75,9 @@ public void startReplication(boolean switchTransactionLog) { "SolrCore not found:" + coreName + " in " + CloudUtil.getLoadedCoreNamesAsString(cc)); } } - SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo(); - String pollIntervalStr = "00:00:03"; - if (System.getProperty("jetty.testMode") != null) { - pollIntervalStr = "00:00:01"; - } - String calculatedPollIntervalString = determinePollInterval(uinfo); - if (calculatedPollIntervalString != null) { - pollIntervalStr = calculatedPollIntervalString; - } + final SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo(); + final String pollIntervalStr = determinePollInterval(uinfo); log.info("Will start replication from leader with poll interval: {}", pollIntervalStr); NamedList followerConfig = new NamedList<>(); @@ -134,18 +127,25 @@ public static String getCommitVersion(SolrCore solrCore) { } /** - * Determine the poll interval for replicas based on the auto soft/hard commit schedule + * Determine the poll interval for replicas based on the auto soft/hard commit schedule or + * configured commit poll interval * * @param uinfo the update handler info containing soft/hard commit configuration * @return a poll interval string representing a cadence of polling frequency in the form of - * hh:mm:ss + * hh:mm:ss, never null */ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) { int hardCommitMaxTime = uinfo.autoCommmitMaxTime; int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime; boolean hardCommitNewSearcher = uinfo.openSearcher; - String pollIntervalStr = null; - if (hardCommitMaxTime != -1) { + String customCommitPollInterval = uinfo.commitPollInterval; + String pollIntervalStr = "00:00:03"; + + if (System.getProperty("jetty.testMode") != null) { + pollIntervalStr = "00:00:01"; + } else if (customCommitPollInterval != null) { + pollIntervalStr = customCommitPollInterval; + } else if (hardCommitMaxTime != -1) { // configured hardCommit places a ceiling on the interval at which new segments will be // available to replicate if (softCommitMaxTime != -1 @@ -168,6 +168,9 @@ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) { pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2); } + // validate poll interval and fail early + ReplicationHandler.readIntervalNs(pollIntervalStr); + assert pollIntervalStr != null; return pollIntervalStr; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 4034fb42e2a..4d7962a5ea5 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -800,6 +800,7 @@ public static class UpdateHandlerInfo implements MapSerializable { public final long autoCommitMaxSizeBytes; public final boolean openSearcher; // is opening a new searcher part of hard autocommit? public final boolean commitWithinSoftCommit; + public final String commitPollInterval; public final boolean aggregateNodeLevelMetricsEnabled; /** @@ -815,7 +816,8 @@ public UpdateHandlerInfo( boolean openSearcher, int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, - boolean commitWithinSoftCommit) { + boolean commitWithinSoftCommit, + String commitPollInterval) { this.className = className; this.autoCommmitMaxDocs = autoCommmitMaxDocs; this.autoCommmitMaxTime = autoCommmitMaxTime; @@ -826,6 +828,7 @@ public UpdateHandlerInfo( this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime; this.commitWithinSoftCommit = commitWithinSoftCommit; + this.commitPollInterval = commitPollInterval; this.aggregateNodeLevelMetricsEnabled = false; } @@ -841,6 +844,7 @@ public UpdateHandlerInfo(ConfigNode updateHandler) { this.autoSoftCommmitMaxTime = updateHandler.get("autoSoftCommit").get("maxTime").intVal(-1); this.commitWithinSoftCommit = updateHandler.get("commitWithin").get("softCommit").boolVal(true); + this.commitPollInterval = updateHandler.get("commitPollInterval").txt(); this.aggregateNodeLevelMetricsEnabled = updateHandler.boolAttr("aggregateNodeLevelMetricsEnabled", false); } @@ -857,6 +861,7 @@ public Map toMap(Map map) { map.put( "autoSoftCommit", Map.of("maxDocs", autoSoftCommmitMaxDocs, "maxTime", autoSoftCommmitMaxTime)); + map.put("commitPollInterval", commitPollInterval); return map; } } diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index c506a875294..8367e9166f9 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1490,7 +1490,7 @@ private Long readIntervalMs(String interval) { return TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); } - private Long readIntervalNs(String interval) { + public static Long readIntervalNs(String interval) { if (interval == null) return null; int result = 0; Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java index 723eda5874d..aa4ae37e1c5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java @@ -17,55 +17,88 @@ package org.apache.solr.cloud; -import static org.junit.Assert.assertEquals; - +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.core.SolrConfig; import org.junit.Test; -public class ReplicateFromLeaderTest { +public class ReplicateFromLeaderTest extends SolrTestCaseJ4 { + + @Test + public void determineTestPollIntervalString() { + SolrConfig.UpdateHandlerInfo updateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56"); + String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); + assertEquals("00:00:01", pollInterval); + } @Test public void determinePollIntervalString() { + // disable jetty test mode + System.clearProperty("jetty.testMode"); + SolrConfig.UpdateHandlerInfo updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false, null); String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:7", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:7", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); + + updateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56"); + pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); + assertEquals("0:0:56", pollInterval); + + final SolrConfig.UpdateHandlerInfo illegalUpdateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", + -1, + 60000, + -1, + true, + -1, + -1, + false, + "garbage-unfortunately"); + assertThrows( + SolrException.class, + () -> ReplicateFromLeader.determinePollInterval(illegalUpdateHandlerInfo)); } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc index 812bbcbc3a8..c13790c9da3 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc @@ -62,6 +62,10 @@ It is recommended that this be set for as long as is reasonable given the applic A hard commit means that, if a server crashes, Solr will know exactly where your data was stored; a soft commit means that the data is stored, but the location information isn't yet stored. The tradeoff is that a soft commit gives you faster visibility because it's not waiting for background merges to finish. +In a TLOG/PULL replica setup, the commit configuration also influences the interval at which the replica is polling the shard leader. +Users wishing to use a different polling interval in their TLOG/PULL replicas can do so by specifying a `commitPollInterval` value of the form "hh:mm:ss". +"01:00:00" to poll every hour, "00:15:00" to poll every fifteen minutes, etc. + === Explicit Commits When a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds and deletes on an update are written to disk as soon as index updates are completed.