diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java index 06bbbefedbe..ce18090b8e4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java @@ -75,16 +75,9 @@ public void startReplication(boolean switchTransactionLog) { "SolrCore not found:" + coreName + " in " + CloudUtil.getLoadedCoreNamesAsString(cc)); } } - SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo(); - String pollIntervalStr = "00:00:03"; - if (System.getProperty("jetty.testMode") != null) { - pollIntervalStr = "00:00:01"; - } - String calculatedPollIntervalString = determinePollInterval(uinfo); - if (calculatedPollIntervalString != null) { - pollIntervalStr = calculatedPollIntervalString; - } + final SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo(); + final String pollIntervalStr = determinePollInterval(uinfo); log.info("Will start replication from leader with poll interval: {}", pollIntervalStr); NamedList followerConfig = new NamedList<>(); @@ -134,18 +127,25 @@ public static String getCommitVersion(SolrCore solrCore) { } /** - * Determine the poll interval for replicas based on the auto soft/hard commit schedule + * Determine the poll interval for replicas based on the auto soft/hard commit schedule or + * configured commit poll interval * * @param uinfo the update handler info containing soft/hard commit configuration * @return a poll interval string representing a cadence of polling frequency in the form of - * hh:mm:ss + * hh:mm:ss, never null */ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) { int hardCommitMaxTime = uinfo.autoCommmitMaxTime; int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime; boolean hardCommitNewSearcher = uinfo.openSearcher; - String pollIntervalStr = null; - if (hardCommitMaxTime != -1) { + String customCommitPollInterval = uinfo.commitPollInterval; + String pollIntervalStr = "00:00:03"; + + if (System.getProperty("jetty.testMode") != null) { + pollIntervalStr = "00:00:01"; + } else if (customCommitPollInterval != null) { + pollIntervalStr = customCommitPollInterval; + } else if (hardCommitMaxTime != -1) { // configured hardCommit places a ceiling on the interval at which new segments will be // available to replicate if (softCommitMaxTime != -1 @@ -168,6 +168,8 @@ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) { pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2); } + // validate poll interval and fail early + ReplicationHandler.readIntervalNs(pollIntervalStr); return pollIntervalStr; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 968b7aa51f5..b91f55e6579 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -803,6 +803,7 @@ public static class UpdateHandlerInfo implements MapSerializable { public final long autoCommitMaxSizeBytes; public final boolean openSearcher; // is opening a new searcher part of hard autocommit? public final boolean commitWithinSoftCommit; + public final String commitPollInterval; public final boolean aggregateNodeLevelMetricsEnabled; /** @@ -818,7 +819,8 @@ public UpdateHandlerInfo( boolean openSearcher, int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, - boolean commitWithinSoftCommit) { + boolean commitWithinSoftCommit, + String commitPollInterval) { this.className = className; this.autoCommmitMaxDocs = autoCommmitMaxDocs; this.autoCommmitMaxTime = autoCommmitMaxTime; @@ -829,6 +831,7 @@ public UpdateHandlerInfo( this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime; this.commitWithinSoftCommit = commitWithinSoftCommit; + this.commitPollInterval = commitPollInterval; this.aggregateNodeLevelMetricsEnabled = false; } @@ -844,6 +847,7 @@ public UpdateHandlerInfo(ConfigNode updateHandler) { this.autoSoftCommmitMaxTime = updateHandler.get("autoSoftCommit").get("maxTime").intVal(-1); this.commitWithinSoftCommit = updateHandler.get("commitWithin").get("softCommit").boolVal(true); + this.commitPollInterval = updateHandler.get("commitPollInterval").txt(); this.aggregateNodeLevelMetricsEnabled = updateHandler.boolAttr("aggregateNodeLevelMetricsEnabled", false); } @@ -860,6 +864,7 @@ public Map toMap(Map map) { map.put( "autoSoftCommit", Map.of("maxDocs", autoSoftCommmitMaxDocs, "maxTime", autoSoftCommmitMaxTime)); + map.put("commitPollInterval", commitPollInterval); return map; } } diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 8b126b5cc06..aad7e5e94a6 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -30,6 +30,7 @@ import static org.apache.solr.handler.admin.api.ReplicationAPIBase.STATUS; import static org.apache.solr.handler.admin.api.ReplicationAPIBase.TLOG_FILE; +import com.google.common.annotations.VisibleForTesting; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -1485,7 +1486,8 @@ private Long readIntervalMs(String interval) { return TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); } - private Long readIntervalNs(String interval) { + @VisibleForTesting + public static Long readIntervalNs(String interval) { if (interval == null) return null; int result = 0; Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java index 723eda5874d..148ab9b01ca 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java @@ -18,54 +18,105 @@ package org.apache.solr.cloud; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import org.apache.solr.common.SolrException; import org.apache.solr.core.SolrConfig; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class ReplicateFromLeaderTest { + private String jettyTestMode; + + @Before + public void setUp() { + jettyTestMode = System.getProperty("jetty.testMode"); + } + + @After + public void tearDown() { + if (jettyTestMode != null) { + System.setProperty("jetty.testMode", jettyTestMode); + } + } + + @Test + public void determineTestPollIntervalString() { + SolrConfig.UpdateHandlerInfo updateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56"); + String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); + assertEquals("00:00:01", pollInterval); + } + @Test public void determinePollIntervalString() { + // disable jetty test mode + System.clearProperty("jetty.testMode"); + SolrConfig.UpdateHandlerInfo updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false, null); String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:7", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false); + "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false); + "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:7", pollInterval); updateHandlerInfo = new SolrConfig.UpdateHandlerInfo( - "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false); + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, null); pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); assertEquals("0:0:30", pollInterval); + + updateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false, "0:0:56"); + pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo); + assertEquals("0:0:56", pollInterval); + + final SolrConfig.UpdateHandlerInfo illegalUpdateHandlerInfo = + new SolrConfig.UpdateHandlerInfo( + "solr.DirectUpdateHandler2", + -1, + 60000, + -1, + true, + -1, + -1, + false, + "garbage-unfortunately"); + assertThrows( + SolrException.class, + () -> ReplicateFromLeader.determinePollInterval(illegalUpdateHandlerInfo)); } } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc index 812bbcbc3a8..231e4c9d412 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc @@ -62,6 +62,8 @@ It is recommended that this be set for as long as is reasonable given the applic A hard commit means that, if a server crashes, Solr will know exactly where your data was stored; a soft commit means that the data is stored, but the location information isn't yet stored. The tradeoff is that a soft commit gives you faster visibility because it's not waiting for background merges to finish. +In a TLOG/PULL replica setup, the commit configuration also influences the interval at which the replica is polling the shard leader. You may optionally configure a custom `commitPollInterval`. + === Explicit Commits When a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds and deletes on an update are written to disk as soon as index updates are completed.