-
Notifications
You must be signed in to change notification settings - Fork 690
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
SOLR-16903: Migrate off java.io.File to java.nio.file.Path from core files #2924
Changes from 6 commits
6cc27d7
bc2db00
4c01109
f17b809
36d0b92
374a4a1
63fb99d
fb4daf3
5f8abae
679fb29
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,7 +17,6 @@ | |
package org.apache.solr.cloud; | ||
|
||
import java.io.ByteArrayInputStream; | ||
import java.io.File; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.lang.invoke.MethodHandles; | ||
|
@@ -121,7 +120,7 @@ public InputStream openResource(String resource) throws IOException { | |
|
||
try { | ||
// delegate to the class loader (looking into $INSTANCE_DIR/lib jars) | ||
is = classLoader.getResourceAsStream(resource.replace(File.separatorChar, '/')); | ||
is = classLoader.getResourceAsStream(Path.of(resource).toString()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Assuming that actually works, I think a little comment is needed like "normalize path separator" There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. But I think it doesn't work; not on Windows. I thought I commented elsewhere in that lengthy review that all |
||
} catch (Exception e) { | ||
throw new IOException("Error opening " + resource, e); | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,7 +16,6 @@ | |
*/ | ||
package org.apache.solr.core; | ||
|
||
import java.io.File; | ||
import java.io.IOException; | ||
import java.io.Reader; | ||
import java.lang.invoke.MethodHandles; | ||
|
@@ -64,7 +63,7 @@ public class CoreDescriptor { | |
public static final String SOLR_CORE_PROP_PREFIX = "solr.core."; | ||
|
||
public static final String DEFAULT_EXTERNAL_PROPERTIES_FILE = | ||
dsmiley marked this conversation as resolved.
Show resolved
Hide resolved
|
||
"conf" + File.separator + "solrcore.properties"; | ||
Path.of("conf/solrcore.properties").toString(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is there any chance of different file seperators? I wasn't sure if that was one reason for using the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was worried about this so started reading I haven't actually tested this on something like a windows operating system to confirm though... There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a really great benefit of the migration! |
||
|
||
/** | ||
* Whether this core was configured using a configSet that was trusted. This helps in avoiding the | ||
|
@@ -96,7 +95,7 @@ public Properties getPersistableUserProperties() { | |
CORE_CONFIG, "solrconfig.xml", | ||
CORE_SCHEMA, "schema.xml", | ||
CORE_CONFIGSET_PROPERTIES, ConfigSetProperties.DEFAULT_FILENAME, | ||
CORE_DATADIR, "data" + File.separator, | ||
CORE_DATADIR, Path.of("data/").toString(), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The trailing slash should ideally not be necessary. I'd prefer to just drop it and we deal with the consequences. If needed, the previous code was clearer IMO so I'd rather see something similarly clear. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed |
||
CORE_TRANSIENT, "false", | ||
CORE_LOADONSTARTUP, "true"); | ||
|
||
|
@@ -240,7 +239,8 @@ public CoreDescriptor( | |
* <p>File paths are taken as read from the core's instance directory if they are not absolute. | ||
*/ | ||
protected void loadExtraProperties() { | ||
String filename = coreProperties.getProperty(CORE_PROPERTIES, DEFAULT_EXTERNAL_PROPERTIES_FILE); | ||
String filename = | ||
coreProperties.getProperty(CORE_PROPERTIES, DEFAULT_EXTERNAL_PROPERTIES_FILE.toString()); | ||
Path propertiesFile = instanceDir.resolve(filename); | ||
if (Files.exists(propertiesFile)) { | ||
try (Reader r = Files.newBufferedReader(propertiesFile, StandardCharsets.UTF_8)) { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,7 +22,6 @@ | |
import com.codahale.metrics.Counter; | ||
import com.codahale.metrics.Timer; | ||
import java.io.Closeable; | ||
import java.io.File; | ||
import java.io.FileNotFoundException; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
|
@@ -33,6 +32,7 @@ | |
import java.lang.invoke.MethodHandles; | ||
import java.lang.reflect.Constructor; | ||
import java.nio.charset.StandardCharsets; | ||
import java.nio.file.Files; | ||
import java.nio.file.NoSuchFileException; | ||
import java.nio.file.Path; | ||
import java.nio.file.Paths; | ||
|
@@ -1104,9 +1104,6 @@ protected SolrCore( | |
checkVersionFieldExistsInSchema(schema, coreDescriptor); | ||
setLatestSchema(schema); | ||
|
||
// initialize core metrics | ||
initializeMetrics(solrMetricsContext, null); | ||
|
||
// init pluggable circuit breakers, after metrics because some circuit breakers use metrics | ||
initPlugins(null, CircuitBreaker.class); | ||
|
||
|
@@ -1124,6 +1121,9 @@ protected SolrCore( | |
this.snapshotMgr = initSnapshotMetaDataManager(); | ||
this.solrDelPolicy = initDeletionPolicy(delPolicy); | ||
|
||
// initialize core metrics | ||
initializeMetrics(solrMetricsContext, null); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why did you move this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was a mistake. I forgot to revert. This has to do with metrics being pre-computed with File disk usage and was trying to get the data directory initialized before the metrics registered. I eventually stopped and forgot to move it back |
||
|
||
this.codec = initCodec(solrConfig, this.schema); | ||
initIndex(prev != null, reload); | ||
|
||
|
@@ -1354,16 +1354,29 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) { | |
Category.CORE.toString()); | ||
} | ||
|
||
// TODO SOLR-8282 move to PATH | ||
// initialize disk total / free metrics | ||
Path dataDirPath = Paths.get(dataDir); | ||
File dataDirFile = dataDirPath.toFile(); | ||
parentContext.gauge( | ||
() -> dataDirFile.getTotalSpace(), true, "totalSpace", Category.CORE.toString(), "fs"); | ||
Path dataDirFile = Paths.get(dataDir); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. really minor but I think dataDirPath is a better name |
||
long totalSpace; | ||
long useableSpace; | ||
|
||
try { | ||
totalSpace = Files.getFileStore(dataDirFile).getTotalSpace(); | ||
useableSpace = Files.getFileStore(dataDirFile).getUsableSpace(); | ||
} catch (Exception e) { | ||
// TODO Metrics used to default to 0 with java.io.File even if directory didn't exist | ||
// Should throw an exception and initialize data directory before metrics | ||
totalSpace = 0L; | ||
useableSpace = 0L; | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This here was questionable. Upon a Solr core being initialized, metrics was trying to calculate and initialize a gauge for the But also if this is initializing metrics for a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do not pre-compute this stuff; it should only be calculated/fetched when the metric is fetched. Maybe could clarify that the TODO is to move metrics initialization to after data directory initialization, thus avoiding this exception (hopefully) but that may not be entirely possible. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just initialized the gauges to 0 and added the TODO. Could be worth moving metric initialization to after the directory is created if we want that data point of the disk usage. |
||
|
||
final long finalTotalSpace = totalSpace; | ||
final long finalUsableSpace = useableSpace; | ||
parentContext.gauge(() -> finalTotalSpace, true, "totalSpace", Category.CORE.toString(), "fs"); | ||
parentContext.gauge( | ||
() -> dataDirFile.getUsableSpace(), true, "usableSpace", Category.CORE.toString(), "fs"); | ||
() -> finalUsableSpace, true, "usableSpace", Category.CORE.toString(), "fs"); | ||
|
||
parentContext.gauge( | ||
() -> dataDirPath.toAbsolutePath().toString(), | ||
() -> dataDirFile.toAbsolutePath().toString(), | ||
true, | ||
"path", | ||
Category.CORE.toString(), | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,7 +17,6 @@ | |
|
||
package org.apache.solr.core; | ||
|
||
import java.io.File; | ||
import java.lang.invoke.MethodHandles; | ||
import java.nio.file.Path; | ||
import java.nio.file.Paths; | ||
|
@@ -43,9 +42,7 @@ private SolrPaths() {} // don't create this | |
|
||
/** Ensures a directory name always ends with a '/'. */ | ||
public static String normalizeDir(String path) { | ||
return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) | ||
? path + File.separator | ||
: path; | ||
return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) ? path + "/" : path; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. wow that's gnarly.. Is there no existing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The code was fine as it was but perhaps got your attention because File is referenced. Since we only need to know the separator, try There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was thinking, we shouldn't need to call |
||
} | ||
|
||
/** | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,7 +18,6 @@ | |
|
||
import com.google.common.annotations.VisibleForTesting; | ||
import java.io.Closeable; | ||
import java.io.File; | ||
import java.io.FilterInputStream; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
|
@@ -34,6 +33,7 @@ | |
import java.nio.file.Files; | ||
import java.nio.file.Path; | ||
import java.nio.file.PathMatcher; | ||
import java.nio.file.Paths; | ||
import java.util.ArrayList; | ||
import java.util.Arrays; | ||
import java.util.Collections; | ||
|
@@ -353,12 +353,14 @@ public ClassLoader getClassLoader() { | |
*/ | ||
@Override | ||
public InputStream openResource(String resource) throws IOException { | ||
if (resource.trim().startsWith("\\\\")) { // Always disallow UNC paths | ||
throw new SolrResourceNotFoundException("Resource '" + resource + "' could not be loaded."); | ||
String pathResource = Paths.get(resource).normalize().toString(); | ||
if (pathResource.trim().startsWith("\\\\")) { // Always disallow UNC paths | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I saw we have a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why trim after we've called normalize? If trimming had been first thing; probably needs to continue to be first thing. I don't like trim in most places including here though (it's generally sloppy); in a major version we can get away with dropping it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the find, Eric. I don't know why I overcomplicated this line with the normalize. Just passing the resource here instead so not much should have changed. Also switched to |
||
throw new SolrResourceNotFoundException( | ||
"Resource '" + pathResource + "' could not be loaded."); | ||
} | ||
Path instanceDir = getInstancePath().normalize(); | ||
Path inInstanceDir = getInstancePath().resolve(resource).normalize(); | ||
Path inConfigDir = instanceDir.resolve("conf").resolve(resource).normalize(); | ||
Path inInstanceDir = getInstancePath().resolve(pathResource).normalize(); | ||
Path inConfigDir = instanceDir.resolve("conf").resolve(pathResource).normalize(); | ||
if (allowUnsafeResourceloading || inInstanceDir.startsWith(instanceDir)) { | ||
// The resource is either inside instance dir or we allow unsafe loading, so allow testing if | ||
// file exists | ||
|
@@ -373,17 +375,17 @@ public InputStream openResource(String resource) throws IOException { | |
|
||
// Delegate to the class loader (looking into $INSTANCE_DIR/lib jars). | ||
// We need a ClassLoader-compatible (forward-slashes) path here! | ||
InputStream is = classLoader.getResourceAsStream(resource.replace(File.separatorChar, '/')); | ||
InputStream is = classLoader.getResourceAsStream(pathResource); | ||
|
||
// This is a hack just for tests (it is not done in ZKResourceLoader)! | ||
// TODO can we nuke this? | ||
if (is == null && System.getProperty("jetty.testMode") != null) { | ||
is = classLoader.getResourceAsStream(("conf/" + resource).replace(File.separatorChar, '/')); | ||
is = classLoader.getResourceAsStream(Path.of("conf").resolve(pathResource).toString()); | ||
} | ||
|
||
if (is == null) { | ||
throw new SolrResourceNotFoundException( | ||
"Can't find resource '" + resource + "' in classpath or '" + instanceDir + "'"); | ||
"Can't find resource '" + pathResource + "' in classpath or '" + instanceDir + "'"); | ||
} | ||
return is; | ||
} | ||
|
@@ -404,8 +406,7 @@ public String resourceLocation(String resource) { | |
return inInstanceDir.normalize().toString(); | ||
} | ||
|
||
try (InputStream is = | ||
classLoader.getResourceAsStream(resource.replace(File.separatorChar, '/'))) { | ||
try (InputStream is = classLoader.getResourceAsStream(Path.of(resource).toString())) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. On Windows, wouldn't this actually convert to the platform's separator char? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You're correct, bad that I removed the replace. Brought it back but changed to |
||
if (is != null) return "classpath:" + resource; | ||
} catch (IOException e) { | ||
// ignore | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,14 +16,14 @@ | |
*/ | ||
package org.apache.solr.core.backup; | ||
|
||
import java.io.File; | ||
import java.io.IOException; | ||
import java.io.OutputStream; | ||
import java.io.OutputStreamWriter; | ||
import java.io.Writer; | ||
import java.lang.invoke.MethodHandles; | ||
import java.net.URI; | ||
import java.nio.charset.StandardCharsets; | ||
import java.nio.file.Path; | ||
import java.time.Instant; | ||
import java.util.Collections; | ||
import java.util.List; | ||
|
@@ -343,7 +343,7 @@ private void downloadConfigToRepo(ConfigSetService configSetService, String conf | |
// getAllConfigFiles always separates file paths with '/' | ||
for (String filePath : filePaths) { | ||
// Replace '/' to ensure that propre file is resolved for writing. | ||
URI uri = repository.resolve(dir, filePath.replace('/', File.separatorChar)); | ||
URI uri = repository.resolve(dir, Path.of(filePath).toString()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. could There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No Eric; the BackupRepository abstraction is based around URI & String. We could change it (maybe that's your proposal) but that's a big change that deserves its own discussion/issue. I'd rather here see the existing code or something very close to it. |
||
// checking for '/' is correct for a directory since ConfigSetService#getAllConfigFiles | ||
// always separates file paths with '/' | ||
if (!filePath.endsWith("/")) { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just wanted to confirm this throw was needed? Maybe because of something else furthur down?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually no it isn't needed. It was due to my changes on
List<FileDetails> list()
method underDistribFileStore
so I threw a SolrException instead which seems correct.