Skip to content

Commit 00250e4

Browse files
ConeyLiuGuo Chenzhao
ConeyLiu
authored and
Guo Chenzhao
committed
Add support Intel Optane DC persistent memory cache (#865)
What changes were proposed in this pull request? - Add new memory manager based on Intel Optane DC persistent memory. - Add NativeLibrarayLoader to load the native library. - Add new conf/persistent-memory.xml.template to describe the persistent memory config. - CMakeLists.txt used to make native code. - new pom profile to active native code compile and produce share library. How was this patch tested? mvn clean package -Pspark-2.1 -Ppersistent-memory
1 parent cafea8f commit 00250e4

File tree

14 files changed

+764
-30
lines changed

14 files changed

+764
-30
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,5 @@ dependency-reduced-pom.xml
2424
derby.log
2525
metastore_db/
2626
null/
27+
src/main/native/build
28+
conf/persistent-memory.xml

‎.travis.yml

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,18 @@
11
sudo: required
22
dist: trusty
33
language: java
4+
before_install:
5+
- sudo apt-get install cmake
6+
- sudo apt-get install libpthread-stubs0-dev
7+
- sudo apt-get install libnuma-dev
8+
before_script:
9+
- cd /tmp
10+
- git clone https://github.com/memkind/memkind.git
11+
- cd memkind && ./build.sh
12+
- make
13+
- sudo make install
14+
- cd ${TRAVIS_BUILD_DIR}
415
script:
5-
- mvn clean -q -Pspark-2.1 test
6-
- mvn clean -q -Pspark-2.2 test
7-
- mvn clean -q -Pspark-2.3 test
16+
- mvn clean -q -Pspark-2.1 -Ppersistent-memory test
17+
- mvn clean -q -Pspark-2.2 -Ppersistent-memory test
18+
- mvn clean -q -Pspark-2.3 -Ppersistent-memory test

‎conf/persistent-memory.xml.template

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?xml version="1.0"?>
2+
3+
<!--
4+
Licensed to the Apache Software Foundation (ASF) under one or more
5+
contributor license agreements. See the NOTICE file distributed with
6+
this work for additional information regarding copyright ownership.
7+
The ASF licenses this file to You under the Apache License, Version 2.0
8+
(the "License"); you may not use this file except in compliance with
9+
the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<persistentMemoryPool>
21+
<!--The numa id-->
22+
<numanode id="0">
23+
<!--The initial path for Intel Optane DC persistent memory-->
24+
<initialPath>path to initial path</initialPath>
25+
</numanode>
26+
<numanode id="1">
27+
<initialPath>path to initial path</initialPath>
28+
</numanode>
29+
</persistentMemoryPool>

‎pom.xml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
<maven.compiler.useIncrementalCompilation>false</maven.compiler.useIncrementalCompilation>
3838
<basedir>./</basedir>
3939
<jetty.version>9.2.16.v20160414</jetty.version>
40+
<exec.maven.version>1.6.0</exec.maven.version>
4041
</properties>
4142

4243
<repositories>
@@ -822,5 +823,37 @@
822823
</plugins>
823824
</build>
824825
</profile>
826+
<profile>
827+
<id>persistent-memory</id>
828+
<build>
829+
<resources>
830+
<resource>
831+
<directory>src/main/resources</directory>
832+
</resource>
833+
<resource>
834+
<directory>${basedir}/conf</directory>
835+
</resource>
836+
</resources>
837+
<plugins>
838+
<plugin>
839+
<artifactId>exec-maven-plugin</artifactId>
840+
<groupId>org.codehaus.mojo</groupId>
841+
<version>${exec.maven.version}</version>
842+
<executions>
843+
<execution>
844+
<id>Compile native code and produce share library</id>
845+
<phase>generate-resources</phase>
846+
<goals>
847+
<goal>exec</goal>
848+
</goals>
849+
<configuration>
850+
<executable>${basedir}/src/main/native/compile.sh</executable>
851+
</configuration>
852+
</execution>
853+
</executions>
854+
</plugin>
855+
</plugins>
856+
</build>
857+
</profile>
825858
</profiles>
826859
</project>

‎src/main/java/org/apache/spark/unsafe/PersistentMemoryPlatform.java

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.unsafe;
19+
20+
import java.io.File;
21+
22+
import com.google.common.base.Preconditions;
23+
import org.apache.spark.util.NativeLibraryLoader;
24+
25+
/**
26+
* A platform used to allocate/free volatile memory from
27+
* <a href="https://en.wikipedia.org/wiki/Persistent_memory>Persistent Memory</a></a>
28+
* e.g. Intel Optane DC persistent memory.
29+
*/
30+
public class PersistentMemoryPlatform {
31+
private static volatile boolean initialized = false;
32+
private static final String LIBNAME = "pmplatform";
33+
34+
static {
35+
NativeLibraryLoader.load(LIBNAME);
36+
}
37+
38+
/**
39+
* Initialize the persistent memory.
40+
* @param path The initial path which should be a directory.
41+
* @param size The initial size
42+
*/
43+
public static void initialize(String path, long size) {
44+
synchronized (PersistentMemoryPlatform.class) {
45+
if (!initialized) {
46+
Preconditions.checkNotNull(path, "Persistent memory initial path can't be null");
47+
File dir = new File(path);
48+
Preconditions.checkArgument(dir.exists() && dir.isDirectory(), "Persistent memory " +
49+
"initial path should be a directory");
50+
Preconditions.checkArgument(size > 0,
51+
"Persistent memory initial size must be a positive number");
52+
try {
53+
initializeNative(path, size);
54+
} catch (Exception e) {
55+
throw new ExceptionInInitializerError("Persistent memory initialize (path: " + path +
56+
", size: " + size + ") failed. Please check the path permission and initial size.");
57+
}
58+
initialized = true;
59+
}
60+
}
61+
}
62+
63+
private static native void initializeNative(String path, long size);
64+
65+
/**
66+
* Allocate volatile memory from persistent memory.
67+
* @param size the requested size
68+
* @return the address which same as Platform.allocateMemory, it can be operated by
69+
* Platform which same as OFF_HEAP memory.
70+
*/
71+
public static native long allocateVolatileMemory(long size);
72+
73+
/**
74+
* Get the actual occupied size of the given address. The occupied size should be different
75+
* with the requested size because of the memory management of Intel Optane DC persistent
76+
* memory is based on jemalloc.
77+
* @param address the memory block address.
78+
* @return actual occupied size.
79+
*/
80+
public static native long getOccupiedSize(long address);
81+
82+
/**
83+
* Free the memory by address.
84+
*/
85+
public static native void freeMemory(long address);
86+
}

‎src/main/java/org/apache/spark/util/NativeLibraryLoader.java

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.util;
19+
20+
import java.io.*;
21+
import java.util.HashMap;
22+
import java.util.Map;
23+
24+
import org.slf4j.Logger;
25+
import org.slf4j.LoggerFactory;
26+
27+
/**
28+
* A native library loader which used to load native library.
29+
*/
30+
public class NativeLibraryLoader {
31+
private static final Logger LOGGER = LoggerFactory.getLogger(NativeLibraryLoader.class);
32+
33+
private static Map<String, Boolean> loadRecord = new HashMap<>();
34+
35+
private static String osName() {
36+
String os = System.getProperty("os.name").toLowerCase().replace(' ', '_');
37+
//TODO: improve this
38+
if (os.startsWith("linux")){
39+
return "linux";
40+
} else if (os.startsWith("mac")) {
41+
return "mac";
42+
} else {
43+
throw new UnsupportedOperationException("The platform: " + os + "is not supported.");
44+
}
45+
}
46+
47+
private static String osArch() {
48+
String arch = System.getProperty("os.arch");
49+
if (arch.contains("64")) {
50+
arch = "64";
51+
} else {
52+
arch = "32";
53+
}
54+
return arch;
55+
}
56+
57+
private static String resourceName(final String libname) {
58+
return "/" + osName() + "/" + osArch() + "/lib/" + System.mapLibraryName(libname);
59+
}
60+
61+
/**
62+
* Loading the native library with given name. The name should not include the prefix and suffix.
63+
*/
64+
public static synchronized void load(final String libname) {
65+
if (loadRecord.containsKey(libname) && loadRecord.get(libname)) {
66+
return;
67+
}
68+
69+
try {
70+
System.loadLibrary(libname);
71+
} catch (UnsatisfiedLinkError e) {
72+
LOGGER.warn("Loading library: {} from system libraries failed, trying to load it from " +
73+
"package", libname);
74+
loadFromPackage(libname);
75+
}
76+
77+
loadRecord.put(libname, true);
78+
}
79+
80+
private static void loadFromPackage(final String libname) {
81+
InputStream is = null;
82+
OutputStream out = null;
83+
File tmpLib = null;
84+
try {
85+
is = NativeLibraryLoader.class.getResourceAsStream(resourceName(libname));
86+
if (is == null) {
87+
String errorMsg = "Unsupported OS/arch, cannot find " + resourceName(libname) + " or " +
88+
"load " + libname + " from system libraries. Please try building from source the jar" +
89+
" or providing " + libname + " in you system.";
90+
throw new RuntimeException(errorMsg);
91+
}
92+
93+
tmpLib = File.createTempFile(libname, ".tmp", null);
94+
95+
out = new FileOutputStream(tmpLib);
96+
byte[] buf = new byte[4096];
97+
while (true) {
98+
int read = is.read(buf);
99+
if (read == -1) {
100+
break;
101+
}
102+
out.write(buf, 0, read);
103+
}
104+
105+
out.flush();
106+
out.close();
107+
out = null;
108+
109+
System.load(tmpLib.getAbsolutePath());
110+
} catch (IOException | UnsatisfiedLinkError e) {
111+
throw new RuntimeException("Can't load library: " + libname + " from jar.", e);
112+
} finally {
113+
safeClose(is, "Close resource input stream failed. Ignore it.");
114+
safeClose(out, "Close tmp lib output stream failed. Ignore it.");
115+
116+
if (tmpLib != null && !tmpLib.delete()) {
117+
LOGGER.warn("Delete tmp lib: {} failed, ignore it.", tmpLib.getAbsolutePath());
118+
}
119+
}
120+
}
121+
122+
private static void safeClose(Closeable closeable, String warnMsg) {
123+
if (closeable != null) {
124+
try {
125+
closeable.close();
126+
} catch (IOException e) {
127+
LOGGER.warn(warnMsg, e.fillInStackTrace());
128+
}
129+
}
130+
}
131+
}

‎src/main/native/CMakeLists.txt

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# TODO maybe need to check the version of Memkind
19+
20+
CMAKE_MINIMUM_REQUIRED (VERSION 2.6)
21+
22+
PROJECT(OAP)
23+
24+
FIND_PACKAGE(JNI REQUIRED)
25+
26+
INCLUDE_DIRECTORIES(${JNI_INCLUDE_DIRS})
27+
28+
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
29+
30+
SET(SOURCE_FILES org_apache_spark_unsafe_PMPlatform.cpp)
31+
32+
ADD_LIBRARY(pmplatform SHARED ${SOURCE_FILES})
33+
34+
INSTALL(TARGETS pmplatform LIBRARY DESTINATION lib)
35+
36+
TARGET_LINK_LIBRARIES(pmplatform memkind)

0 commit comments

Comments
 (0)