Skip to content

Commit 3f5f16d

Browse files
chqy99chqy99
authored
[Feature](mlu-ops): check dependency of mlu-ops (Cambricon#913)
1 parent ce2e19c commit 3f5f16d

File tree

5 files changed

+223
-43
lines changed

5 files changed

+223
-43
lines changed

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
</a>
2121
</b>
2222
</div>
23-
23+
2424
<div>&nbsp;</div>
2525

2626
[![ci](https://github.com/Cambricon/mlu-ops/actions/workflows/ci.yaml/badge.svg)](https://github.com/Cambricon/mlu-ops/actions/workflows/ci.yaml)
@@ -54,7 +54,7 @@ MLU-OPS 提供了以下功能：
5454
- 外部链接库：
5555
- libxml2-dev、libprotobuf-dev<=3.8.0、protobuf-compiler<=3.8.0、llvm-6.0-dev
5656
- Python环境：
57-
- 依赖Python-3.8.0版本
57+
- 依赖Python-3版本（默认版本 python 3.8.0，最低要求 python 3.6.0）
5858

5959

6060
## 依赖环境准备

‎build.property

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
{
22
"version": "0.11.0-1",
3-
"build_requires": {"cntoolkit": ["release","3.8.4-1"], "cnnl":["release","1.21.1-1"]},
3+
"python": "3.6.0",
4+
"build_requires": {"cntoolkit": ["release","3.8.4-1"],
5+
"cnnl":["release","1.21.1-1"],
6+
"driver": "5.10.15",
7+
"eigen3": "3.4.0",
8+
"libxml2": "2.9.0",
9+
"protoc": "3.8.0"},
410
"package_type": ["rpm","deb"]
511
}

‎independent_build.sh

Lines changed: 38 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,32 @@ add_mlu_arch_support () {
7171
MLUOP_MLU_ARCH_LIST+=${bang_arch}
7272
}
7373

74+
# check python
75+
python3_version_check() {
76+
cur_python_ver=(`python3 --version`)
77+
stat=$?
78+
if [ ${stat} != 0 ]; then
79+
echo "Not found python3"
80+
exit ${stat}
81+
fi
82+
required_python_version=$(cat build.property|grep "python"|cut -d ':' -f2|cut -d '"' -f2)
83+
if [[ "$(printf '%s\n' "${cur_python_ver[1]}" "${required_python_version}" \
84+
| sort -V | head -n1)" == "${cur_python_ver[1]}" ]]; then
85+
echo "python version should no less than ${required_python_version}"
86+
exit 1
87+
fi
88+
}
89+
python3_version_check
90+
91+
build_requires_version_check() {
92+
# check build_requires
93+
python3 version_pre_check.py check_build_requires
94+
stat=$?
95+
if [ ${stat} != 0 ]; then
96+
exit ${stat}
97+
fi
98+
}
99+
74100
usage () {
75101
echo "USAGE: ./independent_build.sh <options>"
76102
echo
@@ -99,33 +125,21 @@ usage () {
99125
}
100126

101127
prepare_cntoolkit () {
102-
python2 json_parser.py
103-
output='dependency.txt'
104-
MODULE_VERSION=""
105128
PACKAGE_ARCH="$(uname -m)"
106-
107-
# dep-package-version
108-
PACKAGE_MODULES=`cat $output | awk -F ':' '{print $1}'`
109-
prog_log_info "PACKAGE_MODULES: $PACKAGE_MODULES"
110-
111-
PACKAGE_BRANCH=`cat $output | awk -F ':' '{print $2}'`
112-
prog_log_info "PACKAGE_BRANCH: $PACKAGE_BRANCH"
113-
114-
PACKAGE_MODULE_VERS=`cat $output | awk -F ':' '{print $3}'`
115-
prog_log_info "PACKAGE_MODULE_VERS: $PACKAGE_MODULE_VERS"
116-
117129
PACKAGE_SERVER="http://daily.software.cambricon.com"
118130
PACKAGE_OS="Linux"
119131

120-
121-
arr_modules=(`echo $PACKAGE_MODULES`)
122-
arr_branch=(`echo $PACKAGE_BRANCH`)
123-
arr_vers=(`echo $PACKAGE_MODULE_VERS`)
132+
# read build.property, print cntoolkit and cnnl dep-package-version
133+
build_requires=(`python version_pre_check.py get_build_requires`)
134+
# build_requires is an array(cntoolkit release cntoolkit-version cnnl release cnnl-version)
135+
arr_modules=(${build_requires[0]} ${build_requires[3]})
136+
arr_branch=(${build_requires[1]} ${build_requires[4]})
137+
arr_vers=(${build_requires[2]} ${build_requires[5]})
124138

125139
n=${#arr_vers[@]}
126140

127141
sub_pkg_to_extract=(cncc cnas cnperf cngdb cndrv cnrt cnbin cnpapi cndev cntoolkit-cloud)
128-
142+
129143
if [ -d ${PACKAGE_EXTRACT_DIR} ]; then
130144
rm -rf ${PACKAGE_EXTRACT_DIR}
131145
fi
@@ -142,7 +156,7 @@ prepare_cntoolkit () {
142156
REAL_PATH=`echo ${PACKAGE_PATH} | awk -F '//' '{print $2}'`
143157
prog_log_info "${arr_modules[$i]} url: ${REAL_PATH}"
144158
wget -A deb -m -p -E -k -K -np -q --reject-regex 'static' ${PACKAGE_PATH}
145-
159+
146160
pushd ${PACKAGE_EXTRACT_DIR} > /dev/null
147161
for filename in ../${REAL_PATH}*.deb; do
148162
dpkg -x --force-overwrite ${filename} .
@@ -290,7 +304,7 @@ if [ $# != 0 ]; then
290304
export BUILD_JOBS=$1
291305
shift
292306
;;
293-
-t)
307+
-t)
294308
shift
295309
export RELEASE_TYPE=$1
296310
export MLUOP_PACKAGE_INFO_SET="ON"
@@ -371,6 +385,9 @@ if [ "${MLUOP_BUILD_PREPARE_ONLY}" = "ON" ]; then
371385
exit -1
372386
elif [ "${MLUOP_BUILD_PREPARE}" = "ON" ]; then
373387
prepare_cntoolkit
388+
build_requires_version_check
389+
else
390+
build_requires_version_check
374391
fi
375392

376393
if [ ! -z "${NEUWARE_HOME}" ]; then

‎json_parser.py

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎version_pre_check.py

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
import sys
2+
import json
3+
import os
4+
from distutils.version import LooseVersion
5+
6+
build_file = "build.property"
7+
required_version = {}
8+
9+
# cntoolkit, cnnl
10+
modules = ["cntoolkit", "cnnl"]
11+
# NEUWARE_HOME = env_vars["NEUWARE_HOME"]
12+
env_vars = dict(os.environ)
13+
14+
15+
def get_build_requires(print_mode=1):
16+
global required_version
17+
with open(build_file) as build_property:
18+
data = json.load(build_property)
19+
required_version = data["build_requires"]
20+
for key in modules:
21+
if print_mode == 1:
22+
print(
23+
"%s %s %s"
24+
% (key, required_version[key][0], required_version[key][1])
25+
)
26+
required_version[key] = required_version[key][1].split("-")[0]
27+
28+
29+
def check_cntoolkit():
30+
toolkit_ver_path = env_vars["NEUWARE_HOME"] + "/version.txt"
31+
if not os.path.exists(toolkit_ver_path):
32+
print("Not found toolkit")
33+
exit(2)
34+
35+
# check cntoolkit
36+
with open(toolkit_ver_path) as tk_f:
37+
data = tk_f.readlines()
38+
for line in data:
39+
if "Neuware Version" in line:
40+
cur_tk_ver = line.strip("\n").split(" ")[-1]
41+
if LooseVersion(required_version["cntoolkit"]) > LooseVersion(
42+
cur_tk_ver
43+
):
44+
print(
45+
"The version of cntoolkit must be at least "
46+
+ required_version["cntoolkit"]
47+
+ ", but local version is "
48+
+ cur_tk_ver
49+
)
50+
exit(1)
51+
52+
53+
def check_cnnl():
54+
cnnl_ver_pre = env_vars["NEUWARE_HOME"] + "/lib64/"
55+
if not os.path.exists(cnnl_ver_pre):
56+
print("Not found cnnl")
57+
exit(2)
58+
59+
# check cnnl
60+
for filePath in os.listdir(cnnl_ver_pre):
61+
if "libcnnl.so." in filePath:
62+
tmp = filePath.split(".")
63+
if len(tmp) > 3:
64+
cur_cnnl_ver = filePath[11:]
65+
if LooseVersion(required_version["cnnl"]) > LooseVersion(cur_cnnl_ver):
66+
print(
67+
"The version of cnnl must be at least "
68+
+ required_version["cnnl"]
69+
+ ", but local version is "
70+
+ cur_cnnl_ver
71+
)
72+
exit(1)
73+
74+
75+
def check_driver():
76+
sys_out = os.popen("cnmon version").readline()
77+
if len(sys_out) == 0:
78+
print("Warning: not found cnmon.")
79+
print("If compilation failed, please check driver version")
80+
return
81+
82+
sys_out = sys_out.strip("\n").split(":")[-1]
83+
if LooseVersion(required_version["driver"]) > LooseVersion(sys_out):
84+
print(
85+
"The version of driver must be at least "
86+
+ required_version["driver"]
87+
+ ", but local version is "
88+
+ sys_out
89+
)
90+
exit(1)
91+
92+
93+
def check_protoc():
94+
sys_out = os.popen("protoc --version").readline()
95+
if len(sys_out) == 0:
96+
print("Not found protoc")
97+
exit(2)
98+
99+
sys_out = sys_out.strip("\n").split(" ")[-1]
100+
if LooseVersion(required_version["protoc"]) < LooseVersion(sys_out):
101+
print(
102+
"The version of protoc must be at most "
103+
+ required_version["protoc"]
104+
+ ", but local version is "
105+
+ sys_out
106+
)
107+
exit(1)
108+
109+
110+
def check_libxml2():
111+
sys_out = os.popen("xml2-config --version").readline()
112+
if len(sys_out) == 0:
113+
print("Not found libxml2")
114+
exit(2)
115+
116+
sys_out = sys_out.strip("\n")
117+
if LooseVersion(required_version["libxml2"]) > LooseVersion(sys_out):
118+
print(
119+
"The version of libxml2 must be at least "
120+
+ required_version["libxml2"]
121+
+ ", but local version is "
122+
+ sys_out
123+
)
124+
exit(1)
125+
126+
127+
def check_eigen3():
128+
if os.path.exists("/usr/local/include/eigen3/Eigen/src/Core/util/Macros.h"):
129+
h_file = open("/usr/local/include/eigen3/Eigen/src/Core/util/Macros.h")
130+
elif os.path.exists("/usr/include/eigen3/Eigen/src/Core/util/Macros.h"):
131+
h_file = open("/usr/include/eigen3/Eigen/src/Core/util/Macros.h")
132+
else:
133+
print("Not found eigen3")
134+
exit(2)
135+
136+
line = h_file.readline()
137+
eigen_ver = ""
138+
while len(line) > 0:
139+
if "Eigen version and basic defaults" in line:
140+
line = h_file.readline()
141+
line = h_file.readline()
142+
eigen_ver = h_file.readline()[28:-1]
143+
eigen_ver += "." + h_file.readline()[28:-1]
144+
eigen_ver += "." + h_file.readline()[28:-1]
145+
break
146+
line = h_file.readline()
147+
148+
if LooseVersion(required_version["eigen3"]) > LooseVersion(eigen_ver):
149+
print(
150+
"The version of eigen3 must be at least "
151+
+ required_version["eigen3"]
152+
+ ", but local version is "
153+
+ eigen_ver
154+
)
155+
exit(1)
156+
157+
158+
def check_build_requires():
159+
get_build_requires(0)
160+
check_cntoolkit()
161+
check_cnnl()
162+
check_driver()
163+
check_protoc()
164+
check_libxml2()
165+
check_eigen3()
166+
167+
168+
argvs = sys.argv[1:]
169+
if len(argvs) == 1:
170+
eval(argvs[0])()
171+
elif len(argvs) == 2:
172+
eval(argvs[0])(argvs[1])
173+
elif len(argvs) == 3:
174+
eval(argvs[0])(argvs[1], argvs[2])
175+
else:
176+
exit(3)

0 commit comments

Comments
 (0)