This repository has been archived by the owner on Aug 9, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_epoch
executable file
·166 lines (137 loc) · 4.16 KB
/
run_epoch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
#!/bin/bash
set -u
readonly PROGNAME="$(basename $0)"
readonly USAGE="Usage: $PROGNAME [-d] -n nodes -p ppn -m mem -t walltime -- root_dir"
# constants set by minerva
readonly MAX_PPN=12
readonly MAX_NODES=48 # 576/12
readonly MAX_MEM=2012
readonly MAX_WALLTIME="48:00:00"
readonly DEV_PPN=12
readonly DEV_NODES=4 # 48/12
readonly DEV_MEM=2012
readonly DEV_WALLTIME="01:00:00"
# constants given as options
# TODO: enforce limits based on devel/normal queues
NODES=1
PPN=12
MEM=2012
WALLTIME=
LABEL=
RUN=false
DEVEL=false
ARRAY=false
INPUT_DECK="$HOME/input/input.deck"
N_ARRAY=
command=cat
sub_flags="-A midpluswarwick ${sub_flags:-}"
pbs_flags=
err () {
echo -e "$PROGNAME: $@""\nTry -h for usage." >&2; exit 1
}
# process command line flags
while getopts ":l:a:i:n:p:m:t:drh" flag; do
case "$flag" in
l) LABEL="$OPTARG" ;;
a) ARRAY=true; N_ARRAY="$OPTARG" ;;
i) INPUT_DECK="$(abspath $OPTARG)" ;;
n) NODES="$OPTARG" ;;
p) PPN="$OPTARG" ;;
m) MEM="$OPTARG" ;;
t) WALLTIME="$OPTARG" ;;
d) DEVEL=true; sub_flags="$sub_flags -q midplusdevel" ;;
r) RUN=true ;;
h) echo $USAGE; exit ;;
:) err "Option -$OPTARG requires argument" ;;
\?) echo "Unrecognized option -$OPTARG" >&2 ;;
*) err "Unexpected optargs output" ;;
esac
done
readonly NODES PPN MEM WALLTIME ARRAY N_ARRAY INPUT_DECK DEVEL RUN LABEL
# quit if mandatory constants not given
if [[ -z "$NODES" || -z "$PPN" || -z "$MEM" || -z "$WALLTIME" ]]; then
err "Required flags not set."
fi
# quit if MEM out of bounds
[ "$MEM" -le 2012 ] || err "-m: memory must be less than or equal to 2012"
# quit if input file doesn't exist
if [ ! -f "$INPUT_DECK" ]; then
err "-i: input deck not found: $INPUT_DECK"
elif [ ! -r "$INPUT_DECK" ]; then
err "-i: cannot read input deck: $INPUT_DECK"
fi
# first argument after flags is code root directory
shift $((${OPTIND} - 1))
# check necessary directory layout
[ -z "$1" ] && err No build folder given
readonly ROOT_DIR="$(abspath $1)"
readonly EPOCH_DIR="${ROOT_DIR}/epoch3d"
readonly DATA_DIR="${EPOCH_DIR}/Data"
readonly EPOCH="${EPOCH_DIR}/bin/epoch3d"
if [ -z "$ROOT_DIR" ]; then
err "root_dir not given"
elif [ ! -f "$EPOCH" ]; then
err "could not find file $EPOCH"
elif [ ! -x "$EPOCH" ]; then
err "file $EPOCH not executable"
fi
# TODO: would be nice to have mkuniqdir -p
n_proc=$(($NODES * $PPN))
output_dir="$HOME/output"
if [ -n "$LABEL" ]; then
output_dir="${output_dir}/${LABEL}"
mkdir ${output_dir} 2>/dev/null
fi
output_dir="${output_dir}/$(basename ${ROOT_DIR})"
output_dir="${output_dir}_${n_proc}proc"
[[ $DEVEL == true ]] && output_dir="${output_dir}_devel"
output_dir="$(mkuniqdir $output_dir)" # append a unique number
if [[ $ARRAY == true ]]; then
pbs_flags="$pbs_flags -t 1-${N_ARRAY}:1"
fi
if [[ $RUN == true ]]; then
command="msub $sub_flags"
else
command="cat"
rmdir $output_dir
if [ -n "$LABEL" ]; then
rmdir $(sed "s/\(${LABEL}\).*$/\1/" <<<"$output_dir")
fi
echo "sub_flags: $sub_flags"
echo "output_dir: $output_dir"
echo "========================"
echo "begin submission script:"
echo "========================"
fi
## prepend pbs_flags with #PBS
[ -n "$pbs_flags" ] && pbs_flags="#PBS $pbs_flags"
${command} <<MSUB_SUBMISSION_SCRIPT---------------------------------------
#!/bin/bash
#PBS -l nodes=${NODES}:ppn=${PPN},pvmem=${MEM}mb,walltime=${WALLTIME}
$pbs_flags
module load ompi/1.6.4/intel/12.1
cd \$PBS_O_WORKDIR
output_dir="${output_dir}"
if [[ $ARRAY == true ]]; then
output_dir="\${output_dir}/\${MOAB_JOBARRAYINDEX}"
mkdir "\$output_dir"
fi
time_file="\${output_dir}/time.out"
profile_file="\${output_dir}/profile.out"
stdout_file="\${output_dir}/out"
stderr_file="\${output_dir}/err"
cd "\${output_dir}"
cp "$INPUT_DECK" "\${output_dir}/input.deck"
{ time mpirun $EPOCH <<<"\$output_dir" ; } \\
1>out \\
2>err
# move time output to own file
tail -3 < err >time.out
head -n -3 err > err_tmp
mv err_tmp err
# generate profile and clean up
gprof $EPOCH > profile.out
#rm gmon.out
rm 000[1-9].sdf 00[1-4][0-9].sdf
rm *.sdf
MSUB_SUBMISSION_SCRIPT---------------------------------------