-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpreprocess_mice_trials.py
294 lines (262 loc) · 15.8 KB
/
preprocess_mice_trials.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
# this script, together with the lib process_mouse_trials_lib
# is used to process the raw excel files that have the tracking
# of the mouse in the arena looking for the hidden food
#
# the experiments were originally performed by Kelly Xu from Len Maler lab
#
# the original excel files are not going to be included in this repository
#
# instead, we provide only the processed files (in .mat format)
# that contain all the relevant information for the trajectories
# extracted from the excel files...
#
# this script is only here for reference on how the excel files were processed
import os
import sys
import numpy
import modules.io as io
import modules.helper_func_class as misc
import modules.process_mouse_trials_lib as plib
def main():
clear_output_directory = True
ntrials_to_process = None
"""
# experiments where mice are trained in "static entrance protocol" until first probe at least
# 2019-05-23 # preliminary
# 2021-06-22 # two targets
# 2021-11-19 # two targets
# 2022-08-12 # two targets rotated probe
# 2022-09-20 # two targets with rotated probe (female mice)
# 2022-10-11 # two targets rotated probe mixed sex
# 2022-11-04 # relative target (aka static entrance) - after trial 21 (at the probe), two mice (out of 4) get 90 degrees rotation
# relative target (aka static entrance)
# 2019-10-07
# 2019-09-06
# fixed target (aka random entrance)
# 2021-07-16
# 2021-11-15
# two targets
# 2021-11-19
# 2021-06-22
# preliminary experiments
# 2019-05-23
# two targets with rotated probe
# 2022-08-12
# two targets with rotated probe and mixed sex mice
# 2022-10-11
# two targets with rotated probe (female mice)
# 2022-09-20 :: two_targets_rot_fem
# relative target (aka static entrance) - after trial 21, two mice get 90 degrees rotation
# 2022-11-04
"""
preprocess_experiment('pilot' , clear_output_directory, ntrials_to_process)
preprocess_experiment('relative_target' , clear_output_directory, ntrials_to_process)
preprocess_experiment('fixed_target' , clear_output_directory, ntrials_to_process)
preprocess_experiment('two_targets' , clear_output_directory, ntrials_to_process)
preprocess_experiment('preliminary' , clear_output_directory, ntrials_to_process)
preprocess_experiment('two_targets_rot' , clear_output_directory, ntrials_to_process)
preprocess_experiment('two_targets_rot_mixsex' , clear_output_directory, ntrials_to_process)
preprocess_experiment('two_targets_rot_fem' , clear_output_directory, ntrials_to_process)
preprocess_experiment('relative_target_90deg' , clear_output_directory, ntrials_to_process)
#end main
def preprocess_experiment(experiment,clear_output_directory,ntrials_to_process):
experiment = experiment.lower()
valid_experiments = ['relative_target','fixed_target','two_targets','preliminary','two_targets_rot','two_targets_rot_mixsex','two_targets_rot_fem','relative_target_90deg', 'pilot']
assert experiment in valid_experiments, "experiment must one of %s"%( str(valid_experiments)[1:-1].replace("'",'') )
this_directory = sys.path[0]
process_param = []
if experiment == 'relative_target':
clear_output_directory = [clear_output_directory,False]
process_param = [
# second set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2019-10-07_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/relative_target' ),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False,
mouse_gender = {} ,
correct_arena_center = False ),
# third set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2019-09-06_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/relative_target' ),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False ,
mouse_gender = {} ,
correct_arena_center = False )
]
elif experiment == 'fixed_target':
clear_output_directory = [clear_output_directory,False]
process_param = [
# third set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2021-07-16_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/fixed_target' ),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False ,
mouse_gender = {} ,
correct_arena_center = False ),
# third set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2021-11-15_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/fixed_target' ),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = True , # we want to correct the distortion in this experiment to match the 16-July-2021 one
mouse_gender = {} ,
correct_arena_center = True )
]
elif experiment == 'two_targets':
clear_output_directory = [clear_output_directory,False]
process_param = [
# fourth set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2021-11-19_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/two_target_no_cues'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {'Reverse':'Probe'}, # we are renaming the 'Reverse' trials to 'Probe' ### this dictionary is used to replace the key trial label with the value label; in this case,
correct_distortion = False, # False for correct distortion
mouse_gender = {} ,
correct_arena_center = False ),
# fourth set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2021-06-22_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/two_target_no_cues'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False ,
mouse_gender = {} ,
correct_arena_center = False )
]
elif experiment == 'two_targets_rot':
clear_output_directory = [clear_output_directory]
process_param = [
# fifth set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2022-08-12_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/two_targets_rot'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = True ,
mouse_gender = {} ,
correct_arena_center = True )
]
elif experiment == 'two_targets_rot_mixsex':
clear_output_directory = [clear_output_directory]
process_param = [
# fifth set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2022-10-11_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/two_targets_rot_mixsex'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = True ,
mouse_gender = { 1:'F', 6:'F', 8:'F' },
correct_arena_center = True ) # dict that links mouse_number to mouse_gender flag ('F' for female, 'M' for male; default: 'M'); mouse_number must be int or convertible to int
]
elif experiment == 'preliminary':
clear_output_directory = [clear_output_directory]
process_param = [
# first set of experiments
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2019-05-23_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/preliminary' ),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False ,
mouse_gender = {} ,
correct_arena_center = False )
]
elif experiment == 'two_targets_rot_fem':
clear_output_directory = [clear_output_directory]
process_param = [
# another set (not sure if this belongs here... waiting Kelly's answer)
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2022-09-20_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/two_targets_rot_fem'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False,
mouse_gender = {73:'F', 74:'F', 75:'F', 76:'F'} ,
correct_arena_center = False)
]
elif experiment == 'relative_target_90deg':
clear_output_directory = [clear_output_directory]
process_param = [
# another set (not sure if this belongs here... waiting Kelly's answer)
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/2022-11-04_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/relative_target_90deg'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False,
mouse_gender = {} ,
correct_arena_center = False)
]
elif experiment == 'pilot':
clear_output_directory = [clear_output_directory]
process_param = [
dict( trial_dir = os.path.join(this_directory,'../EthovisionPathAnalysis/Raw Trial Data/pilot_Raw Trial Data'),
out_dir = os.path.join(this_directory,'experiments/pilot'),
trials_to_process = 'all', # trials to process; available values in plib.ethovision_to_track_matfile
replacement_dict = {}, # no trial label replacements
correct_distortion = False,
mouse_gender = {} ,
correct_arena_center = False)
]
else:
raise ValueError('Unknown experiment %s'%experiment)
for parval,cleardir in zip(process_param,clear_output_directory):
process_trajectory_files(True,cleardir,ntrials_to_process=ntrials_to_process,**parval)
io.write_txt_header_from_dict(os.path.join(process_param[0]['out_dir'],'experiment_process_params.txt'),process_param,replace=True,hchar='#',verbose=True)
#end preprocess_experiment
def process_trajectory_files(do_processing,clear_output_directory,ntrials_to_process=None,trial_dir=None,out_dir=None,trials_to_process=None,replacement_dict=None,correct_distortion=None,mouse_gender=None,correct_arena_center=False):
########################
################
########
# input/output configuration
# number of rows to skip in the beginning of each trial excel file
#n_rows_skip = 37 # defined automatically for each ioDir group defined below
if not misc.exists(ntrials_to_process):
ntrials_to_process = -1
if clear_output_directory:
io.clear_directory(out_dir,verbose=True)
# out_dir_list = list(dict.fromkeys([ d for p,_,d in ioDir if p ]))
# io.clear_directory(out_dir_list,verbose=True)
# for debug only
#ioDir = [
## first set of experiments
# ( True,
# 'D:/Dropbox/p/uottawa/data/animal_trajectories/EthovisionPathAnalysis/Raw Trial Data/2019-10-07_Raw Trial Data',
# 'D:/Dropbox/p/uottawa/data/animal_trajectories/mouse_track/debug_output_mouse_trajectories' )
# ]
########################
################
########
# running
#for do_processing,trial_dir,out_dir,trials_to_process,replacement_dict,correct_distortion in ioDir:
# if not do_processing:
# continue
if do_processing:
print('--------------------------')
print('--------------------------')
print('--------------------------')
print('--------------------------')
print('STARTING TO PROCESS ... %s'%io.get_filename(trial_dir))
io.create_output_dir(out_dir)
trial_files = plib.get_trial_files(trial_dir,file_ptrn='*-Trial*.xlsx')
n_rows_skip = plib.get_n_header_rows(trial_files[0][1])
n_processed_files = 0
mouse_dir = []
for _,fname in trial_files:
n_processed_files += 1
if (ntrials_to_process > 0) and (n_processed_files == ntrials_to_process):
print(' reached the limit number of files -> %d ::: Stopping'%ntrials_to_process)
break
percent_process = 100.0*float(n_processed_files)/float(len(trial_files))
print('%.2f%% -- Processing %s ...'%(percent_process,io.get_filename(fname)))
track = plib.ethovision_to_track_matfile(fname,num_of_header_rows_in_excel=n_rows_skip-1,trials_to_process=trials_to_process,correct_distortion=correct_distortion,mouse_gender=mouse_gender,correct_arena_center=correct_arena_center)
if misc.exists(track):
fn = io.save_trial_file(out_dir,track)
mouse_dir.append(fn)
else:
print(' ... ::: skipping')
if len(replacement_dict) > 0:
mouse_dir = numpy.unique([ os.path.split(md)[0] for md in mouse_dir ])
for md in mouse_dir:
io.replace_tracks_trial_label(str(md),replacement_dict)
#end process_trajectory_files
if __name__ == '__main__':
main()