Skip to content

Commit

Permalink
Minor tweaks to jupyter notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@VisLab
VisLab committed Oct 24, 2023
1 parent e63986f commit 642dd60
Show file tree
Hide file tree
Showing 7 changed files with 100 additions and 149 deletions.
116 changes: 44 additions & 72 deletions hedcode/jupyter_notebooks/bids/find_event_combinations.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,77 +33,46 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 3,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" key_counts event_code cond_code event_type focus_modality \\\n",
"0 96 1 1 hear_word auditory \n",
"1 96 1 2 hear_word visual \n",
"2 288 1 3 hear_word auditory \n",
"3 96 2 1 look_word auditory \n",
"4 96 2 2 look_word visual \n",
"5 287 2 3 look_word visual \n",
"6 192 3 1 high_tone auditory \n",
"7 192 3 2 high_tone visual \n",
"8 192 4 1 light_bar auditory \n",
"9 192 4 2 light_bar visual \n",
"10 767 5 1 low_tone auditory \n",
"11 767 5 2 low_tone visual \n",
"12 766 6 1 dark_bar auditory \n",
"13 767 6 2 dark_bar visual \n",
"14 385 7 3 high_tone auditory \n",
"15 384 8 3 light_bar visual \n",
"16 191 9 3 high_tone visual \n",
"17 192 10 3 light_bar auditory \n",
"18 1531 11 3 low_tone auditory \n",
"19 1525 12 3 dark_bar visual \n",
"20 771 13 3 low_tone visual \n",
"21 774 14 3 dark_bar auditory \n",
"22 195 201 1 button_press auditory \n",
"23 191 201 2 button_press visual \n",
"24 390 201 3 button_press auditory \n",
"25 381 201 3 button_press visual \n",
"26 5 202 1 pause_recording auditory \n",
"27 6 202 2 pause_recording visual \n",
"28 6 202 3 pause_recording auditory \n",
"29 1 202 3 pause_recording nan \n",
"30 8 202 3 pause_recording visual \n",
"\n",
" attention_status task_role condition \n",
"0 unattended cue_auditory attend_auditory \n",
"1 unattended cue_visual attend_visual \n",
"2 attended cue_auditory shift_attention \n",
"3 unattended cue_auditory attend_auditory \n",
"4 unattended cue_visual attend_visual \n",
"5 attended cue_visual shift_attention \n",
"6 attended infrequent_stimulus attend_auditory \n",
"7 unattended infrequent_stimulus attend_visual \n",
"8 unattended infrequent_stimulus attend_auditory \n",
"9 attended infrequent_stimulus attend_visual \n",
"10 attended frequent_stimulus attend_auditory \n",
"11 unattended frequent_stimulus attend_visual \n",
"12 unattended frequent_stimulus attend_auditory \n",
"13 attended frequent_stimulus attend_visual \n",
"14 attended infrequent_stimulus shift_attention \n",
"15 attended infrequent_stimulus shift_attention \n",
"16 unattended infrequent_stimulus shift_attention \n",
"17 unattended infrequent_stimulus shift_attention \n",
"18 attended frequent_stimulus shift_attention \n",
"19 attended frequent_stimulus shift_attention \n",
"20 unattended frequent_stimulus shift_attention \n",
"21 unattended frequent_stimulus shift_attention \n",
"22 nan target_detected attend_auditory \n",
"23 nan target_detected attend_visual \n",
"24 nan target_detected shift_attention \n",
"25 nan target_detected shift_attention \n",
"26 nan nan attend_auditory \n",
"27 nan nan attend_visual \n",
"28 nan nan shift_attention \n",
"29 nan nan shift_attention \n",
"30 nan nan shift_attention \n"
"sub-002_task-FaceRecognition_events.tsv\n",
"sub-003_task-FaceRecognition_events.tsv\n",
"sub-004_task-FaceRecognition_events.tsv\n",
"sub-005_task-FaceRecognition_events.tsv\n",
"sub-006_task-FaceRecognition_events.tsv\n",
"sub-007_task-FaceRecognition_events.tsv\n",
"sub-008_task-FaceRecognition_events.tsv\n",
"sub-009_task-FaceRecognition_events.tsv\n",
"sub-010_task-FaceRecognition_events.tsv\n",
"sub-011_task-FaceRecognition_events.tsv\n",
"sub-012_task-FaceRecognition_events.tsv\n",
"sub-013_task-FaceRecognition_events.tsv\n",
"sub-014_task-FaceRecognition_events.tsv\n",
"sub-015_task-FaceRecognition_events.tsv\n",
"sub-016_task-FaceRecognition_events.tsv\n",
"sub-017_task-FaceRecognition_events.tsv\n",
"sub-018_task-FaceRecognition_events.tsv\n",
"sub-019_task-FaceRecognition_events.tsv\n",
"The total count of the keys is:31448\n",
" key_counts trial_type value\n",
"0 90 boundary 0\n",
"1 2700 famous_new 5\n",
"2 1313 famous_second_early 6\n",
"3 1291 famous_second_late 7\n",
"4 3532 left_nonsym 256\n",
"5 3381 left_sym 256\n",
"6 3616 right_nonsym 4096\n",
"7 4900 right_sym 4096\n",
"8 2700 scrambled_new 17\n",
"9 1271 scrambled_second_early 18\n",
"10 1334 scrambled_second_late 19\n",
"11 2700 unfamiliar_new 13\n",
"12 1304 unfamiliar_second_early 14\n",
"13 1316 unfamiliar_second_late 15\n"
]
}
],
Expand All @@ -114,22 +83,25 @@
"from hed.tools.util.io_util import get_file_list\n",
"\n",
"# Variables to set for the specific dataset\n",
"dataset_root_path = os.path.realpath('../../../datasets/eeg_ds002893s_hed_attention_shift')\n",
"data_root = 'T:/summaryTests/ds002718-download'\n",
"output_path = ''\n",
"exclude_dirs = ['stimuli']\n",
"exclude_dirs = ['stimuli', 'derivatives', 'code', 'sourcecode']\n",
"\n",
"# Construct the key map\n",
"key_columns = [ \"event_code\", \"cond_code\", \"event_type\", \"focus_modality\", \"attention_status\", \"task_role\", \"condition\"]\n",
"key_columns = [ \"trial_type\", \"value\"]\n",
"key_map = KeyMap(key_columns)\n",
"\n",
"# Construct the unique combinations\n",
"event_files = get_file_list(dataset_root_path, extensions=[\".tsv\"], name_suffix=\"_events\", exclude_dirs=exclude_dirs)\n",
"event_files = get_file_list(data_root, extensions=[\".tsv\"], name_suffix=\"_events\", exclude_dirs=exclude_dirs)\n",
"for event_file in event_files:\n",
" print(f\"{os.path.basename(event_file)}\")\n",
" df = get_new_dataframe(event_file)\n",
" key_map.update(df)\n",
"\n",
"key_map.resort()\n",
"template = key_map.make_template()\n",
"key_counts_sum = template['key_counts'].sum()\n",
"print(f\"The total count of the keys is:{key_counts_sum}\")\n",
"if output_path:\n",
" template.to_csv(output_path, sep='\\t', index=False, header=True)\n",
"else:\n",
Expand All @@ -138,8 +110,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-09-03T19:43:55.968067600Z",
"start_time": "2023-09-03T19:43:48.336417200Z"
"end_time": "2023-10-24T20:08:40.958637400Z",
"start_time": "2023-10-24T20:08:24.603887900Z"
}
}
}
Expand Down
28 changes: 20 additions & 8 deletions hedcode/jupyter_notebooks/bids/validate_bids_dataset.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"execution_count": 6,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using HEDTOOLS version: {'date': '2023-10-12T10:29:43-0500', 'dirty': True, 'error': None, 'full-revisionid': 'e9e63e39f6a99487c5e69ae96e653c4580045ec0', 'version': '0.3.0+164.ge9e63e3.dirty'}\n",
"HED Examples version: {'version': '0.2.0+173.g63d5080.dirty', 'full-revisionid': '63d50808c4c7e97bdfc89076cb19af9e13252057', 'dirty': True, 'error': None, 'date': '2023-10-23T11:09:31-0500'}\n",
"No HED validation errors\n"
]
}
],
"source": [
"from hed.errors import get_printable_issue_string\n",
"from hed.tools import BidsDataset\n",
Expand All @@ -45,10 +55,8 @@
"\n",
"## Set the dataset location and the check_for_warnings flag\n",
"check_for_warnings = False\n",
"#dataset_path = '../../../datasets/eeg_ds003645s_hed_column'\n",
"#dataset_path = '../../../datasets/eeg_ds004105s_hed'\n",
"dataset_path = 't:/summaryTests/ds004105-download'\n",
"outfile = 't:/ds004105_errors.txt'\n",
"dataset_path = 't:/summaryTests/ds002718-download'\n",
"outfile = 't:/ds002718_errors.txt'\n",
"## Validate the dataset\n",
"bids = BidsDataset(dataset_path)\n",
"issue_list = bids.validate(check_for_warnings=check_for_warnings)\n",
Expand All @@ -57,12 +65,16 @@
"else:\n",
" issue_str = \"No HED validation errors\"\n",
"print(issue_str)\n",
"if outfile:\n",
"if outfile and issue_list:\n",
" with open(outfile, 'w') as fp:\n",
" fp.write(issue_str)\n"
],
"metadata": {
"collapsed": false
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-10-24T18:51:33.095713100Z",
"start_time": "2023-10-24T18:51:31.396158Z"
}
}
}
],
Expand Down
14 changes: 7 additions & 7 deletions hedcode/jupyter_notebooks/remodeling/run_remodel_restore.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
"This notebook uses `run_remodel_restore(arglist)` to restore the event files from\n",
"a backup created by `run_remodel_backup`.\n",
"\n",
"| parameter | default | dest | other | meaning |\n",
"| --------- | ------- | ---- | ----- | ------- |\n",
"| data_dir | | data_dir | | Full path of dataset root directory. |\n",
"| -n, --backup_name | default_back | backup_name | | Name of the default backup for remodeling. |\n",
"| -v, --verbose| | verbose | action=store_true | If present, output informative messages as computation progresses. |"
"| parameter | default | dest | other | meaning |\n",
"|--------------------| ------- | ---- | ----- | ------- |\n",
"| data_dir | | data_dir | | Full path of dataset root directory. |\n",
"| -bn, --backup_name | default_back | backup_name | | Name of the default backup for remodeling. |\n",
"| -v, --verbose | | verbose | action=store_true | If present, output informative messages as computation progresses. |"
]
},
{
Expand All @@ -31,7 +31,7 @@
"\n",
"backup_name = 'test_backup'\n",
"data_dir = os.path.realpath('../../../datasets/eeg_ds003645s_hed_remodel')\n",
"arg_list = [data_dir, '-n', backup_name]\n",
"arg_list = [data_dir, '-bn', backup_name]\n",
"cli_restore.main(arg_list)"
],
"metadata": {
Expand Down Expand Up @@ -63,4 +63,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}
8 changes: 8 additions & 0 deletions hedcode/matlab_scripts/data_cleaning/getEventTable.m
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
function eventTable = getEventTable(eventsFile, columnTypes, renameColumns)
% Read the table of events from the events file
%
% Parameters:
% eventsFile - the path of a BIDS tabular events file.
% columnTypes - a cell array of two-element values:
% column-name, column-type
% renameColumns - a cell array of two-element values:
% old-name, new-name
%
typeMap = containers.Map(columnTypes(:, 1), columnTypes(:, 2));
optsDect = detectImportOptions(eventsFile, 'FileType', 'delimitedtext');

Expand Down
45 changes: 2 additions & 43 deletions hedcode/matlab_scripts/data_cleaning/runEeglabImportEvents.m
View file
Original file line number Diff line number Diff line change
@@ -1,49 +1,8 @@
%% This imports the _events.tsv into the corresponding EEG.set file

%% Set up the specifics for your dataset
filename = 'T:SummaryTests/ds004105-download/sub-01/ses-01/eeg/sub-01_ses-01_task-DriveRandomSound_run-1_events.tsv';
eventstruct = importevent( filename, [], 1024, 'key1', 'value1', ...);
% rootPath = '/XXX/SternbergWorkingPhaseTwo';
% setname = '';
% log_name = 'sternberg_12_import_events_log.txt';
% renameColumns = {'event_type', 'type'; 'onset', 'latency'};
%
% % rootPath = 'G:/AttentionShift/AttentionShiftWorkingPhaseTwo';
% % setname = 'Auditory Visual Attention Shift';
% % log_name = 'attention_shift_18_import_events_log.txt';
%
% % rootPath = 's:/bcit/AdvancedGuardDutyWorkingPhaseTwo';
% % setname = 'BCIT Advanced Guard Duty';
% % log_name = 'bcit_advanced_guard_duty_10_import_events_log.txt';
% %
% % rootPath = 's:/bcit/AuditoryCueingWorkingPhaseTwo';
% % setname = 'BCIT Auditory Cueing';
% % log_name = 'bcit_auditory_cueing_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/BaselineDrivingWorkingPhaseTwo';
% % setname = 'BCIT Baseline Driving';
% % log_name = 'bcit_baseline_driving_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/BasicGuardDutyWorkingPhaseTwo';
% % setname = 'BCIT Basic Guard Duty';
% % log_name = 'bcit_basic_guard_duty_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/CalibrationDrivingWorkingPhaseTwo';
% % setname = 'BCIT Calibration Driving';
% % log_name = 'bcit_calibration_driving_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/MindWanderingWorkingPhaseTwo';
% % setname = 'BCIT Mind Wandering';
% % log_name = 'bcit_mind_wandering_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/SpeedControlWorkingPhaseTwo';
% % setname = 'BCIT Speed Control';
% % log_name = 'bcit_speed_control_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/TrafficComplexityWorkingPhaseTwo';
% % setname = 'BCIT Traffic Complexity';
% % log_name = 'bcit_traffic_complexity_10_import_events_log.txt';
%
filename = 'T:/SummaryTests/ds004105-download/sub-01/ses-01/eeg/sub-01_ses-01_task-DriveRandomSound_run-1_events.tsv';

% excludeDirs = {'sourcedata', 'code', 'stimuli', 'derivatives'};
% namePrefix = '';
% nameSuffix = '_eeg';
Expand Down
6 changes: 3 additions & 3 deletions hedcode/matlab_scripts/web_services/runAllTests.m
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
dochost = 'https://hedtools.org/hed';
%host = 'https://hedtools.org/hed_dev';
host = 'http://127.0.0.1:5000';
host = 'https://hedtools.org/hed';
host = 'https://hedtools.org/hed_dev';
%host = 'http://127.0.0.1:5000';


errorMap = containers.Map('KeyType', 'char', 'ValueType', 'any');
Expand Down
32 changes: 16 additions & 16 deletions hedcode/matlab_scripts/web_services/testEventSearchServices.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
'schema_version', '8.0.0', ...
'sidecar_string', data.jsonText, ...
'events_string', data.eventsText, ...
'query', '[[Intended-effect, Cue]]');
'query', '{Intended-effect, Cue}');

response1 = webwrite(servicesUrl, request1, options);
response1 = jsondecode(response1);
Expand All @@ -25,19 +25,19 @@
errors{end + 1} = 'Example 1 failed execute the search.';
end

% %% Example 2: Search an events file for HED
% request2 = struct('service', 'events_search', ...
% 'schema_version', '8.0.0', ...
% 'sidecar_string', data.jsonText, ...
% 'events_string', data.eventsText, ...
% 'columns_included', '', ...
% 'query', '[[Intended-effect, Cue]]');
% request2.columns_included = {'onset'};
% response2 = webwrite(servicesUrl, request2, options);
% response2 = jsondecode(response2);
% outputReport(response2, 'Example 2 Querying an events file with extra columns');
% if ~isempty(response2.error_type) || ...
% ~strcmpi(response2.results.msg_category, 'success')
% errors{end + 1} = 'Example 2 failed execute the search.';
% end
%% Example 2: Search an events file for HED
request2 = struct('service', 'events_search', ...
'schema_version', '8.0.0', ...
'sidecar_string', data.jsonText, ...
'events_string', data.eventsText, ...
'columns_included', '', ...
'query', '{Intended-effect, Cue}');
request2.columns_included = {'onset'};
response2 = webwrite(servicesUrl, request2, options);
response2 = jsondecode(response2);
outputReport(response2, 'Example 2 Querying an events file with extra columns');
if ~isempty(response2.error_type) || ...
~strcmpi(response2.results.msg_category, 'success')
errors{end + 1} = 'Example 2 failed execute the search.';
end

0 comments on commit 642dd60

Please sign in to comment.