From c079aa16f2db7f6d93626eeed276f74ac29ce02d Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 08:03:13 +0100 Subject: [PATCH 01/48] Basic display of html table, json and csv --- .../annotations/fileanns_underscore.html | 7 ++++ .../webclient/annotations/omero_table.csv | 2 ++ .../webclient/annotations/omero_table.html | 27 +++++++++++++++ .../tools/OmeroWeb/omeroweb/webclient/urls.py | 3 ++ .../OmeroWeb/omeroweb/webclient/views.py | 34 +++++++++++++++++++ 5 files changed, 73 insertions(+) create mode 100644 components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv create mode 100644 components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html index 6c02b21df44..d0b9564c5b8 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html @@ -45,6 +45,13 @@
+ <% if (ann.ns && ann.ns === 'openmicroscopy.org/omero/bulk_annotations'){ %> + + + + <% } %> + <% if (ann.link.permissions.canDelete) { %> – diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv new file mode 100644 index 00000000000..3a3904ea582 --- /dev/null +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv @@ -0,0 +1,2 @@ +{% for col in columns %}{{ col }},{% endfor %} +{% for row in rows %}{% for col in row %}{{ col }},{% endfor %}\n{% endfor %} \ No newline at end of file diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html new file mode 100644 index 00000000000..189407aa580 --- /dev/null +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -0,0 +1,27 @@ + + + + + + + + + + {% for col in columns %} + + {% endfor %} + + + {% for row in rows %} + + {% for col in row %} + + {% endfor %} + + {% endfor %} + + \ No newline at end of file diff --git a/components/tools/OmeroWeb/omeroweb/webclient/urls.py b/components/tools/OmeroWeb/omeroweb/webclient/urls.py index 0282562f8ea..b2a107d3f5c 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/urls.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/urls.py @@ -246,6 +246,9 @@ url(r'^get_original_file/(?:(?P[0-9]+)/)?$', views.get_original_file, name="get_original_file"), # for stderr, stdout etc + url(r'^omero_table/(?P[0-9]+)/(?P((?i)json|csv))/$', + views.omero_table, + name="omero_table"), url(r'^download_original_file/(?:(?P[0-9]+)/)?$', views.get_original_file, {'download': True}, diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 44f9ea1b263..0fb3e8bd67d 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2937,6 +2937,40 @@ def get_original_file(request, fileId, download=False, conn=None, **kwargs): return rsp +@login_required() +@render_response() +def omero_table(request, file_id, mtype, download=False, conn=None, **kwargs): + # e.g. mtype = csv or json + + r = conn.getSharedResources() + t = r.openTable(omero.model.OriginalFileI(file_id), conn.SERVICE_OPTS) + if not t: + raise Http404('table not found') + + cols = t.getHeaders() + rows = t.getNumberOfRows() + + hits = range(rows) + + col_names = [col.name for col in cols] + rows = [] + for hit in hits: + row = [col.values[0] for col in t.read(range(len(cols)), hit, hit+1).columns] + rows.append(row) + + context = { + 'columns': col_names, + 'rows': rows, + } + print 'mtype', mtype + if mtype == 'html': + context['template'] = 'webclient/annotations/omero_table.html' + elif mtype == 'csv': + context['template'] = 'webclient/annotations/omero_table.csv' + + return context + + @login_required(doConnectionCleanup=False) def download_annotation(request, annId, conn=None, **kwargs): """ Returns the file annotation as an http response for download """ From fc387b0f8cf61ab53f294f79f915158f7b8d7da0 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 10:59:33 +0100 Subject: [PATCH 02/48] Use _table_query from webgateway.views --- .../tools/OmeroWeb/omeroweb/webclient/urls.py | 2 +- .../OmeroWeb/omeroweb/webclient/views.py | 32 +++++++------------ .../OmeroWeb/omeroweb/webgateway/views.py | 5 +-- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/urls.py b/components/tools/OmeroWeb/omeroweb/webclient/urls.py index b2a107d3f5c..ef67d83e486 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/urls.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/urls.py @@ -246,7 +246,7 @@ url(r'^get_original_file/(?:(?P[0-9]+)/)?$', views.get_original_file, name="get_original_file"), # for stderr, stdout etc - url(r'^omero_table/(?P[0-9]+)/(?P((?i)json|csv))/$', + url(r'^omero_table/(?P[0-9]+)/(?:(?P((?i)json|csv))/)?$', views.omero_table, name="omero_table"), url(r'^download_original_file/(?:(?P[0-9]+)/)?$', diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 0fb3e8bd67d..47407a8d71e 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2939,34 +2939,26 @@ def get_original_file(request, fileId, download=False, conn=None, **kwargs): @login_required() @render_response() -def omero_table(request, file_id, mtype, download=False, conn=None, **kwargs): +def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwargs): # e.g. mtype = csv or json - r = conn.getSharedResources() - t = r.openTable(omero.model.OriginalFileI(file_id), conn.SERVICE_OPTS) - if not t: - raise Http404('table not found') + result = webgateway_views._table_query(request, file_id, + query="*", conn=conn) - cols = t.getHeaders() - rows = t.getNumberOfRows() - - hits = range(rows) - - col_names = [col.name for col in cols] - rows = [] - for hit in hits: - row = [col.values[0] for col in t.read(range(len(cols)), hit, hit+1).columns] - rows.append(row) + if result.get('error') or not result.get('data'): + return JsonResponse(result) + table_data = result.get('data') context = { - 'columns': col_names, - 'rows': rows, + 'columns': table_data.get('columns'), + 'rows': table_data.get('rows'), } print 'mtype', mtype - if mtype == 'html': - context['template'] = 'webclient/annotations/omero_table.html' - elif mtype == 'csv': + + if mtype == 'csv': context['template'] = 'webclient/annotations/omero_table.csv' + elif mtype == None: + context['template'] = 'webclient/annotations/omero_table.html' return context diff --git a/components/tools/OmeroWeb/omeroweb/webgateway/views.py b/components/tools/OmeroWeb/omeroweb/webgateway/views.py index 8e68abd6a6d..c5a2f9355be 100644 --- a/components/tools/OmeroWeb/omeroweb/webgateway/views.py +++ b/components/tools/OmeroWeb/omeroweb/webgateway/views.py @@ -2765,7 +2765,7 @@ def _annotations(request, objtype, objid, conn=None, **kwargs): annotations = login_required()(jsonp(_annotations)) -def _table_query(request, fileid, conn=None, **kwargs): +def _table_query(request, fileid, query=None, conn=None, **kwargs): """ Query a table specified by fileid Returns a dictionary with query result if successful, error information @@ -2784,7 +2784,8 @@ def _table_query(request, fileid, conn=None, **kwargs): 'columns' (an array of column names) and 'rows' (an array of rows, each an array of values) """ - query = request.GET.get('query') + if query is None: + query = request.GET.get('query') if not query: return dict( error='Must specify query parameter, use * to retrieve all') From 4f972604fdd047d5026036c7f3c6d5efde8fcccd Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 15:45:01 +0100 Subject: [PATCH 03/48] Open table in new tab --- .../templates/webclient/annotations/fileanns_underscore.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html index d0b9564c5b8..05e93a7a544 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html @@ -46,8 +46,8 @@
<% if (ann.ns && ann.ns === 'openmicroscopy.org/omero/bulk_annotations'){ %> - + <% } %> From f78eb816ea69656d982a15d4f38a17f46cfb8b09 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 15:45:29 +0100 Subject: [PATCH 04/48] omero_table html page has links to csv and json --- .../webclient/annotations/omero_table.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index 189407aa580..fae6397c959 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -2,13 +2,27 @@ +

Table: {{ name }}

+

+ Download as CSV + | + View as JSON +

{{ col }}
{{ col }}
{% for col in columns %} From b676aad135f649ae78f7d8dbf8351273befd73a1 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 15:45:59 +0100 Subject: [PATCH 05/48] Handle file not found --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 47407a8d71e..2d66bc09651 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2942,6 +2942,12 @@ def get_original_file(request, fileId, download=False, conn=None, **kwargs): def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwargs): # e.g. mtype = csv or json + # Check if file exists since _table_query() doesn't check + file_id = long(file_id) + orig_file = conn.getQueryService().find('OriginalFile', file_id) + if orig_file is None: + raise Http404("OriginalFile %s not found" % file_id); + result = webgateway_views._table_query(request, file_id, query="*", conn=conn) From 2e3a831d075b135b85f1b58230435a4f40d5b176 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 15:46:28 +0100 Subject: [PATCH 06/48] download csv table --- .../tools/OmeroWeb/omeroweb/webclient/views.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 2d66bc09651..e14d8786a33 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2958,11 +2958,26 @@ def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwarg context = { 'columns': table_data.get('columns'), 'rows': table_data.get('rows'), + 'name': orig_file.name.val, + 'path': orig_file.path.val, + 'id': file_id, } - print 'mtype', mtype + # by default, return context as JSON data + # OR, return as csv or html if mtype == 'csv': context['template'] = 'webclient/annotations/omero_table.csv' + csv_rows = [",".join(table_data.get('columns'))] + for row in table_data.get('rows'): + csv_rows.append(",".join([str(r).replace(',','.') for r in row])) + csv_data = '\n'.join(csv_rows) + rsp = HttpResponse(csv_data, content_type='text/csv') + rsp['Content-Type'] = 'application/force-download' + rsp['Content-Length'] = len(csv_data) + downloadName = orig_file.name.val.replace(" ", "_").replace(",", ".") + downloadName = downloadName + ".csv" + rsp['Content-Disposition'] = 'attachment; filename=%s' % downloadName + return rsp elif mtype == None: context['template'] = 'webclient/annotations/omero_table.html' From 5e1f6310471a35876aff660788e03204a8c685c8 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 15:47:20 +0100 Subject: [PATCH 07/48] Remove unused omero_table.csv template --- .../webclient/templates/webclient/annotations/omero_table.csv | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv deleted file mode 100644 index 3a3904ea582..00000000000 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.csv +++ /dev/null @@ -1,2 +0,0 @@ -{% for col in columns %}{{ col }},{% endfor %} -{% for row in rows %}{% for col in row %}{{ col }},{% endfor %}\n{% endfor %} \ No newline at end of file From 4adb04db1136617b7f600143789243e80f32e734 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 17:47:04 +0100 Subject: [PATCH 08/48] json data wrap in 'data' --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index e14d8786a33..fe33c063344 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2966,7 +2966,6 @@ def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwarg # by default, return context as JSON data # OR, return as csv or html if mtype == 'csv': - context['template'] = 'webclient/annotations/omero_table.csv' csv_rows = [",".join(table_data.get('columns'))] for row in table_data.get('rows'): csv_rows.append(",".join([str(r).replace(',','.') for r in row])) @@ -2980,6 +2979,9 @@ def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwarg return rsp elif mtype == None: context['template'] = 'webclient/annotations/omero_table.html' + else: + # json: nest everything in 'data' + context = {'data': context} return context From 51892f6d32bcf3c59b2b76ca7daf50b8db11c93c Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 17:48:08 +0100 Subject: [PATCH 09/48] Add test for omero_table as csv, json or html --- .../test/integration/test_plategrid.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/test/integration/test_plategrid.py b/components/tools/OmeroWeb/test/integration/test_plategrid.py index 933f5967345..bc4ed1a0047 100644 --- a/components/tools/OmeroWeb/test/integration/test_plategrid.py +++ b/components/tools/OmeroWeb/test/integration/test_plategrid.py @@ -10,7 +10,7 @@ """ import pytest -from omeroweb.testlib import IWebTest +from omeroweb.testlib import IWebTest, get, get_json from omero.model import PlateI, WellI, WellSampleI from omero.model import FileAnnotationI, OriginalFileI, PlateAnnotationLinkI @@ -432,3 +432,47 @@ def test_get_plate_table(self, django_client, plate_well_table, conn): assert rspJson['addedBy'] == userName assert rspJson['owner'] == userName assert rspJson['parentType'] == "Plate" + + def test_table_html(self, django_client, plate_well_table, conn): + """ + Do a simple GET request to query the metadata for a single well + attached to the plate in JSON form + """ + plate, wellIds = plate_well_table + wellId = wellIds[0] + + request_url = reverse("webgateway_annotations", + args=["Plate", plate.id.val]) + rsp = get_json(django_client, request_url) + assert len(rsp['data']) == 1 + file_id = rsp['data'][0]['file'] + + # expected table data + cols = ['Well', 'TestColumn'] + rows = [[wellId, 'foobar']] + + # GET json + request_url = reverse("omero_table", args=[file_id, 'json']) + rsp = get_json(django_client, request_url) + assert rsp['data']['rows'] == rows + assert rsp['data']['columns'] == cols + assert rsp['data']['name'].startswith('plate_well_table_test') + assert rsp['data']['id'] == file_id + + # GET html + request_url = reverse("omero_table", args=[file_id]) + rsp = get(django_client, request_url) + html = rsp.content + for col in cols: + assert ('' % col) in html + for row in rows: + for td in row: + assert ('' % td) in html + + # GET csv + request_url = reverse("omero_table", args=[file_id, 'csv']) + rsp = get(django_client, request_url) + csv_data = rsp.content + cols_csv = ','.join(cols) + rows_csv = '/n'.join([','.join([str(td) for td in row]) for row in rows]) + assert csv_data == '%s\n%s' % (cols_csv, rows_csv) From 6b0759bcc64e92b644af2ce1c230ea95a041b931 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2019 22:09:00 +0100 Subject: [PATCH 10/48] flake8 fixes and docstring --- .../tools/OmeroWeb/omeroweb/webclient/views.py | 16 +++++++++++----- .../OmeroWeb/test/integration/test_plategrid.py | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index fe33c063344..ef8b9f8f556 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2939,14 +2939,20 @@ def get_original_file(request, fileId, download=False, conn=None, **kwargs): @login_required() @render_response() -def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwargs): - # e.g. mtype = csv or json +def omero_table(request, file_id, mtype=None, conn=None, **kwargs): + """ + Download OMERO.table as CSV or show as HTML table + + @param file_id: OriginalFile ID + @param mtype: None for html table or 'csv' or 'json' + @param conn: BlitzGateway connection + """ # Check if file exists since _table_query() doesn't check file_id = long(file_id) orig_file = conn.getQueryService().find('OriginalFile', file_id) if orig_file is None: - raise Http404("OriginalFile %s not found" % file_id); + raise Http404("OriginalFile %s not found" % file_id) result = webgateway_views._table_query(request, file_id, query="*", conn=conn) @@ -2968,7 +2974,7 @@ def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwarg if mtype == 'csv': csv_rows = [",".join(table_data.get('columns'))] for row in table_data.get('rows'): - csv_rows.append(",".join([str(r).replace(',','.') for r in row])) + csv_rows.append(",".join([str(r).replace(',', '.') for r in row])) csv_data = '\n'.join(csv_rows) rsp = HttpResponse(csv_data, content_type='text/csv') rsp['Content-Type'] = 'application/force-download' @@ -2977,7 +2983,7 @@ def omero_table(request, file_id, mtype=None, download=False, conn=None, **kwarg downloadName = downloadName + ".csv" rsp['Content-Disposition'] = 'attachment; filename=%s' % downloadName return rsp - elif mtype == None: + elif mtype is None: context['template'] = 'webclient/annotations/omero_table.html' else: # json: nest everything in 'data' diff --git a/components/tools/OmeroWeb/test/integration/test_plategrid.py b/components/tools/OmeroWeb/test/integration/test_plategrid.py index bc4ed1a0047..be99d32efbb 100644 --- a/components/tools/OmeroWeb/test/integration/test_plategrid.py +++ b/components/tools/OmeroWeb/test/integration/test_plategrid.py @@ -474,5 +474,6 @@ def test_table_html(self, django_client, plate_well_table, conn): rsp = get(django_client, request_url) csv_data = rsp.content cols_csv = ','.join(cols) - rows_csv = '/n'.join([','.join([str(td) for td in row]) for row in rows]) + rows_csv = '/n'.join([','.join( + [str(td) for td in row]) for row in rows]) assert csv_data == '%s\n%s' % (cols_csv, rows_csv) From 7464d53599a49fff61311b0f8c7e198653ff14b3 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jun 2019 23:15:29 +0100 Subject: [PATCH 11/48] Remove json link --- .../webclient/templates/webclient/annotations/omero_table.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index fae6397c959..d77e64c9f15 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -20,8 +20,6 @@

Table: {{ name }}

Download as CSV - | - View as JSON

%s%s
From be1409107b605a0b26ae5fc404f65961ca2dcef0 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jun 2019 23:16:46 +0100 Subject: [PATCH 12/48] Fix test failures - switch order of query param --- components/tools/OmeroWeb/omeroweb/webgateway/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webgateway/views.py b/components/tools/OmeroWeb/omeroweb/webgateway/views.py index c5a2f9355be..e740359a6ab 100644 --- a/components/tools/OmeroWeb/omeroweb/webgateway/views.py +++ b/components/tools/OmeroWeb/omeroweb/webgateway/views.py @@ -2765,7 +2765,7 @@ def _annotations(request, objtype, objid, conn=None, **kwargs): annotations = login_required()(jsonp(_annotations)) -def _table_query(request, fileid, query=None, conn=None, **kwargs): +def _table_query(request, fileid, conn=None, query=None, **kwargs): """ Query a table specified by fileid Returns a dictionary with query result if successful, error information From e6e22cbd53c99e18dca9d1731461acc780ae2c43 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 17 Jun 2019 23:19:06 +0100 Subject: [PATCH 13/48] Allow e.g. ?query=Image>123 in URL query --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index ef8b9f8f556..349a0bfedbd 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2948,6 +2948,7 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): @param conn: BlitzGateway connection """ + query = request.GET.get('query', '*') # Check if file exists since _table_query() doesn't check file_id = long(file_id) orig_file = conn.getQueryService().find('OriginalFile', file_id) @@ -2955,7 +2956,7 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): raise Http404("OriginalFile %s not found" % file_id) result = webgateway_views._table_query(request, file_id, - query="*", conn=conn) + conn=conn, query=query) if result.get('error') or not result.get('data'): return JsonResponse(result) From 046234d3baf38db250dedfd409342640f101d0bd Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 18 Jun 2019 12:19:57 +0100 Subject: [PATCH 14/48] Support pagination for omero_table --- .../webclient/annotations/omero_table.html | 32 ++++++++++++-- .../OmeroWeb/omeroweb/webclient/views.py | 43 ++++++++++++------- .../OmeroWeb/omeroweb/webgateway/views.py | 34 +++++++++++---- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index d77e64c9f15..e06e625ba09 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -17,18 +17,42 @@ -

Table: {{ name }}

+

+ {{ data.name }}   +

+

Download table as CSV

- Download as CSV + {% if meta.query == '*' %} + Table rows: {{ meta.rowCount }}. + {% else %} + Query "{{ meta.query }}" returned + {{ meta.totalCount }}/{{ meta.rowCount }} rows. + {% endif %} + {% if meta.page %} + Page {{ meta.page }} ({{ meta.limit }} rows). + {% else %} + Pagination offset: {{ meta.offset }} limit: {{ meta.limit }}. + {% endif %} + {% if meta.prev %} + Prev + {% else %} + Prev + {% endif %} + | + {% if meta.next %} + Next + {% else %} + Next + {% endif %}

- {% for col in columns %} + {% for col in data.columns %} {% endfor %} - {% for row in rows %} + {% for row in data.rows %} {% for col in row %} diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 349a0bfedbd..61de95da767 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2949,30 +2949,44 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): """ query = request.GET.get('query', '*') + offset = get_long_or_default(request, 'offset', 0) + limit = get_long_or_default(request, 'limit', settings.PAGE) + # Check if file exists since _table_query() doesn't check file_id = long(file_id) - orig_file = conn.getQueryService().find('OriginalFile', file_id) + orig_file = conn.getQueryService().find('OriginalFile', file_id, + conn.SERVICE_OPTS) if orig_file is None: raise Http404("OriginalFile %s not found" % file_id) - result = webgateway_views._table_query(request, file_id, - conn=conn, query=query) + context = webgateway_views._table_query(request, file_id, conn=conn, + query=query, offset=offset, + limit=limit) - if result.get('error') or not result.get('data'): - return JsonResponse(result) + if context.get('error') or not context.get('data'): + return JsonResponse(context) + + context['data']['name'] = orig_file.name.val + context['data']['path'] = orig_file.path.val + context['data']['id'] = file_id + context['meta']['query'] = query + + # if we're on an exact page: + if offset==0 or float(offset)/limit == offset/limit: + context['meta']['page'] = (offset/limit) + 1 if offset > 0 else 1 + + url = reverse('omero_table', args=[file_id]) + url += '?query=%s&limit=%s' % (query, limit) + if (offset + limit) < context['meta']['rowCount']: + context['meta']['next'] = url + '&offset=%s' % (offset + limit) + if offset > 0: + context['meta']['prev'] = url + '&offset=%s' % (max(0, offset - limit)) - table_data = result.get('data') - context = { - 'columns': table_data.get('columns'), - 'rows': table_data.get('rows'), - 'name': orig_file.name.val, - 'path': orig_file.path.val, - 'id': file_id, - } # by default, return context as JSON data # OR, return as csv or html if mtype == 'csv': + table_data = context.get('data') csv_rows = [",".join(table_data.get('columns'))] for row in table_data.get('rows'): csv_rows.append(",".join([str(r).replace(',', '.') for r in row])) @@ -2986,9 +3000,6 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): return rsp elif mtype is None: context['template'] = 'webclient/annotations/omero_table.html' - else: - # json: nest everything in 'data' - context = {'data': context} return context diff --git a/components/tools/OmeroWeb/omeroweb/webgateway/views.py b/components/tools/OmeroWeb/omeroweb/webgateway/views.py index e740359a6ab..c3345b3b226 100644 --- a/components/tools/OmeroWeb/omeroweb/webgateway/views.py +++ b/components/tools/OmeroWeb/omeroweb/webgateway/views.py @@ -2801,24 +2801,42 @@ def _table_query(request, fileid, conn=None, query=None, **kwargs): cols = t.getHeaders() rows = t.getNumberOfRows() + offset = kwargs.get('offset', 0) + limit = kwargs.get('limit', None) + + range_start = offset + range_end = kwargs.get('limit', rows) + range_end = min(rows, range_start + range_end) + if query == '*': - hits = range(rows) + hits = range(range_start, range_end) + totalCount = rows else: match = re.match(r'^(\w+)-(\d+)', query) if match: query = '(%s==%s)' % (match.group(1), match.group(2)) try: hits = t.getWhereList(query, None, 0, rows, 1) + totalCount = len(hits) + # paginate the hits + hits = hits[range_start: range_end] except Exception: return dict(error='Error executing query: %s' % query) - return dict(data=dict( - columns=[col.name for col in cols], - rows=[[col.values[0] for col in t.read(range(len(cols)), hit, - hit+1).columns] - for hit in hits], - ) - ) + return { + 'data': { + 'columns': [col.name for col in cols], + 'rows': [[col.values[0] for col in t.read(range(len(cols)), hit, + hit+1).columns] + for hit in hits] + }, + 'meta': { + 'rowCount': rows, + 'totalCount': totalCount, + 'limit': limit, + 'offset': offset, + } + } table_query = login_required()(jsonp(_table_query)) From 9cca128f3ce98f3632d04494e43d1a41b0533fa6 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 18 Jun 2019 22:15:42 +0100 Subject: [PATCH 15/48] flake8 fixes --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 61de95da767..3e329317eca 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2960,8 +2960,8 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): raise Http404("OriginalFile %s not found" % file_id) context = webgateway_views._table_query(request, file_id, conn=conn, - query=query, offset=offset, - limit=limit) + query=query, offset=offset, + limit=limit) if context.get('error') or not context.get('data'): return JsonResponse(context) @@ -2972,7 +2972,7 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): context['meta']['query'] = query # if we're on an exact page: - if offset==0 or float(offset)/limit == offset/limit: + if offset == 0 or float(offset)/limit == offset/limit: context['meta']['page'] = (offset/limit) + 1 if offset > 0 else 1 url = reverse('omero_table', args=[file_id]) @@ -2982,7 +2982,6 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): if offset > 0: context['meta']['prev'] = url + '&offset=%s' % (max(0, offset - limit)) - # by default, return context as JSON data # OR, return as csv or html if mtype == 'csv': From 936be8a2ba0418e4374011199c64b496d09cc79a Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 18 Jun 2019 22:40:17 +0100 Subject: [PATCH 16/48] Handle file not found - fix SecurityViolation --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 3e329317eca..14fa1619aa4 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2954,8 +2954,7 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): # Check if file exists since _table_query() doesn't check file_id = long(file_id) - orig_file = conn.getQueryService().find('OriginalFile', file_id, - conn.SERVICE_OPTS) + orig_file = conn.getObject('OriginalFile', file_id) if orig_file is None: raise Http404("OriginalFile %s not found" % file_id) @@ -2966,8 +2965,8 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): if context.get('error') or not context.get('data'): return JsonResponse(context) - context['data']['name'] = orig_file.name.val - context['data']['path'] = orig_file.path.val + context['data']['name'] = orig_file.name + context['data']['path'] = orig_file.path context['data']['id'] = file_id context['meta']['query'] = query From b3b29f2c2a5c05438817f39d0cfcb7b6d9726c7c Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 00:06:07 +0100 Subject: [PATCH 17/48] Add column_types to _table_query dict --- components/tools/OmeroWeb/omeroweb/webgateway/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/tools/OmeroWeb/omeroweb/webgateway/views.py b/components/tools/OmeroWeb/omeroweb/webgateway/views.py index c3345b3b226..d80ee95baf1 100644 --- a/components/tools/OmeroWeb/omeroweb/webgateway/views.py +++ b/components/tools/OmeroWeb/omeroweb/webgateway/views.py @@ -2825,6 +2825,7 @@ def _table_query(request, fileid, conn=None, query=None, **kwargs): return { 'data': { + 'column_types': [col.__class__.__name__ for col in cols], 'columns': [col.name for col in cols], 'rows': [[col.values[0] for col in t.read(range(len(cols)), hit, hit+1).columns] @@ -2838,6 +2839,7 @@ def _table_query(request, fileid, conn=None, query=None, **kwargs): } } + table_query = login_required()(jsonp(_table_query)) From 2814a606641fbb1d59074e148f3919cfcbea3114 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 00:06:37 +0100 Subject: [PATCH 18/48] Add example queries to omero_table page --- .../webclient/annotations/omero_table.html | 11 +++++++--- .../OmeroWeb/omeroweb/webclient/views.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index e06e625ba09..e273e9082e6 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -17,9 +17,14 @@ -

- {{ data.name }}   -

+

{{ data.name }}

+ {% if query_eg %} +

+ NB: query rows via URL query. E.g. + {{ query_eg }} + or {{ query_eg2 }} +

+ {% endif %}

Download table as CSV

{% if meta.query == '*' %} diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 14fa1619aa4..6f29fe00c4b 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2974,13 +2974,16 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): if offset == 0 or float(offset)/limit == offset/limit: context['meta']['page'] = (offset/limit) + 1 if offset > 0 else 1 + # pagination links url = reverse('omero_table', args=[file_id]) + context['meta']['url'] = url url += '?query=%s&limit=%s' % (query, limit) if (offset + limit) < context['meta']['rowCount']: context['meta']['next'] = url + '&offset=%s' % (offset + limit) if offset > 0: context['meta']['prev'] = url + '&offset=%s' % (max(0, offset - limit)) + # by default, return context as JSON data # OR, return as csv or html if mtype == 'csv': @@ -2998,6 +3001,23 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): return rsp elif mtype is None: context['template'] = 'webclient/annotations/omero_table.html' + # provide example queries - pick first DoubleColumn... + for idx, c_type in enumerate(context['data']['column_types']): + if c_type == 'DoubleColumn': + col_name = context['data']['columns'][idx] + if ' ' in col_name: + # Don't support queries on columns with spaces + continue + col_val1 = context['data']['rows'][0][idx] + col_val2 = context['data']['rows'][1][idx] + context['query_eg'] = '%s>%s' % (col_name, + min(col_val1, col_val2)) + greater = '(%s>=%s)' % (col_name, min(col_val1, col_val2)) + lower = '(%s<%s)' % (col_name, max(col_val1, col_val2)) + context['query_eg2'] = greater + '&' + lower + # Need to replace '&' for this to be valid query string + context['eg2_url'] = greater + '%26' + lower + break return context From f58c166f6517dc321eecff61e752c009e5ade9a7 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 14:47:10 +0100 Subject: [PATCH 19/48] flake8 fix --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index 6f29fe00c4b..a5950cb7dc2 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2983,7 +2983,6 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): if offset > 0: context['meta']['prev'] = url + '&offset=%s' % (max(0, offset - limit)) - # by default, return context as JSON data # OR, return as csv or html if mtype == 'csv': From 0e0dfc936c4d416c5846a9ba708da1efffb7f047 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 15:44:38 +0100 Subject: [PATCH 20/48] Improve text and links on table page --- .../webclient/annotations/omero_table.html | 73 +++++++++++++------ .../OmeroWeb/omeroweb/webclient/views.py | 16 ++-- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index e273e9082e6..f5227ce2461 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -13,19 +13,41 @@ border: solid #ddd 1px; padding: 5px; } + .download { + position: absolute; + top: 10px; + right: 20px; + } + .back { + position: absolute; + top: 10px; + left: 20px; + } -

{{ data.name }}

- {% if query_eg %} -

- NB: query rows via URL query. E.g. - {{ query_eg }} - or {{ query_eg2 }} -

+
+ Download table as CSV +
+ {% if meta.query != '*' %} +
+ < Remove query +
{% endif %} -

Download table as CSV

+

{{ data.name }}

+

+ To filter rows you can use a query based on named columns. + For example, to filter for rows where {{ example_column }} + is greater than {{ example_min_value }} add + + ?query={{ example_column }}>{{ example_min_value }} + to the URL.
+ For a more complex example, try + + ?query=({{ example_column }}>{{ example_min_value }})&({{ example_column }}<{{ example_max_value }}) + +

{% if meta.query == '*' %} Table rows: {{ meta.rowCount }}. @@ -33,21 +55,26 @@

{{ data.name }}

Query "{{ meta.query }}" returned {{ meta.totalCount }}/{{ meta.rowCount }} rows. {% endif %} - {% if meta.page %} - Page {{ meta.page }} ({{ meta.limit }} rows). - {% else %} - Pagination offset: {{ meta.offset }} limit: {{ meta.limit }}. - {% endif %} - {% if meta.prev %} - Prev - {% else %} - Prev - {% endif %} - | - {% if meta.next %} - Next - {% else %} - Next + + +
+ {% if meta.prev or meta.next %} + {% if meta.page %} + Showing page {{ meta.page }} ({{ data.rows | length }} rows). + {% else %} + Pagination offset: {{ meta.offset }} limit: {{ meta.limit }}. + {% endif %} + {% if meta.prev %} + Prev + {% else %} + Prev + {% endif %} + | + {% if meta.next %} + Next + {% else %} + Next + {% endif %} {% endif %}

{{ col }}
{{ col }}
diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index a5950cb7dc2..bfc7fc303a9 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2977,8 +2977,10 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): # pagination links url = reverse('omero_table', args=[file_id]) context['meta']['url'] = url - url += '?query=%s&limit=%s' % (query, limit) - if (offset + limit) < context['meta']['rowCount']: + url += '?limit=%s' % limit + if query != '*': + url += '&query=%s' % query + if (offset + limit) < context['meta']['totalCount']: context['meta']['next'] = url + '&offset=%s' % (offset + limit) if offset > 0: context['meta']['prev'] = url + '&offset=%s' % (max(0, offset - limit)) @@ -3009,13 +3011,9 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): continue col_val1 = context['data']['rows'][0][idx] col_val2 = context['data']['rows'][1][idx] - context['query_eg'] = '%s>%s' % (col_name, - min(col_val1, col_val2)) - greater = '(%s>=%s)' % (col_name, min(col_val1, col_val2)) - lower = '(%s<%s)' % (col_name, max(col_val1, col_val2)) - context['query_eg2'] = greater + '&' + lower - # Need to replace '&' for this to be valid query string - context['eg2_url'] = greater + '%26' + lower + context['example_column'] = col_name + context['example_min_value'] = min(col_val1, col_val2) + context['example_max_value'] = max(col_val1, col_val2) break return context From a7393f197c04a0f23158a16dfdb33dbd684aaed2 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 16:31:33 +0100 Subject: [PATCH 21/48] Fix table name in csv download --- components/tools/OmeroWeb/omeroweb/webclient/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/views.py b/components/tools/OmeroWeb/omeroweb/webclient/views.py index bfc7fc303a9..968fffea997 100755 --- a/components/tools/OmeroWeb/omeroweb/webclient/views.py +++ b/components/tools/OmeroWeb/omeroweb/webclient/views.py @@ -2996,7 +2996,7 @@ def omero_table(request, file_id, mtype=None, conn=None, **kwargs): rsp = HttpResponse(csv_data, content_type='text/csv') rsp['Content-Type'] = 'application/force-download' rsp['Content-Length'] = len(csv_data) - downloadName = orig_file.name.val.replace(" ", "_").replace(",", ".") + downloadName = orig_file.name.replace(" ", "_").replace(",", ".") downloadName = downloadName + ".csv" rsp['Content-Disposition'] = 'attachment; filename=%s' % downloadName return rsp From 9d568481e2e84fa0a0ad4cecb946fe29a96e57b1 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 16:37:33 +0100 Subject: [PATCH 22/48] Move table test to new test_table.py --- .../test/integration/test_plategrid.py | 45 -------- .../OmeroWeb/test/integration/test_table.py | 105 ++++++++++++++++++ 2 files changed, 105 insertions(+), 45 deletions(-) create mode 100644 components/tools/OmeroWeb/test/integration/test_table.py diff --git a/components/tools/OmeroWeb/test/integration/test_plategrid.py b/components/tools/OmeroWeb/test/integration/test_plategrid.py index be99d32efbb..674a0dbcc10 100644 --- a/components/tools/OmeroWeb/test/integration/test_plategrid.py +++ b/components/tools/OmeroWeb/test/integration/test_plategrid.py @@ -432,48 +432,3 @@ def test_get_plate_table(self, django_client, plate_well_table, conn): assert rspJson['addedBy'] == userName assert rspJson['owner'] == userName assert rspJson['parentType'] == "Plate" - - def test_table_html(self, django_client, plate_well_table, conn): - """ - Do a simple GET request to query the metadata for a single well - attached to the plate in JSON form - """ - plate, wellIds = plate_well_table - wellId = wellIds[0] - - request_url = reverse("webgateway_annotations", - args=["Plate", plate.id.val]) - rsp = get_json(django_client, request_url) - assert len(rsp['data']) == 1 - file_id = rsp['data'][0]['file'] - - # expected table data - cols = ['Well', 'TestColumn'] - rows = [[wellId, 'foobar']] - - # GET json - request_url = reverse("omero_table", args=[file_id, 'json']) - rsp = get_json(django_client, request_url) - assert rsp['data']['rows'] == rows - assert rsp['data']['columns'] == cols - assert rsp['data']['name'].startswith('plate_well_table_test') - assert rsp['data']['id'] == file_id - - # GET html - request_url = reverse("omero_table", args=[file_id]) - rsp = get(django_client, request_url) - html = rsp.content - for col in cols: - assert ('' % col) in html - for row in rows: - for td in row: - assert ('' % td) in html - - # GET csv - request_url = reverse("omero_table", args=[file_id, 'csv']) - rsp = get(django_client, request_url) - csv_data = rsp.content - cols_csv = ','.join(cols) - rows_csv = '/n'.join([','.join( - [str(td) for td in row]) for row in rows]) - assert csv_data == '%s\n%s' % (cols_csv, rows_csv) diff --git a/components/tools/OmeroWeb/test/integration/test_table.py b/components/tools/OmeroWeb/test/integration/test_table.py new file mode 100644 index 00000000000..78ea4850843 --- /dev/null +++ b/components/tools/OmeroWeb/test/integration/test_table.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (C) 2019 University of Dundee & Open Microscopy Environment. +# All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests for querying OMERO.tables.""" + +import pytest +from omeroweb.testlib import IWebTest, get, get_json +from test_api_projects import get_connection + +from omero.grid import WellColumn, StringColumn + +from django.core.urlresolvers import reverse +from random import random + + +class TestOmeroTables(IWebTest): + """Tests querying of OMERO.table.""" + + @pytest.fixture() + def user1(self): + """Return a new user in a read-annotate group.""" + group = self.new_group(perms='rwra--') + return self.new_client_and_user(group=group) + + @pytest.fixture() + def django_client(self, user1): + """Return new Django client.""" + conn = get_connection(user1) + user_name = conn.getUser().getName() + return self.new_django_client(user_name, user_name) + + @pytest.fixture() + def omero_table_file(self, user1): + """Create a new OMERO Table and returns the original file ID.""" + client = user1[0] + col1 = WellColumn('Well', '', []) + col2 = StringColumn('TestColumn', '', 64, []) + + columns = [col1, col2] + tablename = "plate_well_table_test:%s" % str(random()) + table = client.sf.sharedResources().newTable(1, tablename) + table.initialize(columns) + + wellIds = [1] + + data1 = WellColumn('Well', '', wellIds) + data2 = StringColumn('TestColumn', '', 64, ["foobar"]) + data = [data1, data2] + table.addData(data) + table.close() + + orig_file = table.getOriginalFile() + return orig_file.id.val + + def test_table_html(self, omero_table_file, django_client): + """Do a GET request to query table data.""" + file_id = omero_table_file + wellId = 1 + + # expected table data + cols = ['Well', 'TestColumn'] + rows = [[wellId, 'foobar']] + + # GET json + request_url = reverse("omero_table", args=[file_id, 'json']) + rsp = get_json(django_client, request_url) + assert rsp['data']['rows'] == rows + assert rsp['data']['columns'] == cols + assert rsp['data']['name'].startswith('plate_well_table_test') + assert rsp['data']['id'] == file_id + + # GET html + request_url = reverse("omero_table", args=[file_id]) + rsp = get(django_client, request_url) + html = rsp.content + for col in cols: + assert ('' % col) in html + for row in rows: + for td in row: + assert ('' % td) in html + + # GET csv + request_url = reverse("omero_table", args=[file_id, 'csv']) + rsp = get(django_client, request_url) + csv_data = rsp.content + cols_csv = ','.join(cols) + rows_csv = '/n'.join([','.join( + [str(td) for td in row]) for row in rows]) + assert csv_data == '%s\n%s' % (cols_csv, rows_csv) From 4699c8c1fe5be83df06bb14f9b964fcf2ef934f3 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 21:36:29 +0100 Subject: [PATCH 23/48] Use more data in tables test --- .../OmeroWeb/test/integration/test_table.py | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/components/tools/OmeroWeb/test/integration/test_table.py b/components/tools/OmeroWeb/test/integration/test_table.py index 78ea4850843..087e7e3a920 100644 --- a/components/tools/OmeroWeb/test/integration/test_table.py +++ b/components/tools/OmeroWeb/test/integration/test_table.py @@ -23,7 +23,7 @@ from omeroweb.testlib import IWebTest, get, get_json from test_api_projects import get_connection -from omero.grid import WellColumn, StringColumn +from omero.grid import WellColumn, StringColumn, DoubleColumn from django.core.urlresolvers import reverse from random import random @@ -46,50 +46,71 @@ def django_client(self, user1): return self.new_django_client(user_name, user_name) @pytest.fixture() - def omero_table_file(self, user1): + def table_data(self): + """Return column classes, column names & row data.""" + col_types = [ + WellColumn, StringColumn, DoubleColumn, DoubleColumn + ] + col_names = ["Well", "TestColumn", "SmallNumbers", "BigNumbers"] + rows = [ + [1, 'test', 0.5, 135345.0], + [2, 'string', 1.0, 345345.121], + [3, 'column', 0.75, 356575.012], + [4, 'data', 0.12345, 13579.0] + ] + return (col_types, col_names, rows) + + @pytest.fixture() + def omero_table_file(self, user1, table_data): """Create a new OMERO Table and returns the original file ID.""" client = user1[0] - col1 = WellColumn('Well', '', []) - col2 = StringColumn('TestColumn', '', 64, []) + col_types, col_names, rows = table_data + + columns = [] + for col_type, name in zip(col_types, col_names): + if col_type == StringColumn: + columns.append(StringColumn(name, '', 64, [])) + else: + columns.append(col_type(name, '', [])) - columns = [col1, col2] - tablename = "plate_well_table_test:%s" % str(random()) + tablename = "omero_table_test:%s" % str(random()) table = client.sf.sharedResources().newTable(1, tablename) table.initialize(columns) - wellIds = [1] + data = [] + for col_type, name, idx in zip(col_types, col_names, range(len(col_names))): + col_data = [row[idx] for row in rows] + if col_type == StringColumn: + data.append(StringColumn(name, '', 64, col_data)) + else: + data.append(col_type(name, '', col_data)) - data1 = WellColumn('Well', '', wellIds) - data2 = StringColumn('TestColumn', '', 64, ["foobar"]) - data = [data1, data2] table.addData(data) table.close() orig_file = table.getOriginalFile() return orig_file.id.val - def test_table_html(self, omero_table_file, django_client): + def test_table_html(self, omero_table_file, django_client, table_data): """Do a GET request to query table data.""" file_id = omero_table_file - wellId = 1 # expected table data - cols = ['Well', 'TestColumn'] - rows = [[wellId, 'foobar']] + col_types, col_names, rows = table_data # GET json request_url = reverse("omero_table", args=[file_id, 'json']) rsp = get_json(django_client, request_url) assert rsp['data']['rows'] == rows - assert rsp['data']['columns'] == cols - assert rsp['data']['name'].startswith('plate_well_table_test') + assert rsp['data']['columns'] == col_names + assert rsp['data']['name'].startswith('omero_table_test') assert rsp['data']['id'] == file_id # GET html request_url = reverse("omero_table", args=[file_id]) rsp = get(django_client, request_url) html = rsp.content - for col in cols: + for col in col_names: assert ('' % col) in html for row in rows: for td in row: @@ -99,7 +120,7 @@ def test_table_html(self, omero_table_file, django_client): request_url = reverse("omero_table", args=[file_id, 'csv']) rsp = get(django_client, request_url) csv_data = rsp.content - cols_csv = ','.join(cols) - rows_csv = '/n'.join([','.join( + cols_csv = ','.join(col_names) + rows_csv = '\n'.join([','.join( [str(td) for td in row]) for row in rows]) assert csv_data == '%s\n%s' % (cols_csv, rows_csv) From 365262118329f1d4b48effc72b06d2a3fed32c08 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 23:11:07 +0100 Subject: [PATCH 24/48] Add table tests for pagination and querying --- .../OmeroWeb/test/integration/test_table.py | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/components/tools/OmeroWeb/test/integration/test_table.py b/components/tools/OmeroWeb/test/integration/test_table.py index 087e7e3a920..c05190199b8 100644 --- a/components/tools/OmeroWeb/test/integration/test_table.py +++ b/components/tools/OmeroWeb/test/integration/test_table.py @@ -56,7 +56,8 @@ def table_data(self): [1, 'test', 0.5, 135345.0], [2, 'string', 1.0, 345345.121], [3, 'column', 0.75, 356575.012], - [4, 'data', 0.12345, 13579.0] + [4, 'data', 0.12345, 13579.0], + [5, 'five', 0.01, 500.05] ] return (col_types, col_names, rows) @@ -78,7 +79,8 @@ def omero_table_file(self, user1, table_data): table.initialize(columns) data = [] - for col_type, name, idx in zip(col_types, col_names, range(len(col_names))): + for col_type, name, idx in zip(col_types, col_names, + range(len(col_names))): col_data = [row[idx] for row in rows] if col_type == StringColumn: data.append(StringColumn(name, '', 64, col_data)) @@ -124,3 +126,39 @@ def test_table_html(self, omero_table_file, django_client, table_data): rows_csv = '\n'.join([','.join( [str(td) for td in row]) for row in rows]) assert csv_data == '%s\n%s' % (cols_csv, rows_csv) + + def test_table_pagination(self, omero_table_file, django_client, + table_data): + """Test pagination of table data as JSON.""" + file_id = omero_table_file + + # expected table data + col_types, col_names, rows = table_data + + # GET json + limit = 2 + request_url = reverse("omero_table", args=[file_id, 'json']) + for offset in [0, 2, 4]: + request_url += '?limit=%s&offset=%s' % (limit, offset) + rsp = get_json(django_client, request_url) + assert rsp['data']['rows'] == rows[offset: offset + limit] + + def test_table_query(self, omero_table_file, django_client, table_data): + """Test pagination of table data as JSON.""" + file_id = omero_table_file + + # expected table data + col_types, col_names, rows = table_data + queries = ['SmallNumbers>0.5', + '(SmallNumbers>=0.75)%26(BigNumbers<350000.5)', + 'SmallNumbers==0.01'] + filtered_rows = [ + [r for r in rows if r[2] > 0.5], + [r for r in rows if r[2] >= 0.75 and r[3] < 350000.5], + [r for r in rows if r[2] == 0.01]] + + for query, expected in zip(queries, filtered_rows): + request_url = reverse("omero_table", args=[file_id, 'json']) + request_url += '?query=%s' % query + rsp = get_json(django_client, request_url) + assert rsp['data']['rows'] == expected From 215ccf15b6a6e99361a097cee0782d12ea901b2a Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 19 Jun 2019 23:40:18 +0100 Subject: [PATCH 25/48] flake8 fix --- components/tools/OmeroWeb/test/integration/test_plategrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/tools/OmeroWeb/test/integration/test_plategrid.py b/components/tools/OmeroWeb/test/integration/test_plategrid.py index 674a0dbcc10..933f5967345 100644 --- a/components/tools/OmeroWeb/test/integration/test_plategrid.py +++ b/components/tools/OmeroWeb/test/integration/test_plategrid.py @@ -10,7 +10,7 @@ """ import pytest -from omeroweb.testlib import IWebTest, get, get_json +from omeroweb.testlib import IWebTest from omero.model import PlateI, WellI, WellSampleI from omero.model import FileAnnotationI, OriginalFileI, PlateAnnotationLinkI From 56070d95ccea03ef975084658a408b95eb23c3a4 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 21 Jun 2019 08:49:33 +0100 Subject: [PATCH 26/48] Fix btn_view icon when deployed with URL prefix --- .../webclient/annotations/fileanns_underscore.html | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html index 05e93a7a544..b2ade84e97b 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/fileanns_underscore.html @@ -46,10 +46,8 @@
<% if (ann.ns && ann.ns === 'openmicroscopy.org/omero/bulk_annotations'){ %> - - - +   <% } %> <% if (ann.link.permissions.canDelete) { %> From 0568265a463b6055d0da9864aa040b9ca7699076 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 21 Jun 2019 09:25:05 +0100 Subject: [PATCH 27/48] Improve picking example query or hide if not found --- .../webclient/annotations/omero_table.html | 29 +++++++++++-------- .../OmeroWeb/omeroweb/webclient/views.py | 15 ++++++---- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html index f5227ce2461..61743154926 100644 --- a/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html +++ b/components/tools/OmeroWeb/omeroweb/webclient/templates/webclient/annotations/omero_table.html @@ -2,6 +2,9 @@ - - - -
- Download as CSV - Whole Table | - This View -
- {% if meta.query != '*' %} -
- < Remove query -
- {% endif %} -

{{ data.name }}

- {% if example_column %} -

- To filter rows you can use a query based on named columns. - For example, to filter for rows where {{ example_column }} - is greater than {{ example_min_value }} add - - ?query={{ example_column }}>{{ example_min_value }} - to the URL.
- For a more complex example, try - - ?query=({{ example_column }}>{{ example_min_value }})&({{ example_column }}<{{ example_max_value }}) - -

- {% endif %} -

- {% if meta.query == '*' %} - Table rows: {{ meta.rowCount }}. - {% else %} - Query "{{ meta.query }}" returned - {{ meta.totalCount }}/{{ meta.rowCount }} rows. - {% endif %} - - -
- {% if meta.prev or meta.next %} - {% if meta.page %} - Showing page {{ meta.page }} ({{ data.rows | length }} rows). - {% else %} - Pagination offset: {{ meta.offset }} limit: {{ meta.limit }}. - {% endif %} - {% if meta.prev %} - Prev - {% else %} - Prev - {% endif %} - | - {% if meta.next %} - Next - {% else %} - Next - {% endif %} - {% endif %} -

-
%s%s%s%s%s
- - {% for col, col_type in data.columns|zip:data.column_types %} - - {% endfor %} - - - {% for row in data.rows %} - - {% for col in row %} - {% if image_column_index == forloop.counter0 %} - - {% elif well_column_index == forloop.counter0 %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - - \ No newline at end of file From cc9f221a9b844d505412ca20ce23c85e810132e8 Mon Sep 17 00:00:00 2001 From: jmoore Date: Tue, 5 Nov 2019 13:29:28 +0100 Subject: [PATCH 48/48] test_table: decode request content --- components/tools/OmeroWeb/test/integration/test_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/tools/OmeroWeb/test/integration/test_table.py b/components/tools/OmeroWeb/test/integration/test_table.py index 607257167ce..9f0e2194357 100644 --- a/components/tools/OmeroWeb/test/integration/test_table.py +++ b/components/tools/OmeroWeb/test/integration/test_table.py @@ -114,7 +114,7 @@ def test_table_html(self, omero_table_file, django_client, table_data): # GET html request_url = reverse("omero_table", args=[file_id]) rsp = get(django_client, request_url) - html = rsp.content + html = rsp.content.decode("utf-8") for col_type, col in zip(col_types, col_names): assert ('' % (col_type, col)) in html well_col_index = col_types.index('WellColumn') @@ -132,7 +132,7 @@ def test_table_html(self, omero_table_file, django_client, table_data): # GET csv request_url = reverse("omero_table", args=[file_id, 'csv']) rsp = get(django_client, request_url) - csv_data = rsp.content + csv_data = rsp.content.decode("utf-8") cols_csv = ','.join(col_names) rows_csv = '\n'.join([','.join( [str(td) for td in row]) for row in rows])
{{ col }}
{{ col }}{{ col }}{{ col }}
%s