Skip to content

Commit ac03de0

Browse files
authoredMar 11, 2025··
Closing several issues pre-production deployment (#233)
* Fix typo defined date for collection date * Removed upload metadata section from intranet until fixed * Lab_name is shown when no samples are found * split samples by submitting institution * Included submitting_institution prop in Sample model * Filtering bio_stats by submitting_institution instead * New function get_collecting_insts_from_lab(lab_name) * Getting ena and gisaid data from pre-processed state * Adapting samples queries to submitting_institution * Improving search sample query * Now graphic data is log-scaled if needed * Get samples per lab and ccaa from pre-processed data * Fixing gender-age data from host-info plots * Added samples per lab-ccaa, ena and gisaid data to graphic-json * Adapted data to submitting institution and improved time for ena and gisaid * Built needle plot from scratch instead of dash_bio. Added new functions * Fixed wrong label being shown in num_samples * Adding submitting_institution to api. Linting * Included submitting_institution to Sample model. Linting * Scale ydata when needed. Linting * Load data for lab and ccaa from preprocessed. Linting * Load ena and gisaid data from preprocessed. Linting * Built needle plot from scratch instead of dash_bio. Linting * Removed wrong pdb
1 parent 0bad9fe commit ac03de0

17 files changed

+633
-146
lines changed
 

‎core/api/utils/samples.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ def split_sample_data(data):
6060
.get_state_id()
6161
)
6262
split_data["sample"]["user"] = (
63-
core.utils.samples.get_user_id_from_collecting_institution(
64-
split_data["sample"]["collecting_institution"]
63+
core.utils.samples.get_user_id_from_submitting_institution(
64+
split_data["sample"]["submitting_institution"]
6565
)
6666
)
6767
if core.models.Sample.objects.all().exists():

‎core/api/views.py

+7-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,13 @@ def create_sample_data(request):
108108
return Response(error, status=status.HTTP_400_BAD_REQUEST)
109109
schema_id = schema_obj.get_schema_id()
110110
# check if sample id field and collecting_institution are in the request
111-
if "sequencing_sample_id" not in data or "collecting_institution" not in data:
111+
required_db_fields = [
112+
"sequencing_sample_id",
113+
"collecting_institution",
114+
"submitting_institution",
115+
]
116+
if any(field not in data for field in required_db_fields):
117+
print(f"ERROR. Missing: {[f for f in required_db_fields if f not in data]}")
112118
return Response(status=status.HTTP_400_BAD_REQUEST)
113119
# check if sample is already defined
114120
if core.utils.samples.get_sample_obj_from_sample_name(

‎core/models.py

+5
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def __str__(self):
2121
return self.user.username
2222

2323
def get_lab_name(self):
24+
# Mapped to submitting_institution field in Sample model
2425
return "%s" % (self.laboratory)
2526

2627
def get_lab_code(self):
@@ -703,6 +704,7 @@ class Sample(models.Model):
703704
sequencing_sample_id = models.CharField(max_length=80, null=True, blank=True)
704705
submitting_lab_sample_id = models.CharField(max_length=80, null=True, blank=True)
705706
collecting_institution = models.CharField(max_length=120, null=True, blank=True)
707+
submitting_institution = models.CharField(max_length=120, null=True, blank=True)
706708
sequence_file_R1_fastq = models.CharField(max_length=80, null=True, blank=True)
707709
sequence_file_R2_fastq = models.CharField(max_length=80, null=True, blank=True)
708710
sequence_file_R1_md5 = models.CharField(max_length=80, null=True, blank=True)
@@ -737,6 +739,9 @@ def get_collecting_lab_sample_id(self):
737739
def get_collecting_institution(self):
738740
return "%s" % (self.collecting_institution)
739741

742+
def get_submitting_institution(self):
743+
return "%s" % (self.submitting_institution)
744+
740745
def get_unique_id(self):
741746
return "%s" % (self.sample_unique_id)
742747

‎core/templates/core/intranet.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ <h5>Percentage of GISAID upload samples</h5>
291291
{% else %}
292292
<div class="col-md-7">
293293
<div class="card border-warning mb-3">
294-
<div class="card-header text-center text-warning"><h2 style="text-align:center">No samples are upload yet to Relecov Platform</h2> </div>
294+
<div class="card-header text-center text-warning"><h2 style="text-align:center">No samples found for your laboratory: {{ intra_data.lab|default:"Unknown" }}</h2> </div>
295295
</div> <!-- end card -->
296296
</div> <!-- end col-md-4 -->
297297
{% endif %}

‎core/templates/core/intranetSideBar.html

-6
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,6 @@
1717
<span>Overview</span></a>
1818
</li>
1919
<br>
20-
<li class="nav-item active">
21-
<a class="nav-link" href="/metadataForm">
22-
<i class="bi bi-upload"></i>
23-
<span>Upload Metadata</span></a>
24-
</li>
25-
<br>
2620
<li class="nav-item active">
2721
<a class="nav-link" href="/laboratoryContact">
2822
<i class="bi bi-person-lines-fill"></i>

‎core/templates/core/searchSample.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ <h2 class="card-title">Search sample for {{search_data.labs}}</h2>
125125
</div> <!-- end text input -->
126126
</div>
127127
<div class="row mb-3">
128-
<label class="col-sm-4 control-label" for="sDate" >Sample defined Date</label>
128+
<label class="col-sm-4 control-label" for="sDate" >Sample collection date</label>
129129
<div class="col-sm-7">
130130
<input class="form-control" type="date" name="sDate" id="sDate">
131131
</div>

‎core/utils/bioinfo_analysis.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def get_bio_analysis_stats_from_lab(lab_name=None):
1818
bio_stats["received"] = core.models.Sample.objects.count()
1919
else:
2020
lab_samples = core.models.Sample.objects.filter(
21-
collecting_institution__iexact=lab_name
21+
submitting_institution__iexact=lab_name
2222
)
2323
bio_stats["analized"] = (
2424
lab_samples.select_related("state_id")

‎core/utils/labs.py

+11
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,17 @@ def get_lab_name_from_user(user_obj):
3636
return ""
3737

3838

39+
def get_collecting_insts_from_lab(lab_name):
40+
"""Get a list of collecting institutions associated to the given
41+
laboratory (submitting_institution)
42+
"""
43+
return (
44+
core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
45+
.values_list("collecting_institution", flat=True)
46+
.distinct()
47+
)
48+
49+
3950
def update_contact_lab(old_data, new_data):
4051
"""Update the contact information. If any field is empty it will set the
4152
old value. In case that all new_data are empty returns than no changes

‎core/utils/plotly_graphics.py

+31-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def bar_graphic(data, col_names, legend, yaxis, options):
2525
marker_color=colors if "colors" in options else colors[idx - 1],
2626
)
2727
)
28+
fig = log_ydata_if_needed(fig, data[col_names[idx]], ratio=100)
2829

2930
# Customize aspect
3031
fig.update_traces(
@@ -45,7 +46,6 @@ def bar_graphic(data, col_names, legend, yaxis, options):
4546
)
4647
if "xaxis_tics" in options:
4748
fig.update_layout(xaxis=options["xaxis"])
48-
4949
plot_div = plot(fig, output_type="div", config={"displaylogo": False})
5050

5151
return plot_div
@@ -68,6 +68,7 @@ def line_graphic(x_data, y_data, options):
6868
title_font_color="green",
6969
title_font_size=20,
7070
)
71+
log_ydata_if_needed(fig, y_data, ratio=100)
7172
if "xaxis" in options:
7273
fig.update_layout(xaxis=options["xaxis"])
7374
plot_div = plot(fig, output_type="div", config={"displaylogo": False})
@@ -92,6 +93,8 @@ def histogram_graphic(data, col_names, options):
9293
xaxis_tickangle=-45,
9394
margin=dict(l=20, r=40, t=30, b=20),
9495
)
96+
ydata = data[col_names[1]]
97+
graph = log_ydata_if_needed(graph, ydata, ratio=100)
9598

9699
plot_div = plot(graph, output_type="div", config={"displaylogo": False})
97100
return plot_div
@@ -217,3 +220,30 @@ def needle_plot(m_data):
217220
)
218221
def update_needleplot(show_rangeslider):
219222
return True if show_rangeslider else False
223+
224+
225+
def log_ydata_if_needed(graph, ydata, ratio=100):
226+
"""Try to apply logaritmic scale to ydata if necessary
227+
228+
Args:
229+
graph (plotly.figure): Plotly figure ready to be renderized
230+
ydata (list): list of values or pandas.series (e.g. df[col])
231+
ratio (int): ratio threshold to apply log scale or not
232+
233+
Return:
234+
graph: Graph with ydata scaled with logarithmic scale
235+
"""
236+
min_score = min(ydata)
237+
max_score = max(ydata)
238+
if min_score == 0:
239+
drop_0s = [x for x in ydata if x != 0]
240+
if not drop_0s: # All data is 0 so do not scale
241+
return graph
242+
min_score = min(drop_0s)
243+
minmax_ratio = max_score / min_score
244+
if minmax_ratio >= ratio:
245+
try:
246+
graph.update_layout(yaxis_type="log", yaxis_dtick=1)
247+
except TypeError as e: # Input graph does not accept log scaling
248+
print(f"ERROR while trying to scale ydata: {e}")
249+
return graph

‎core/utils/public_db.py

+35-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Local imports
22
import core.models
33
import core.utils.plotly_graphics
4+
import dashboard.utils.generic_graphic_data
45
from django.db.models import Q
56

67

@@ -11,11 +12,11 @@ def get_public_accession_from_sample_lab(p_field, sample_objs=None):
1112
if sample_objs is None:
1213
return (
1314
core.models.PublicDatabaseValues.objects.filter(
14-
public_database_fieldID__property_name__exact=p_field,
15+
public_database_fieldID__property_name__iexact=p_field,
1516
)
1617
.exclude(Q(value__icontains="Not Provided") | Q(value__isnull=True))
1718
.values_list(
18-
"sampleID__collecting_institution",
19+
"sampleID__submitting_institution",
1920
"sampleID__sequencing_sample_id",
2021
"value",
2122
)
@@ -51,3 +52,35 @@ def get_public_information_from_sample(p_type, sample_id):
5152
public_database_fieldID__database_type__public_type_name__iexact=p_type,
5253
).values_list("public_database_fieldID__label_name", "value")
5354
return []
55+
56+
57+
def get_preprocessed_gisaid_data():
58+
"""Load preprocessed gisaid data in graphic_json table"""
59+
gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
60+
"intranet_gisaid_data"
61+
)
62+
if gisaid_data is None:
63+
# Execute the pre-processed task to get the data
64+
result = dashboard.utils.generic_process_data.pre_proc_intranet_gisaid_data()
65+
if "ERROR" in result:
66+
return result
67+
gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
68+
"intranet_gisaid_data"
69+
)
70+
return gisaid_data
71+
72+
73+
def get_preprocessed_ena_data():
74+
"""Load preprocessed ena data in graphic_json table"""
75+
ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
76+
"intranet_ena_data"
77+
)
78+
if ena_data is None:
79+
# Execute the pre-processed task to get the data
80+
result = dashboard.utils.generic_process_data.pre_proc_intranet_ena_data()
81+
if "ERROR" in result:
82+
return result
83+
ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
84+
"intranet_ena_data"
85+
)
86+
return ena_data

‎core/utils/samples.py

+39-24
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import json
33
import os
44
import shutil
5-
from collections import OrderedDict
5+
from collections import OrderedDict, defaultdict
66
from datetime import datetime
77
import pandas as pd
88
from django.contrib.auth.models import Group, User
@@ -78,10 +78,10 @@ def assign_samples_to_new_user(data):
7878
"""Assign all samples from a laboratory to a new userID"""
7979
user_obj = User.objects.filter(pk__exact=data["userName"])
8080
if core.models.core.models.Sample.objects.filter(
81-
collecting_institution__iexact=data["lab"]
81+
submitting_institution__iexact=data["lab"]
8282
).exists():
8383
core.models.core.models.Sample.objects.filter(
84-
collecting_institution__iexact=data["lab"]
84+
submitting_institution__iexact=data["lab"]
8585
).update(user=user_obj[0])
8686
return {"Success": "Success"}
8787
return {
@@ -304,8 +304,11 @@ def create_dash_bar_for_each_lab():
304304

305305
def perc_gauge_graphic(values):
306306
data = {}
307-
x = values["analized"] / values["received"] * 100
308-
data["value"] = float("{:.2f}".format(x))
307+
if values["received"] == 0:
308+
data["value"] = 0
309+
else:
310+
x = values["analized"] / values["received"] * 100
311+
data["value"] = float("{:.2f}".format(x))
309312
gauge_graph = core.utils.plotly_graphics.gauge_graphic(data)
310313
return gauge_graph
311314

@@ -327,11 +330,11 @@ def get_lab_last_actions(lab_name=None):
327330
if lab_name is None:
328331
lab_actions = []
329332
labs = core.models.core.models.Sample.objects.values_list(
330-
"collecting_institution"
333+
"submitting_institution"
331334
).distinct()
332335
for lab in labs:
333336
sam_obj = core.models.core.models.Sample.objects.filter(
334-
collecting_institution__exact=lab[0]
337+
submitting_institution__exact=lab[0]
335338
).last()
336339
lab_data = [lab[0]]
337340
for action in action_list:
@@ -352,7 +355,7 @@ def get_lab_last_actions(lab_name=None):
352355
else:
353356
actions = {}
354357
last_sample_obj = core.models.core.models.Sample.objects.filter(
355-
collecting_institution__iexact=lab_name
358+
submitting_institution__iexact=lab_name
356359
).last()
357360
action_objs = core.models.DateUpdateState.objects.filter(
358361
sampleID=last_sample_obj
@@ -403,7 +406,7 @@ def get_sample_display_data(sample_id, user):
403406
# Allow to see information obut sample to relecovManager
404407
group = Group.objects.get(name="RelecovManager")
405408
if group not in user.groups.all():
406-
lab_name = sample_obj.get_collecting_institution()
409+
lab_name = sample_obj.get_submitting_institution()
407410
if not core.models.Profile.objects.filter(
408411
user=user, laboratory__iexact=lab_name
409412
).exists():
@@ -530,22 +533,22 @@ def get_sample_per_date_per_lab(lab_name):
530533
samples_per_date = OrderedDict()
531534

532535
s_dates = (
533-
core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
534-
.values_list("sequencing_date", flat=True)
536+
core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
537+
.values_list("collecting_date", flat=True)
535538
.distinct()
536-
.order_by("sequencing_date")
539+
.order_by("collecting_date")
537540
)
538541
for s_date in s_dates:
539-
date = datetime.strftime(s_date, "%d-%B-%Y")
542+
date = datetime.strftime(s_date, "%Y-W%V")
540543
samples_per_date[date] = core.models.Sample.objects.filter(
541-
collecting_institution__iexact=lab_name, sequencing_date=s_date
544+
submitting_institution__iexact=lab_name, collecting_date=s_date
542545
).count()
543546
return samples_per_date
544547

545548

546549
def get_sample_objs_per_lab(lab_name):
547550
"""Get all sample instance for the lab who the user is responsible"""
548-
return core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
551+
return core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
549552

550553

551554
def get_search_data(user_obj):
@@ -570,9 +573,9 @@ def get_search_data(user_obj):
570573
return s_data
571574

572575

573-
def get_user_id_from_collecting_institution(lab):
574-
"""Use the laboratory name defined in the Profile to find out the user.
575-
if no user is not defined with this lab it retruns None
576+
def get_user_id_from_submitting_institution(lab):
577+
"""Use the laboratory name defined in the Profile (submitting-institution)
578+
to find out the user. if no user is not defined with this lab it returns None
576579
"""
577580
if core.models.Profile.objects.filter(laboratory__iexact=lab).exists():
578581
return core.models.Profile.objects.filter(laboratory__iexact=lab).last().user.pk
@@ -618,9 +621,9 @@ def join_sample_and_batch(b_data, user_obj, schema_obj):
618621
def get_all_lab_list():
619622
"""Function gets the lab names and return then in an ordered list"""
620623
return list(
621-
core.models.Sample.objects.values_list("collecting_institution", flat=True)
624+
core.models.Sample.objects.values_list("submitting_institution", flat=True)
622625
.distinct()
623-
.order_by("collecting_institution")
626+
.order_by("submitting_institution")
624627
)
625628

626629

@@ -717,9 +720,23 @@ def save_excel_form_in_samba_folder(m_file, user_name):
717720
def search_samples(sample_name, lab_name, sample_state, s_date, user):
718721
"""Search the samples that match with the query conditions"""
719722
sample_list = []
720-
sample_objs = core.models.Sample.objects.all()
723+
if s_date != "":
724+
samples_with_coldate = core.utils.rest_api.fetch_samples_on_condition(
725+
"collection_sample_date"
726+
)
727+
dates_samples_dict = defaultdict(list)
728+
if "ERROR" in samples_with_coldate:
729+
return samples_with_coldate
730+
for d in samples_with_coldate["DATA"]:
731+
dates_samples_dict[d["collection_sample_date"]].append(d["Sample Name"])
732+
sample_objs = core.models.Sample.objects.filter(
733+
Q(sequencing_sample_id__in=dates_samples_dict.get(s_date, []))
734+
| Q(collecting_lab_sample_id__iexact=dates_samples_dict.get(s_date, []))
735+
)
736+
else:
737+
sample_objs = core.models.Sample.objects.all()
721738
if lab_name != "":
722-
sample_objs = sample_objs.filter(collecting_institution__iexact=lab_name)
739+
sample_objs = sample_objs.filter(submitting_institution__iexact=lab_name)
723740
if sample_name != "":
724741
if sample_objs.filter(
725742
Q(sequencing_sample_id__iexact=sample_name)
@@ -753,8 +770,6 @@ def search_samples(sample_name, lab_name, sample_state, s_date, user):
753770
).values_list("sampleID__pk", flat=True)
754771
)
755772
sample_objs = sample_objs.filter(pk__in=state_ids)
756-
if s_date != "":
757-
sample_objs = sample_objs.filter(sequencing_date__exact=s_date)
758773
if len(sample_objs) == 1:
759774
sample_list.append(sample_objs[0].get_sample_id())
760775
return sample_list

‎core/utils/samples_graphics.py

+29-20
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,26 @@
22
import core.utils.plotly_graphics
33
import core.utils.rest_api
44
import core.utils.samples
5+
import dashboard.utils.generic_process_data
56

67

78
def received_per_ccaa():
89
"""Fetch the data from LIMS and show them in a graphic bar"""
9-
raw_data = core.utils.rest_api.get_summarize_data("")
10-
if "ERROR" in raw_data:
11-
return raw_data
12-
13-
data = {"x": [], "y": []}
14-
for key, value in raw_data["region"].items():
15-
data["x"].append(key)
16-
data["y"].append(value)
17-
10+
samples_per_ccaa = dashboard.utils.generic_graphic_data.get_graphic_json_data(
11+
"samples_received_per_ccaa"
12+
)
13+
if samples_per_ccaa is None:
14+
# Execute the pre-processed task to get the data
15+
result = (
16+
dashboard.utils.generic_process_data.pre_proc_samples_received_per_ccaa()
17+
)
18+
if "ERROR" in result:
19+
return result
20+
samples_per_ccaa = dashboard.utils.generic_graphic_data.get_graphic_json_data(
21+
"samples_received_per_ccaa"
22+
)
1823
return core.utils.plotly_graphics.bar_graphic(
19-
data=data,
24+
data=samples_per_ccaa,
2025
col_names=["x", "y"],
2126
legend=[""],
2227
yaxis={"title": "Number of samples"},
@@ -26,17 +31,21 @@ def received_per_ccaa():
2631

2732
def received_per_lab():
2833
"""Fetch the data from LIMS and show them in a graphic bar"""
29-
raw_data = core.utils.rest_api.get_summarize_data("")
30-
if "ERROR" in raw_data:
31-
return raw_data
32-
33-
data = {"x": [], "y": []}
34-
for key, value in raw_data["laboratory"].items():
35-
data["x"].append(key)
36-
data["y"].append(value)
37-
34+
samples_per_lab = dashboard.utils.generic_graphic_data.get_graphic_json_data(
35+
"samples_received_per_lab"
36+
)
37+
if samples_per_lab is None:
38+
# Execute the pre-processed task to get the data
39+
result = (
40+
dashboard.utils.generic_process_data.pre_proc_samples_received_per_lab()
41+
)
42+
if "ERROR" in result:
43+
return result
44+
samples_per_lab = dashboard.utils.generic_graphic_data.get_graphic_json_data(
45+
"samples_received_per_lab"
46+
)
3847
return core.utils.plotly_graphics.bar_graphic(
39-
data=data,
48+
data=samples_per_lab,
4049
col_names=["x", "y"],
4150
legend=[""],
4251
yaxis={"title": "Number of samples"},

‎core/views.py

+46-27
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import core.utils.samples_map
2121

2222
# End of imports received samples
23+
import time
2324

2425

2526
def index(request):
@@ -221,14 +222,28 @@ def metadata_visualization(request):
221222
def intranet(request):
222223
relecov_group = Group.objects.filter(name="RelecovManager").last()
223224
if relecov_group not in request.user.groups.all():
225+
start = time.time()
224226
intra_data = {}
225227
lab_name = core.utils.labs.get_lab_name_from_user(request.user)
226-
date_lab_samples = core.utils.samples.get_sample_per_date_per_lab(lab_name)
228+
related_insts_list = core.utils.labs.get_collecting_insts_from_lab(lab_name)
229+
all_sample_per_date_detailed = (
230+
core.utils.samples.get_sample_per_date_per_all_lab(detailed=True)
231+
)
232+
date_lab_samples = [
233+
x
234+
for x in all_sample_per_date_detailed
235+
if x["collecting_institution"] in related_insts_list
236+
]
237+
intra_data["lab"] = lab_name
238+
print(f"Took {start - time.time()} seconds for date_lab_samples")
227239
if len(date_lab_samples) > 0:
240+
start = time.time()
228241
sample_lab_objs = core.utils.samples.get_sample_objs_per_lab(lab_name)
242+
print(f"Took {start - time.time()} seconds for sample_lab_objs")
229243
analysis_percent = (
230244
core.utils.bioinfo_analysis.get_bio_analysis_stats_from_lab(lab_name)
231245
)
246+
print(f"Took {start - time.time()} seconds for analysis_percent")
232247
cust_data = {
233248
"col_names": ["Collecting Date", "Number of samples"],
234249
"options": {},
@@ -238,28 +253,30 @@ def intranet(request):
238253
intra_data["sample_bar_graph"] = core.utils.samples.create_date_sample_bar(
239254
date_lab_samples, cust_data
240255
)
256+
print(f"Took {start - time.time()} seconds for sample_bar_graph")
241257
intra_data["sample_gauge_graph"] = core.utils.samples.perc_gauge_graphic(
242258
analysis_percent
243259
)
260+
print(f"Took {start - time.time()} seconds for gauge_graph")
244261
intra_data["actions"] = core.utils.samples.get_lab_last_actions(lab_name)
245-
gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
246-
"gisaid_accession_id", sample_lab_objs
247-
)
248-
if len(gisaid_acc) > 0:
249-
intra_data["gisaid_accession"] = gisaid_acc
250-
intra_data["gisaid_graph"] = core.utils.public_db.percentage_graphic(
251-
len(sample_lab_objs), len(gisaid_acc), ""
252-
)
253-
ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
254-
"ena_sample_accession", sample_lab_objs
255-
)
256-
if len(ena_acc) > 0:
257-
intra_data["ena_accession"] = ena_acc
262+
gisaid_acc = core.utils.public_db.get_preprocessed_gisaid_data()
263+
# NOTE: Filters by lab_name instead of sample_lab_objs to improve performance, but could lead to mismatches
264+
print(f"Took {start - time.time()} seconds for gisaid data")
265+
if gisaid_acc:
266+
gisaid_acc_filtered = gisaid_acc.get(lab_name, [])
267+
intra_data["gisaid_accession"] = gisaid_acc_filtered
268+
intra_data["gisaid_graph"] = core.utils.public_db.percentage_graphic(
269+
len(sample_lab_objs), len(gisaid_acc_filtered), ""
270+
)
271+
print(f"Took {start - time.time()} seconds for gisaig graph")
272+
ena_acc = core.utils.public_db.get_preprocessed_ena_data()
273+
if ena_acc:
274+
ena_acc_filtered = ena_acc.get(lab_name, [])
275+
intra_data["ena_accession"] = ena_acc_filtered
258276
intra_data["ena_graph"] = core.utils.public_db.percentage_graphic(
259-
len(sample_lab_objs), len(ena_acc), ""
277+
len(sample_lab_objs), len(ena_acc_filtered), ""
260278
)
261-
else:
262-
intra_data = f"No samples found for selected laboratory: {lab_name}"
279+
print(f"Took {start - time.time()} seconds for ena graph")
263280
return render(request, "core/intranet.html", {"intra_data": intra_data})
264281
else:
265282
# loged user belongs to Relecov Manager group
@@ -290,25 +307,27 @@ def intranet(request):
290307
# Get the latest action from each lab
291308
manager_intra_data["actions"] = core.utils.samples.get_lab_last_actions()
292309
# Collect GISAID information
293-
gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
294-
"gisaid_accession_id", None
295-
)
310+
gisaid_acc = core.utils.public_db.get_preprocessed_gisaid_data()
311+
full_gisaid_data = [
312+
(lab, *tup) for lab, tuples in gisaid_acc.items() for tup in tuples
313+
]
296314
if len(gisaid_acc) > 0:
297-
manager_intra_data["gisaid_accession"] = gisaid_acc
315+
manager_intra_data["gisaid_accession"] = full_gisaid_data
298316
manager_intra_data["gisaid_graph"] = (
299317
core.utils.public_db.percentage_graphic(
300-
num_of_samples["Defined"], len(gisaid_acc), ""
318+
num_of_samples["Defined"], len(full_gisaid_data), ""
301319
)
302320
)
303321
# Collect Ena information
304-
ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
305-
"ena_sample_accession", None
306-
)
322+
ena_acc = core.utils.public_db.get_preprocessed_ena_data()
323+
full_ena_data = [
324+
(lab, *tup) for lab, tuples in ena_acc.items() for tup in tuples
325+
]
307326
if len(ena_acc) > 0:
308-
manager_intra_data["ena_accession"] = ena_acc
327+
manager_intra_data["ena_accession"] = full_ena_data
309328
manager_intra_data["ena_graph"] = (
310329
core.utils.public_db.percentage_graphic(
311-
num_of_samples["Defined"], len(ena_acc), ""
330+
num_of_samples["Defined"], len(full_ena_data), ""
312331
)
313332
)
314333
return render(

‎dashboard/cron.py

+12-4
Original file line numberDiff line numberDiff line change
@@ -77,16 +77,24 @@ def update_graphic_json_data():
7777
dashboard.utils.generic_process_data.pre_proc_samples_received_map()
7878
print("Running pre_proc_host_info()")
7979
dashboard.utils.generic_process_data.pre_proc_host_info()
80-
uniq_chrom_id_list = [
81-
x["chromosomeID"]
82-
for x in core.models.Gene.objects.values("chromosomeID").distinct()
83-
]
8480
print("Running pre_proc_samples_per_date_all_lab()")
8581
dashboard.utils.generic_process_data.pre_proc_samples_per_date_all_lab()
8682
print("Running pre_proc_samples_per_date_all_lab(detailed=True)")
8783
dashboard.utils.generic_process_data.pre_proc_samples_per_date_all_lab(
8884
detailed=True
8985
)
86+
print("Running pre_proc_samples_received_per_lab()")
87+
dashboard.utils.generic_process_data.pre_proc_samples_received_per_lab()
88+
print("Running pre_proc_samples_received_per_ccaa()")
89+
dashboard.utils.generic_process_data.pre_proc_samples_received_per_ccaa()
90+
print("Running pre_proc_intranet_gisaid_data")
91+
dashboard.utils.generic_process_data.pre_proc_intranet_gisaid_data()
92+
print("Running pre_proc_intranet_ena_data()")
93+
dashboard.utils.generic_process_data.pre_proc_intranet_ena_data()
94+
uniq_chrom_id_list = [
95+
x["chromosomeID"]
96+
for x in core.models.Gene.objects.values("chromosomeID").distinct()
97+
]
9098
print(f"List of extracted unique chromosomes: {uniq_chrom_id_list}")
9199
print("Running pre_proc_variations_per_lineage() for each chromosome")
92100
for chromosome in uniq_chrom_id_list:

‎dashboard/utils/generic_process_data.py

+94-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import core.utils.variants
1313
import core.utils.rest_api
1414
import core.utils.generic_functions
15+
import core.utils.public_db
1516
import dashboard.models
1617
from relecov_platform import settings as relecov_platform_settings
1718

@@ -618,10 +619,26 @@ def fetching_data_for_gender():
618619
host_info_json["gender_label"] = {"ERROR": gender_label}
619620
host_info_json["gender_values"] = {"ERROR": gender_values}
620621
else:
621-
host_info_json["gender_label"] = gender_label
622-
host_info_json["gender_values"] = gender_values
622+
label_val_dict = dict(zip(gender_label, gender_values))
623+
empty_vals = label_val_dict.get("", 0)
624+
if "" in label_val_dict.keys():
625+
del label_val_dict[""]
626+
if "Not Provided" in label_val_dict:
627+
label_val_dict["Not Provided"] += empty_vals
628+
else:
629+
label_val_dict["Not Provided"] = empty_vals
630+
631+
host_info_json["gender_label"] = list(label_val_dict.keys())
632+
host_info_json["gender_values"] = list(label_val_dict.values())
623633
# graphic for gender and age
624634
host_gender_data, invalid_gender_data = fetching_data_for_sex_and_range_data()
635+
empty_vals = host_gender_data.get("", 0)
636+
if "" in host_gender_data.keys():
637+
del host_gender_data[""]
638+
if "Not Provided" in host_gender_data:
639+
host_gender_data["Not Provided"] += empty_vals
640+
else:
641+
host_gender_data["Not Provided"] = empty_vals
625642
total_invalid_data["invalid_gender_data"] = invalid_gender_data
626643
host_info_json["gender_data"] = host_gender_data
627644
host_age_data, invalid_age_data = fetching_data_for_range_age()
@@ -655,7 +672,7 @@ def pre_proc_samples_per_date_all_lab(detailed=None):
655672
x["collection_sample_date"], str
656673
) # If data is in string format, convert it to date first
657674
else x["collection_sample_date"].strftime(
658-
"%d-%B-%Y"
675+
"%Y-W%V"
659676
) # Else just process date directly
660677
)
661678
for x in in_date_samples[
@@ -715,3 +732,77 @@ def pre_proc_samples_per_date_all_lab(detailed=None):
715732
}
716733
)
717734
return {"SUCCESS": "Success"}
735+
736+
737+
def pre_proc_samples_received_per_lab():
738+
"""Fetch the samples received per laboratory data from LIMS and save it"""
739+
raw_data = core.utils.rest_api.get_summarize_data("")
740+
if "ERROR" in raw_data:
741+
return raw_data
742+
743+
data = {"x": [], "y": []}
744+
for key, value in raw_data["laboratory"].items():
745+
data["x"].append(key)
746+
data["y"].append(value)
747+
dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
748+
{
749+
"graphic_name": "samples_received_per_lab",
750+
"graphic_data": data,
751+
}
752+
)
753+
return {"SUCCESS": "Success"}
754+
755+
756+
def pre_proc_samples_received_per_ccaa():
757+
"""Fetch the received samples per ccaa data from LIMS and save it"""
758+
raw_data = core.utils.rest_api.get_summarize_data("")
759+
if "ERROR" in raw_data:
760+
return raw_data
761+
762+
data = {"x": [], "y": []}
763+
for key, value in raw_data["region"].items():
764+
data["x"].append(key)
765+
data["y"].append(value)
766+
dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
767+
{
768+
"graphic_name": "samples_received_per_ccaa",
769+
"graphic_data": data,
770+
}
771+
)
772+
return {"SUCCESS": "Success"}
773+
774+
775+
def pre_proc_intranet_gisaid_data():
776+
"""Get the list of the accesion values for gisaid data to show in intranet"""
777+
gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
778+
"gisaid_accession_id", None
779+
)
780+
gisaid_data = defaultdict(list)
781+
for acc in gisaid_acc:
782+
lab_name = acc[0]
783+
gisaid_data[lab_name].append(acc[1:])
784+
dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
785+
{
786+
"graphic_name": "intranet_gisaid_data",
787+
"graphic_data": gisaid_data,
788+
}
789+
)
790+
return {"SUCCESS": "Success"}
791+
792+
793+
def pre_proc_intranet_ena_data():
794+
"""Get the list of the accesion values for ena data to show in intranet"""
795+
ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
796+
"ena_sample_accession", None
797+
)
798+
ena_data = defaultdict(list)
799+
for acc in ena_acc:
800+
lab_name = acc[0]
801+
ena_data[lab_name].append(acc[1:])
802+
dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
803+
{
804+
"graphic_name": "intranet_ena_data",
805+
"graphic_data": ena_data,
806+
}
807+
)
808+
return {"SUCCESS": "Success"}

‎dashboard/utils/var_lineage_variation_over_time_graph.py

+10-8
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,10 @@ def create_lineages_variations_graphic():
7474
dbc.Col(period_text, md=6),
7575
dbc.Col(
7676
dcc.Graph(
77-
id="lineageGraph", figure="", config={"displaylogo": False}
77+
id="lineageGraph",
78+
figure="",
79+
config={"displaylogo": False},
80+
style={"padding-top": "15px"},
7881
),
7982
md=12,
8083
),
@@ -115,7 +118,6 @@ def update_graph(start_date, end_date):
115118
# samples_per_week = samples_df.groupby(["samples", pd.Grouper(key="Collection date", freq="W-MON")]).sum().reset_index().sort_values("Collection date")
116119
# samples_df["Collection date"] = samples_df.index
117120
lineages = sub_data_df["Lineage"].unique().tolist()
118-
119121
# group samples in variants per weeks
120122
data_week_df = (
121123
sub_data_df.groupby(
@@ -141,7 +143,7 @@ def update_graph(start_date, end_date):
141143
graph_df[lineages] = graph_df[lineages].astype(int)
142144
# Do the percentage calculation
143145
value_per_df = (graph_df.div(graph_df.sum(axis=1), axis=0) * 100).round(2)
144-
# value_per_df = value_per_df
146+
145147
# Create figure with secondary y-axis
146148
fig = make_subplots(specs=[[{"secondary_y": True}]])
147149
if sub_data_df.empty:
@@ -164,20 +166,20 @@ def update_graph(start_date, end_date):
164166
legend_yanchor="top",
165167
legend_orientation="h", # show entries horizontally
166168
legend_x=0.5, # put legend in center of x-axis
167-
legend_y=-0.25,
169+
legend_y=-0.30,
168170
bargap=0, # gap between bars of adjacent location coordinates.
169171
bargroupgap=0, # gap between bars of the same location coordinate.
170172
margin_l=10,
171173
margin_r=10,
172174
margin_b=40,
173-
margin_t=30,
175+
margin_t=40,
174176
height=600,
175177
)
176178
return fig
177179
fig.add_trace(
178180
go.Scatter(
179181
x=value_per_df.index,
180-
y=samples_df["samples"],
182+
y=data_week_df["samples"],
181183
mode="lines",
182184
line_color="#0066cc",
183185
line_width=2,
@@ -219,13 +221,13 @@ def update_graph(start_date, end_date):
219221
legend_yanchor="top",
220222
legend_orientation="h", # show entries horizontally
221223
legend_x=0.5, # put legend in center of x-axis
222-
legend_y=-0.25,
224+
legend_y=-0.30,
223225
bargap=0, # gap between bars of adjacent location coordinates.
224226
bargroupgap=0, # gap between bars of the same location coordinate.
225227
margin_l=10,
226228
margin_r=10,
227229
margin_b=40,
228-
margin_t=30,
230+
margin_t=40,
229231
height=600,
230232
)
231233
return fig

‎dashboard/utils/var_needle_mutation_graph_by_lineage.py

+309-45
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.