Closing several issues pre-production deployment (#233)

* Fix typo defined date for collection date

* Removed upload metadata section from intranet until fixed

* Lab_name is shown when no samples are found

* split samples by submitting institution

* Included submitting_institution prop in Sample model

* Filtering bio_stats by submitting_institution instead

* New function get_collecting_insts_from_lab(lab_name)

* Getting ena and gisaid data from pre-processed state

* Adapting samples queries to submitting_institution

* Improving search sample query

* Now graphic data is log-scaled if needed

* Get samples per lab and ccaa from pre-processed data

* Fixing gender-age data from host-info plots

* Added samples per lab-ccaa, ena and gisaid data to graphic-json

* Adapted data to submitting institution and improved time for ena and gisaid

* Built needle plot from scratch instead of dash_bio. Added new functions

* Fixed wrong label being shown in num_samples

* Adding submitting_institution to api. Linting

* Included submitting_institution to Sample model. Linting

* Scale ydata when needed. Linting

* Load data for lab and ccaa from preprocessed. Linting

* Load ena and gisaid data from preprocessed. Linting

* Built needle plot from scratch instead of dash_bio. Linting

* Removed wrong pdb

Author: Shettland

core/api/utils/samples.py

@@ -60,8 +60,8 @@ def split_sample_data(data):
         .get_state_id()
     )
     split_data["sample"]["user"] = (
-        core.utils.samples.get_user_id_from_collecting_institution(
-            split_data["sample"]["collecting_institution"]
+        core.utils.samples.get_user_id_from_submitting_institution(
+            split_data["sample"]["submitting_institution"]
         )
     )
     if core.models.Sample.objects.all().exists():

core/api/views.py

@@ -108,7 +108,13 @@ def create_sample_data(request):
             return Response(error, status=status.HTTP_400_BAD_REQUEST)
         schema_id = schema_obj.get_schema_id()
         # check if sample id field and collecting_institution are in the request
-        if "sequencing_sample_id" not in data or "collecting_institution" not in data:
+        required_db_fields = [
+            "sequencing_sample_id",
+            "collecting_institution",
+            "submitting_institution",
+        ]
+        if any(field not in data for field in required_db_fields):
+            print(f"ERROR. Missing: {[f for f in required_db_fields if f not in data]}")
             return Response(status=status.HTTP_400_BAD_REQUEST)
         # check if sample is already defined
         if core.utils.samples.get_sample_obj_from_sample_name(

core/models.py

@@ -21,6 +21,7 @@ def __str__(self):
         return self.user.username
 
     def get_lab_name(self):
+        # Mapped to submitting_institution field in Sample model
         return "%s" % (self.laboratory)
 
     def get_lab_code(self):
@@ -703,6 +704,7 @@ class Sample(models.Model):
     sequencing_sample_id = models.CharField(max_length=80, null=True, blank=True)
     submitting_lab_sample_id = models.CharField(max_length=80, null=True, blank=True)
     collecting_institution = models.CharField(max_length=120, null=True, blank=True)
+    submitting_institution = models.CharField(max_length=120, null=True, blank=True)
     sequence_file_R1_fastq = models.CharField(max_length=80, null=True, blank=True)
     sequence_file_R2_fastq = models.CharField(max_length=80, null=True, blank=True)
     sequence_file_R1_md5 = models.CharField(max_length=80, null=True, blank=True)
@@ -737,6 +739,9 @@ def get_collecting_lab_sample_id(self):
     def get_collecting_institution(self):
         return "%s" % (self.collecting_institution)
 
+    def get_submitting_institution(self):
+        return "%s" % (self.submitting_institution)
+
     def get_unique_id(self):
         return "%s" % (self.sample_unique_id)
 

core/templates/core/intranet.html

@@ -291,7 +291,7 @@ <h5>Percentage of GISAID upload samples</h5>
                                                         {% else %}
                                                             <div class="col-md-7">
                                                                 <div class="card border-warning mb-3">
-                                                                    <div class="card-header text-center text-warning"><h2 style="text-align:center">No samples are upload yet to Relecov Platform</h2> </div>
+                                                                    <div class="card-header text-center text-warning"><h2 style="text-align:center">No samples found for your laboratory: {{ intra_data.lab|default:"Unknown" }}</h2> </div>
                                                                 </div> <!-- end card  -->
                                                             </div> <!-- end col-md-4 -->
                                                         {% endif %}

core/templates/core/intranetSideBar.html

@@ -17,12 +17,6 @@
             <span>Overview</span></a>
     </li>
     <br>
-    <li class="nav-item active">
-        <a class="nav-link" href="/metadataForm">
-            <i class="bi bi-upload"></i>
-            <span>Upload Metadata</span></a>
-    </li>
-    <br>
     <li class="nav-item active">
         <a class="nav-link" href="/laboratoryContact">
             <i class="bi bi-person-lines-fill"></i>

core/templates/core/searchSample.html

@@ -125,7 +125,7 @@ <h2 class="card-title">Search sample for {{search_data.labs}}</h2>
                                                     </div>  <!-- end text input -->
                                                 </div>
                                                 <div class="row mb-3">
-                                                    <label class="col-sm-4 control-label" for="sDate" >Sample defined Date</label>
+                                                    <label class="col-sm-4 control-label" for="sDate" >Sample collection date</label>
                                                     <div class="col-sm-7">
                                                         <input class="form-control" type="date" name="sDate" id="sDate">
                                                     </div>

core/utils/bioinfo_analysis.py

@@ -18,7 +18,7 @@ def get_bio_analysis_stats_from_lab(lab_name=None):
         bio_stats["received"] = core.models.Sample.objects.count()
     else:
         lab_samples = core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name
+            submitting_institution__iexact=lab_name
         )
         bio_stats["analized"] = (
             lab_samples.select_related("state_id")

core/utils/labs.py

@@ -36,6 +36,17 @@ def get_lab_name_from_user(user_obj):
         return ""
 
 
+def get_collecting_insts_from_lab(lab_name):
+    """Get a list of collecting institutions associated to the given
+    laboratory (submitting_institution)
+    """
+    return (
+        core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
+        .values_list("collecting_institution", flat=True)
+        .distinct()
+    )
+
+
 def update_contact_lab(old_data, new_data):
     """Update the contact information. If any field is empty it will set the
     old value. In case that all new_data are empty returns than no changes

core/utils/plotly_graphics.py

@@ -25,6 +25,7 @@ def bar_graphic(data, col_names, legend, yaxis, options):
                 marker_color=colors if "colors" in options else colors[idx - 1],
             )
         )
+        fig = log_ydata_if_needed(fig, data[col_names[idx]], ratio=100)
 
     # Customize aspect
     fig.update_traces(
@@ -45,7 +46,6 @@ def bar_graphic(data, col_names, legend, yaxis, options):
     )
     if "xaxis_tics" in options:
         fig.update_layout(xaxis=options["xaxis"])
-
     plot_div = plot(fig, output_type="div", config={"displaylogo": False})
 
     return plot_div
@@ -68,6 +68,7 @@ def line_graphic(x_data, y_data, options):
         title_font_color="green",
         title_font_size=20,
     )
+    log_ydata_if_needed(fig, y_data, ratio=100)
     if "xaxis" in options:
         fig.update_layout(xaxis=options["xaxis"])
     plot_div = plot(fig, output_type="div", config={"displaylogo": False})
@@ -92,6 +93,8 @@ def histogram_graphic(data, col_names, options):
         xaxis_tickangle=-45,
         margin=dict(l=20, r=40, t=30, b=20),
     )
+    ydata = data[col_names[1]]
+    graph = log_ydata_if_needed(graph, ydata, ratio=100)
 
     plot_div = plot(graph, output_type="div", config={"displaylogo": False})
     return plot_div
@@ -217,3 +220,30 @@ def needle_plot(m_data):
     )
     def update_needleplot(show_rangeslider):
         return True if show_rangeslider else False
+
+
+def log_ydata_if_needed(graph, ydata, ratio=100):
+    """Try to apply logaritmic scale to ydata if necessary
+
+    Args:
+        graph (plotly.figure): Plotly figure ready to be renderized
+        ydata (list): list of values or pandas.series (e.g. df[col])
+        ratio (int): ratio threshold to apply log scale or not
+
+    Return:
+        graph: Graph with ydata scaled with logarithmic scale
+    """
+    min_score = min(ydata)
+    max_score = max(ydata)
+    if min_score == 0:
+        drop_0s = [x for x in ydata if x != 0]
+        if not drop_0s:  # All data is 0 so do not scale
+            return graph
+        min_score = min(drop_0s)
+    minmax_ratio = max_score / min_score
+    if minmax_ratio >= ratio:
+        try:
+            graph.update_layout(yaxis_type="log", yaxis_dtick=1)
+        except TypeError as e:  # Input graph does not accept log scaling
+            print(f"ERROR while trying to scale ydata: {e}")
+    return graph

core/utils/public_db.py

@@ -1,6 +1,7 @@
 # Local imports
 import core.models
 import core.utils.plotly_graphics
+import dashboard.utils.generic_graphic_data
 from django.db.models import Q
 
 
@@ -11,11 +12,11 @@ def get_public_accession_from_sample_lab(p_field, sample_objs=None):
     if sample_objs is None:
         return (
             core.models.PublicDatabaseValues.objects.filter(
-                public_database_fieldID__property_name__exact=p_field,
+                public_database_fieldID__property_name__iexact=p_field,
             )
             .exclude(Q(value__icontains="Not Provided") | Q(value__isnull=True))
             .values_list(
-                "sampleID__collecting_institution",
+                "sampleID__submitting_institution",
                 "sampleID__sequencing_sample_id",
                 "value",
             )
@@ -51,3 +52,35 @@ def get_public_information_from_sample(p_type, sample_id):
             public_database_fieldID__database_type__public_type_name__iexact=p_type,
         ).values_list("public_database_fieldID__label_name", "value")
     return []
+
+
+def get_preprocessed_gisaid_data():
+    """Load preprocessed gisaid data in graphic_json table"""
+    gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "intranet_gisaid_data"
+    )
+    if gisaid_data is None:
+        # Execute the pre-processed task to get the data
+        result = dashboard.utils.generic_process_data.pre_proc_intranet_gisaid_data()
+        if "ERROR" in result:
+            return result
+        gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "intranet_gisaid_data"
+        )
+    return gisaid_data
+
+
+def get_preprocessed_ena_data():
+    """Load preprocessed ena data in graphic_json table"""
+    ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "intranet_ena_data"
+    )
+    if ena_data is None:
+        # Execute the pre-processed task to get the data
+        result = dashboard.utils.generic_process_data.pre_proc_intranet_ena_data()
+        if "ERROR" in result:
+            return result
+        ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "intranet_ena_data"
+        )
+    return ena_data

core/utils/samples.py

@@ -2,7 +2,7 @@
 import json
 import os
 import shutil
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from datetime import datetime
 import pandas as pd
 from django.contrib.auth.models import Group, User
@@ -78,10 +78,10 @@ def assign_samples_to_new_user(data):
     """Assign all samples from a laboratory to a new userID"""
     user_obj = User.objects.filter(pk__exact=data["userName"])
     if core.models.core.models.Sample.objects.filter(
-        collecting_institution__iexact=data["lab"]
+        submitting_institution__iexact=data["lab"]
     ).exists():
         core.models.core.models.Sample.objects.filter(
-            collecting_institution__iexact=data["lab"]
+            submitting_institution__iexact=data["lab"]
         ).update(user=user_obj[0])
         return {"Success": "Success"}
     return {
@@ -304,8 +304,11 @@ def create_dash_bar_for_each_lab():
 
 def perc_gauge_graphic(values):
     data = {}
-    x = values["analized"] / values["received"] * 100
-    data["value"] = float("{:.2f}".format(x))
+    if values["received"] == 0:
+        data["value"] = 0
+    else:
+        x = values["analized"] / values["received"] * 100
+        data["value"] = float("{:.2f}".format(x))
     gauge_graph = core.utils.plotly_graphics.gauge_graphic(data)
     return gauge_graph
 
@@ -327,11 +330,11 @@ def get_lab_last_actions(lab_name=None):
     if lab_name is None:
         lab_actions = []
         labs = core.models.core.models.Sample.objects.values_list(
-            "collecting_institution"
+            "submitting_institution"
         ).distinct()
         for lab in labs:
             sam_obj = core.models.core.models.Sample.objects.filter(
-                collecting_institution__exact=lab[0]
+                submitting_institution__exact=lab[0]
             ).last()
             lab_data = [lab[0]]
             for action in action_list:
@@ -352,7 +355,7 @@ def get_lab_last_actions(lab_name=None):
     else:
         actions = {}
         last_sample_obj = core.models.core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name
+            submitting_institution__iexact=lab_name
         ).last()
         action_objs = core.models.DateUpdateState.objects.filter(
             sampleID=last_sample_obj
@@ -403,7 +406,7 @@ def get_sample_display_data(sample_id, user):
     # Allow to see information obut sample to relecovManager
     group = Group.objects.get(name="RelecovManager")
     if group not in user.groups.all():
-        lab_name = sample_obj.get_collecting_institution()
+        lab_name = sample_obj.get_submitting_institution()
         if not core.models.Profile.objects.filter(
             user=user, laboratory__iexact=lab_name
         ).exists():
@@ -530,22 +533,22 @@ def get_sample_per_date_per_lab(lab_name):
     samples_per_date = OrderedDict()
 
     s_dates = (
-        core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
-        .values_list("sequencing_date", flat=True)
+        core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
+        .values_list("collecting_date", flat=True)
         .distinct()
-        .order_by("sequencing_date")
+        .order_by("collecting_date")
     )
     for s_date in s_dates:
-        date = datetime.strftime(s_date, "%d-%B-%Y")
+        date = datetime.strftime(s_date, "%Y-W%V")
         samples_per_date[date] = core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name, sequencing_date=s_date
+            submitting_institution__iexact=lab_name, collecting_date=s_date
         ).count()
     return samples_per_date
 
 
 def get_sample_objs_per_lab(lab_name):
     """Get all sample instance for the lab who the user is responsible"""
-    return core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
+    return core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
 
 
 def get_search_data(user_obj):
@@ -570,9 +573,9 @@ def get_search_data(user_obj):
     return s_data
 
 
-def get_user_id_from_collecting_institution(lab):
-    """Use the laboratory name defined in the Profile to find out the user.
-    if no user is not defined with this lab it retruns None
+def get_user_id_from_submitting_institution(lab):
+    """Use the laboratory name defined in the Profile (submitting-institution)
+    to find out the user. if no user is not defined with this lab it returns None
     """
     if core.models.Profile.objects.filter(laboratory__iexact=lab).exists():
         return core.models.Profile.objects.filter(laboratory__iexact=lab).last().user.pk
@@ -618,9 +621,9 @@ def join_sample_and_batch(b_data, user_obj, schema_obj):
 def get_all_lab_list():
     """Function gets the lab names and return then in an ordered list"""
     return list(
-        core.models.Sample.objects.values_list("collecting_institution", flat=True)
+        core.models.Sample.objects.values_list("submitting_institution", flat=True)
         .distinct()
-        .order_by("collecting_institution")
+        .order_by("submitting_institution")
     )
 
 
@@ -717,9 +720,23 @@ def save_excel_form_in_samba_folder(m_file, user_name):
 def search_samples(sample_name, lab_name, sample_state, s_date, user):
     """Search the samples that match with the query conditions"""
     sample_list = []
-    sample_objs = core.models.Sample.objects.all()
+    if s_date != "":
+        samples_with_coldate = core.utils.rest_api.fetch_samples_on_condition(
+            "collection_sample_date"
+        )
+        dates_samples_dict = defaultdict(list)
+        if "ERROR" in samples_with_coldate:
+            return samples_with_coldate
+        for d in samples_with_coldate["DATA"]:
+            dates_samples_dict[d["collection_sample_date"]].append(d["Sample Name"])
+        sample_objs = core.models.Sample.objects.filter(
+            Q(sequencing_sample_id__in=dates_samples_dict.get(s_date, []))
+            | Q(collecting_lab_sample_id__iexact=dates_samples_dict.get(s_date, []))
+        )
+    else:
+        sample_objs = core.models.Sample.objects.all()
     if lab_name != "":
-        sample_objs = sample_objs.filter(collecting_institution__iexact=lab_name)
+        sample_objs = sample_objs.filter(submitting_institution__iexact=lab_name)
     if sample_name != "":
         if sample_objs.filter(
             Q(sequencing_sample_id__iexact=sample_name)
@@ -753,8 +770,6 @@ def search_samples(sample_name, lab_name, sample_state, s_date, user):
             ).values_list("sampleID__pk", flat=True)
         )
         sample_objs = sample_objs.filter(pk__in=state_ids)
-    if s_date != "":
-        sample_objs = sample_objs.filter(sequencing_date__exact=s_date)
     if len(sample_objs) == 1:
         sample_list.append(sample_objs[0].get_sample_id())
         return sample_list