Closing several issues pre-production deployment (#233)

* Fix typo defined date for collection date

* Removed upload metadata section from intranet until fixed

* Lab_name is shown when no samples are found

* split samples by submitting institution

* Included submitting_institution prop in Sample model

* Filtering bio_stats by submitting_institution instead

* New function get_collecting_insts_from_lab(lab_name)

* Getting ena and gisaid data from pre-processed state

* Adapting samples queries to submitting_institution

* Improving search sample query

* Now graphic data is log-scaled if needed

* Get samples per lab and ccaa from pre-processed data

* Fixing gender-age data from host-info plots

* Added samples per lab-ccaa, ena and gisaid data to graphic-json

* Adapted data to submitting institution and improved time for ena and gisaid

* Built needle plot from scratch instead of dash_bio. Added new functions

* Fixed wrong label being shown in num_samples

* Adding submitting_institution to api. Linting

* Included submitting_institution to Sample model. Linting

* Scale ydata when needed. Linting

* Load data for lab and ccaa from preprocessed. Linting

* Load ena and gisaid data from preprocessed. Linting

* Built needle plot from scratch instead of dash_bio. Linting

* Removed wrong pdb

Author: Shettland

core/api/utils/samples.py

@@ -60,8 +60,8 @@ def split_sample_data(data):
         .get_state_id()
     )
     split_data["sample"]["user"] = (
-        core.utils.samples.get_user_id_from_collecting_institution(
-            split_data["sample"]["collecting_institution"]
+        core.utils.samples.get_user_id_from_submitting_institution(
+            split_data["sample"]["submitting_institution"]
         )
     )
     if core.models.Sample.objects.all().exists():

core/api/views.py

@@ -108,7 +108,13 @@ def create_sample_data(request):
             return Response(error, status=status.HTTP_400_BAD_REQUEST)
         schema_id = schema_obj.get_schema_id()
         # check if sample id field and collecting_institution are in the request
-        if "sequencing_sample_id" not in data or "collecting_institution" not in data:
+        required_db_fields = [
+            "sequencing_sample_id",
+            "collecting_institution",
+            "submitting_institution",
+        ]
+        if any(field not in data for field in required_db_fields):
+            print(f"ERROR. Missing: {[f for f in required_db_fields if f not in data]}")
             return Response(status=status.HTTP_400_BAD_REQUEST)
         # check if sample is already defined
         if core.utils.samples.get_sample_obj_from_sample_name(

core/models.py

@@ -21,6 +21,7 @@ def __str__(self):
         return self.user.username
 
     def get_lab_name(self):
+        # Mapped to submitting_institution field in Sample model
         return "%s" % (self.laboratory)
 
     def get_lab_code(self):
@@ -703,6 +704,7 @@ class Sample(models.Model):
     sequencing_sample_id = models.CharField(max_length=80, null=True, blank=True)
     submitting_lab_sample_id = models.CharField(max_length=80, null=True, blank=True)
     collecting_institution = models.CharField(max_length=120, null=True, blank=True)
+    submitting_institution = models.CharField(max_length=120, null=True, blank=True)
     sequence_file_R1_fastq = models.CharField(max_length=80, null=True, blank=True)
     sequence_file_R2_fastq = models.CharField(max_length=80, null=True, blank=True)
     sequence_file_R1_md5 = models.CharField(max_length=80, null=True, blank=True)
@@ -737,6 +739,9 @@ def get_collecting_lab_sample_id(self):
     def get_collecting_institution(self):
         return "%s" % (self.collecting_institution)
 
+    def get_submitting_institution(self):
+        return "%s" % (self.submitting_institution)
+
     def get_unique_id(self):
         return "%s" % (self.sample_unique_id)
 

core/templates/core/intranet.html

@@ -291,7 +291,7 @@ <h5>Percentage of GISAID upload samples</h5>
                                                         {% else %}
                                                             <div class="col-md-7">
                                                                 <div class="card border-warning mb-3">
-                                                                    <div class="card-header text-center text-warning"><h2 style="text-align:center">No samples are upload yet to Relecov Platform</h2> </div>
+                                                                    <div class="card-header text-center text-warning"><h2 style="text-align:center">No samples found for your laboratory: {{ intra_data.lab|default:"Unknown" }}</h2> </div>
                                                                 </div> <!-- end card  -->
                                                             </div> <!-- end col-md-4 -->
                                                         {% endif %}

core/templates/core/intranetSideBar.html

@@ -17,12 +17,6 @@
             <span>Overview</span></a>
     </li>
     <br>
-    <li class="nav-item active">
-        <a class="nav-link" href="/metadataForm">
-            <i class="bi bi-upload"></i>
-            <span>Upload Metadata</span></a>
-    </li>
-    <br>
     <li class="nav-item active">
         <a class="nav-link" href="/laboratoryContact">
             <i class="bi bi-person-lines-fill"></i>

core/templates/core/searchSample.html

@@ -125,7 +125,7 @@ <h2 class="card-title">Search sample for {{search_data.labs}}</h2>
                                                     </div>  <!-- end text input -->
                                                 </div>
                                                 <div class="row mb-3">
-                                                    <label class="col-sm-4 control-label" for="sDate" >Sample defined Date</label>
+                                                    <label class="col-sm-4 control-label" for="sDate" >Sample collection date</label>
                                                     <div class="col-sm-7">
                                                         <input class="form-control" type="date" name="sDate" id="sDate">
                                                     </div>

core/utils/bioinfo_analysis.py

@@ -18,7 +18,7 @@ def get_bio_analysis_stats_from_lab(lab_name=None):
         bio_stats["received"] = core.models.Sample.objects.count()
     else:
         lab_samples = core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name
+            submitting_institution__iexact=lab_name
         )
         bio_stats["analized"] = (
             lab_samples.select_related("state_id")

core/utils/labs.py

@@ -36,6 +36,17 @@ def get_lab_name_from_user(user_obj):
         return ""
 
 
+def get_collecting_insts_from_lab(lab_name):
+    """Get a list of collecting institutions associated to the given
+    laboratory (submitting_institution)
+    """
+    return (
+        core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
+        .values_list("collecting_institution", flat=True)
+        .distinct()
+    )
+
+
 def update_contact_lab(old_data, new_data):
     """Update the contact information. If any field is empty it will set the
     old value. In case that all new_data are empty returns than no changes

core/utils/plotly_graphics.py

@@ -25,6 +25,7 @@ def bar_graphic(data, col_names, legend, yaxis, options):
                 marker_color=colors if "colors" in options else colors[idx - 1],
             )
         )
+        fig = log_ydata_if_needed(fig, data[col_names[idx]], ratio=100)
 
     # Customize aspect
     fig.update_traces(
@@ -45,7 +46,6 @@ def bar_graphic(data, col_names, legend, yaxis, options):
     )
     if "xaxis_tics" in options:
         fig.update_layout(xaxis=options["xaxis"])
-
     plot_div = plot(fig, output_type="div", config={"displaylogo": False})
 
     return plot_div
@@ -68,6 +68,7 @@ def line_graphic(x_data, y_data, options):
         title_font_color="green",
         title_font_size=20,
     )
+    log_ydata_if_needed(fig, y_data, ratio=100)
     if "xaxis" in options:
         fig.update_layout(xaxis=options["xaxis"])
     plot_div = plot(fig, output_type="div", config={"displaylogo": False})
@@ -92,6 +93,8 @@ def histogram_graphic(data, col_names, options):
         xaxis_tickangle=-45,
         margin=dict(l=20, r=40, t=30, b=20),
     )
+    ydata = data[col_names[1]]
+    graph = log_ydata_if_needed(graph, ydata, ratio=100)
 
     plot_div = plot(graph, output_type="div", config={"displaylogo": False})
     return plot_div
@@ -217,3 +220,30 @@ def needle_plot(m_data):
     )
     def update_needleplot(show_rangeslider):
         return True if show_rangeslider else False
+
+
+def log_ydata_if_needed(graph, ydata, ratio=100):
+    """Try to apply logaritmic scale to ydata if necessary
+
+    Args:
+        graph (plotly.figure): Plotly figure ready to be renderized
+        ydata (list): list of values or pandas.series (e.g. df[col])
+        ratio (int): ratio threshold to apply log scale or not
+
+    Return:
+        graph: Graph with ydata scaled with logarithmic scale
+    """
+    min_score = min(ydata)
+    max_score = max(ydata)
+    if min_score == 0:
+        drop_0s = [x for x in ydata if x != 0]
+        if not drop_0s:  # All data is 0 so do not scale
+            return graph
+        min_score = min(drop_0s)
+    minmax_ratio = max_score / min_score
+    if minmax_ratio >= ratio:
+        try:
+            graph.update_layout(yaxis_type="log", yaxis_dtick=1)
+        except TypeError as e:  # Input graph does not accept log scaling
+            print(f"ERROR while trying to scale ydata: {e}")
+    return graph

core/utils/public_db.py

@@ -1,6 +1,7 @@
 # Local imports
 import core.models
 import core.utils.plotly_graphics
+import dashboard.utils.generic_graphic_data
 from django.db.models import Q
 
 
@@ -11,11 +12,11 @@ def get_public_accession_from_sample_lab(p_field, sample_objs=None):
     if sample_objs is None:
         return (
             core.models.PublicDatabaseValues.objects.filter(
-                public_database_fieldID__property_name__exact=p_field,
+                public_database_fieldID__property_name__iexact=p_field,
             )
             .exclude(Q(value__icontains="Not Provided") | Q(value__isnull=True))
             .values_list(
-                "sampleID__collecting_institution",
+                "sampleID__submitting_institution",
                 "sampleID__sequencing_sample_id",
                 "value",
             )
@@ -51,3 +52,35 @@ def get_public_information_from_sample(p_type, sample_id):
             public_database_fieldID__database_type__public_type_name__iexact=p_type,
         ).values_list("public_database_fieldID__label_name", "value")
     return []
+
+
+def get_preprocessed_gisaid_data():
+    """Load preprocessed gisaid data in graphic_json table"""
+    gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "intranet_gisaid_data"
+    )
+    if gisaid_data is None:
+        # Execute the pre-processed task to get the data
+        result = dashboard.utils.generic_process_data.pre_proc_intranet_gisaid_data()
+        if "ERROR" in result:
+            return result
+        gisaid_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "intranet_gisaid_data"
+        )
+    return gisaid_data
+
+
+def get_preprocessed_ena_data():
+    """Load preprocessed ena data in graphic_json table"""
+    ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "intranet_ena_data"
+    )
+    if ena_data is None:
+        # Execute the pre-processed task to get the data
+        result = dashboard.utils.generic_process_data.pre_proc_intranet_ena_data()
+        if "ERROR" in result:
+            return result
+        ena_data = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "intranet_ena_data"
+        )
+    return ena_data

core/utils/samples.py

@@ -2,7 +2,7 @@
 import json
 import os
 import shutil
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from datetime import datetime
 import pandas as pd
 from django.contrib.auth.models import Group, User
@@ -78,10 +78,10 @@ def assign_samples_to_new_user(data):
     """Assign all samples from a laboratory to a new userID"""
     user_obj = User.objects.filter(pk__exact=data["userName"])
     if core.models.core.models.Sample.objects.filter(
-        collecting_institution__iexact=data["lab"]
+        submitting_institution__iexact=data["lab"]
     ).exists():
         core.models.core.models.Sample.objects.filter(
-            collecting_institution__iexact=data["lab"]
+            submitting_institution__iexact=data["lab"]
         ).update(user=user_obj[0])
         return {"Success": "Success"}
     return {
@@ -304,8 +304,11 @@ def create_dash_bar_for_each_lab():
 
 def perc_gauge_graphic(values):
     data = {}
-    x = values["analized"] / values["received"] * 100
-    data["value"] = float("{:.2f}".format(x))
+    if values["received"] == 0:
+        data["value"] = 0
+    else:
+        x = values["analized"] / values["received"] * 100
+        data["value"] = float("{:.2f}".format(x))
     gauge_graph = core.utils.plotly_graphics.gauge_graphic(data)
     return gauge_graph
 
@@ -327,11 +330,11 @@ def get_lab_last_actions(lab_name=None):
     if lab_name is None:
         lab_actions = []
         labs = core.models.core.models.Sample.objects.values_list(
-            "collecting_institution"
+            "submitting_institution"
         ).distinct()
         for lab in labs:
             sam_obj = core.models.core.models.Sample.objects.filter(
-                collecting_institution__exact=lab[0]
+                submitting_institution__exact=lab[0]
             ).last()
             lab_data = [lab[0]]
             for action in action_list:
@@ -352,7 +355,7 @@ def get_lab_last_actions(lab_name=None):
     else:
         actions = {}
         last_sample_obj = core.models.core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name
+            submitting_institution__iexact=lab_name
         ).last()
         action_objs = core.models.DateUpdateState.objects.filter(
             sampleID=last_sample_obj
@@ -403,7 +406,7 @@ def get_sample_display_data(sample_id, user):
     # Allow to see information obut sample to relecovManager
     group = Group.objects.get(name="RelecovManager")
     if group not in user.groups.all():
-        lab_name = sample_obj.get_collecting_institution()
+        lab_name = sample_obj.get_submitting_institution()
         if not core.models.Profile.objects.filter(
             user=user, laboratory__iexact=lab_name
         ).exists():
@@ -530,22 +533,22 @@ def get_sample_per_date_per_lab(lab_name):
     samples_per_date = OrderedDict()
 
     s_dates = (
-        core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
-        .values_list("sequencing_date", flat=True)
+        core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
+        .values_list("collecting_date", flat=True)
         .distinct()
-        .order_by("sequencing_date")
+        .order_by("collecting_date")
     )
     for s_date in s_dates:
-        date = datetime.strftime(s_date, "%d-%B-%Y")
+        date = datetime.strftime(s_date, "%Y-W%V")
         samples_per_date[date] = core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name, sequencing_date=s_date
+            submitting_institution__iexact=lab_name, collecting_date=s_date
         ).count()
     return samples_per_date
 
 
 def get_sample_objs_per_lab(lab_name):
     """Get all sample instance for the lab who the user is responsible"""
-    return core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
+    return core.models.Sample.objects.filter(submitting_institution__iexact=lab_name)
 
 
 def get_search_data(user_obj):
@@ -570,9 +573,9 @@ def get_search_data(user_obj):
     return s_data
 
 
-def get_user_id_from_collecting_institution(lab):
-    """Use the laboratory name defined in the Profile to find out the user.
-    if no user is not defined with this lab it retruns None
+def get_user_id_from_submitting_institution(lab):
+    """Use the laboratory name defined in the Profile (submitting-institution)
+    to find out the user. if no user is not defined with this lab it returns None
     """
     if core.models.Profile.objects.filter(laboratory__iexact=lab).exists():
         return core.models.Profile.objects.filter(laboratory__iexact=lab).last().user.pk
@@ -618,9 +621,9 @@ def join_sample_and_batch(b_data, user_obj, schema_obj):
 def get_all_lab_list():
     """Function gets the lab names and return then in an ordered list"""
     return list(
-        core.models.Sample.objects.values_list("collecting_institution", flat=True)
+        core.models.Sample.objects.values_list("submitting_institution", flat=True)
         .distinct()
-        .order_by("collecting_institution")
+        .order_by("submitting_institution")
     )
 
 
@@ -717,9 +720,23 @@ def save_excel_form_in_samba_folder(m_file, user_name):
 def search_samples(sample_name, lab_name, sample_state, s_date, user):
     """Search the samples that match with the query conditions"""
     sample_list = []
-    sample_objs = core.models.Sample.objects.all()
+    if s_date != "":
+        samples_with_coldate = core.utils.rest_api.fetch_samples_on_condition(
+            "collection_sample_date"
+        )
+        dates_samples_dict = defaultdict(list)
+        if "ERROR" in samples_with_coldate:
+            return samples_with_coldate
+        for d in samples_with_coldate["DATA"]:
+            dates_samples_dict[d["collection_sample_date"]].append(d["Sample Name"])
+        sample_objs = core.models.Sample.objects.filter(
+            Q(sequencing_sample_id__in=dates_samples_dict.get(s_date, []))
+            | Q(collecting_lab_sample_id__iexact=dates_samples_dict.get(s_date, []))
+        )
+    else:
+        sample_objs = core.models.Sample.objects.all()
     if lab_name != "":
-        sample_objs = sample_objs.filter(collecting_institution__iexact=lab_name)
+        sample_objs = sample_objs.filter(submitting_institution__iexact=lab_name)
     if sample_name != "":
         if sample_objs.filter(
             Q(sequencing_sample_id__iexact=sample_name)
@@ -753,8 +770,6 @@ def search_samples(sample_name, lab_name, sample_state, s_date, user):
             ).values_list("sampleID__pk", flat=True)
         )
         sample_objs = sample_objs.filter(pk__in=state_ids)
-    if s_date != "":
-        sample_objs = sample_objs.filter(sequencing_date__exact=s_date)
     if len(sample_objs) == 1:
         sample_list.append(sample_objs[0].get_sample_id())
         return sample_list

core/utils/samples_graphics.py

@@ -2,21 +2,26 @@
 import core.utils.plotly_graphics
 import core.utils.rest_api
 import core.utils.samples
+import dashboard.utils.generic_process_data
 
 
 def received_per_ccaa():
     """Fetch the data from LIMS and show them in a graphic bar"""
-    raw_data = core.utils.rest_api.get_summarize_data("")
-    if "ERROR" in raw_data:
-        return raw_data
-
-    data = {"x": [], "y": []}
-    for key, value in raw_data["region"].items():
-        data["x"].append(key)
-        data["y"].append(value)
-
+    samples_per_ccaa = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "samples_received_per_ccaa"
+    )
+    if samples_per_ccaa is None:
+        # Execute the pre-processed task to get the data
+        result = (
+            dashboard.utils.generic_process_data.pre_proc_samples_received_per_ccaa()
+        )
+        if "ERROR" in result:
+            return result
+        samples_per_ccaa = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "samples_received_per_ccaa"
+        )
     return core.utils.plotly_graphics.bar_graphic(
-        data=data,
+        data=samples_per_ccaa,
         col_names=["x", "y"],
         legend=[""],
         yaxis={"title": "Number of samples"},
@@ -26,17 +31,21 @@ def received_per_ccaa():
 
 def received_per_lab():
     """Fetch the data from LIMS and show them in a graphic bar"""
-    raw_data = core.utils.rest_api.get_summarize_data("")
-    if "ERROR" in raw_data:
-        return raw_data
-
-    data = {"x": [], "y": []}
-    for key, value in raw_data["laboratory"].items():
-        data["x"].append(key)
-        data["y"].append(value)
-
+    samples_per_lab = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+        "samples_received_per_lab"
+    )
+    if samples_per_lab is None:
+        # Execute the pre-processed task to get the data
+        result = (
+            dashboard.utils.generic_process_data.pre_proc_samples_received_per_lab()
+        )
+        if "ERROR" in result:
+            return result
+        samples_per_lab = dashboard.utils.generic_graphic_data.get_graphic_json_data(
+            "samples_received_per_lab"
+        )
     return core.utils.plotly_graphics.bar_graphic(
-        data=data,
+        data=samples_per_lab,
         col_names=["x", "y"],
         legend=[""],
         yaxis={"title": "Number of samples"},

core/views.py

@@ -20,6 +20,7 @@
 import core.utils.samples_map
 
 #  End of imports  received samples
+import time
 
 
 def index(request):
@@ -221,14 +222,28 @@ def metadata_visualization(request):
 def intranet(request):
     relecov_group = Group.objects.filter(name="RelecovManager").last()
     if relecov_group not in request.user.groups.all():
+        start = time.time()
         intra_data = {}
         lab_name = core.utils.labs.get_lab_name_from_user(request.user)
-        date_lab_samples = core.utils.samples.get_sample_per_date_per_lab(lab_name)
+        related_insts_list = core.utils.labs.get_collecting_insts_from_lab(lab_name)
+        all_sample_per_date_detailed = (
+            core.utils.samples.get_sample_per_date_per_all_lab(detailed=True)
+        )
+        date_lab_samples = [
+            x
+            for x in all_sample_per_date_detailed
+            if x["collecting_institution"] in related_insts_list
+        ]
+        intra_data["lab"] = lab_name
+        print(f"Took {start - time.time()} seconds for date_lab_samples")
         if len(date_lab_samples) > 0:
+            start = time.time()
             sample_lab_objs = core.utils.samples.get_sample_objs_per_lab(lab_name)
+            print(f"Took {start - time.time()} seconds for sample_lab_objs")
             analysis_percent = (
                 core.utils.bioinfo_analysis.get_bio_analysis_stats_from_lab(lab_name)
             )
+            print(f"Took {start - time.time()} seconds for analysis_percent")
             cust_data = {
                 "col_names": ["Collecting Date", "Number of samples"],
                 "options": {},
@@ -238,28 +253,30 @@ def intranet(request):
             intra_data["sample_bar_graph"] = core.utils.samples.create_date_sample_bar(
                 date_lab_samples, cust_data
             )
+            print(f"Took {start - time.time()} seconds for sample_bar_graph")
             intra_data["sample_gauge_graph"] = core.utils.samples.perc_gauge_graphic(
                 analysis_percent
             )
+            print(f"Took {start - time.time()} seconds for gauge_graph")
             intra_data["actions"] = core.utils.samples.get_lab_last_actions(lab_name)
-            gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
-                "gisaid_accession_id", sample_lab_objs
-            )
-            if len(gisaid_acc) > 0:
-                intra_data["gisaid_accession"] = gisaid_acc
-            intra_data["gisaid_graph"] = core.utils.public_db.percentage_graphic(
-                len(sample_lab_objs), len(gisaid_acc), ""
-            )
-            ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
-                "ena_sample_accession", sample_lab_objs
-            )
-            if len(ena_acc) > 0:
-                intra_data["ena_accession"] = ena_acc
+            gisaid_acc = core.utils.public_db.get_preprocessed_gisaid_data()
+            # NOTE: Filters by lab_name instead of sample_lab_objs to improve performance, but could lead to mismatches
+            print(f"Took {start - time.time()} seconds for gisaid data")
+            if gisaid_acc:
+                gisaid_acc_filtered = gisaid_acc.get(lab_name, [])
+                intra_data["gisaid_accession"] = gisaid_acc_filtered
+                intra_data["gisaid_graph"] = core.utils.public_db.percentage_graphic(
+                    len(sample_lab_objs), len(gisaid_acc_filtered), ""
+                )
+            print(f"Took {start - time.time()} seconds for gisaig graph")
+            ena_acc = core.utils.public_db.get_preprocessed_ena_data()
+            if ena_acc:
+                ena_acc_filtered = ena_acc.get(lab_name, [])
+                intra_data["ena_accession"] = ena_acc_filtered
                 intra_data["ena_graph"] = core.utils.public_db.percentage_graphic(
-                    len(sample_lab_objs), len(ena_acc), ""
+                    len(sample_lab_objs), len(ena_acc_filtered), ""
                 )
-        else:
-            intra_data = f"No samples found for selected laboratory: {lab_name}"
+            print(f"Took {start - time.time()} seconds for ena graph")
         return render(request, "core/intranet.html", {"intra_data": intra_data})
     else:
         # loged user belongs to Relecov Manager group
@@ -290,25 +307,27 @@ def intranet(request):
             # Get the latest action from each lab
             manager_intra_data["actions"] = core.utils.samples.get_lab_last_actions()
             # Collect GISAID information
-            gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
-                "gisaid_accession_id", None
-            )
+            gisaid_acc = core.utils.public_db.get_preprocessed_gisaid_data()
+            full_gisaid_data = [
+                (lab, *tup) for lab, tuples in gisaid_acc.items() for tup in tuples
+            ]
             if len(gisaid_acc) > 0:
-                manager_intra_data["gisaid_accession"] = gisaid_acc
+                manager_intra_data["gisaid_accession"] = full_gisaid_data
                 manager_intra_data["gisaid_graph"] = (
                     core.utils.public_db.percentage_graphic(
-                        num_of_samples["Defined"], len(gisaid_acc), ""
+                        num_of_samples["Defined"], len(full_gisaid_data), ""
                     )
                 )
             # Collect Ena information
-            ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
-                "ena_sample_accession", None
-            )
+            ena_acc = core.utils.public_db.get_preprocessed_ena_data()
+            full_ena_data = [
+                (lab, *tup) for lab, tuples in ena_acc.items() for tup in tuples
+            ]
             if len(ena_acc) > 0:
-                manager_intra_data["ena_accession"] = ena_acc
+                manager_intra_data["ena_accession"] = full_ena_data
                 manager_intra_data["ena_graph"] = (
                     core.utils.public_db.percentage_graphic(
-                        num_of_samples["Defined"], len(ena_acc), ""
+                        num_of_samples["Defined"], len(full_ena_data), ""
                     )
                 )
         return render(

dashboard/cron.py

@@ -77,16 +77,24 @@ def update_graphic_json_data():
     dashboard.utils.generic_process_data.pre_proc_samples_received_map()
     print("Running pre_proc_host_info()")
     dashboard.utils.generic_process_data.pre_proc_host_info()
-    uniq_chrom_id_list = [
-        x["chromosomeID"]
-        for x in core.models.Gene.objects.values("chromosomeID").distinct()
-    ]
     print("Running pre_proc_samples_per_date_all_lab()")
     dashboard.utils.generic_process_data.pre_proc_samples_per_date_all_lab()
     print("Running pre_proc_samples_per_date_all_lab(detailed=True)")
     dashboard.utils.generic_process_data.pre_proc_samples_per_date_all_lab(
         detailed=True
     )
+    print("Running pre_proc_samples_received_per_lab()")
+    dashboard.utils.generic_process_data.pre_proc_samples_received_per_lab()
+    print("Running pre_proc_samples_received_per_ccaa()")
+    dashboard.utils.generic_process_data.pre_proc_samples_received_per_ccaa()
+    print("Running pre_proc_intranet_gisaid_data")
+    dashboard.utils.generic_process_data.pre_proc_intranet_gisaid_data()
+    print("Running pre_proc_intranet_ena_data()")
+    dashboard.utils.generic_process_data.pre_proc_intranet_ena_data()
+    uniq_chrom_id_list = [
+        x["chromosomeID"]
+        for x in core.models.Gene.objects.values("chromosomeID").distinct()
+    ]
     print(f"List of extracted unique chromosomes: {uniq_chrom_id_list}")
     print("Running pre_proc_variations_per_lineage() for each chromosome")
     for chromosome in uniq_chrom_id_list:

dashboard/utils/generic_process_data.py

@@ -12,6 +12,7 @@
 import core.utils.variants
 import core.utils.rest_api
 import core.utils.generic_functions
+import core.utils.public_db
 import dashboard.models
 from relecov_platform import settings as relecov_platform_settings
 
@@ -618,10 +619,26 @@ def fetching_data_for_gender():
         host_info_json["gender_label"] = {"ERROR": gender_label}
         host_info_json["gender_values"] = {"ERROR": gender_values}
     else:
-        host_info_json["gender_label"] = gender_label
-        host_info_json["gender_values"] = gender_values
+        label_val_dict = dict(zip(gender_label, gender_values))
+        empty_vals = label_val_dict.get("", 0)
+        if "" in label_val_dict.keys():
+            del label_val_dict[""]
+        if "Not Provided" in label_val_dict:
+            label_val_dict["Not Provided"] += empty_vals
+        else:
+            label_val_dict["Not Provided"] = empty_vals
+
+        host_info_json["gender_label"] = list(label_val_dict.keys())
+        host_info_json["gender_values"] = list(label_val_dict.values())
     # graphic for gender and age
     host_gender_data, invalid_gender_data = fetching_data_for_sex_and_range_data()
+    empty_vals = host_gender_data.get("", 0)
+    if "" in host_gender_data.keys():
+        del host_gender_data[""]
+    if "Not Provided" in host_gender_data:
+        host_gender_data["Not Provided"] += empty_vals
+    else:
+        host_gender_data["Not Provided"] = empty_vals
     total_invalid_data["invalid_gender_data"] = invalid_gender_data
     host_info_json["gender_data"] = host_gender_data
     host_age_data, invalid_age_data = fetching_data_for_range_age()
@@ -655,7 +672,7 @@ def pre_proc_samples_per_date_all_lab(detailed=None):
                     x["collection_sample_date"], str
                 )  # If data is in string format, convert it to date first
                 else x["collection_sample_date"].strftime(
-                    "%d-%B-%Y"
+                    "%Y-W%V"
                 )  # Else just process date directly
             )
             for x in in_date_samples[
@@ -715,3 +732,77 @@ def pre_proc_samples_per_date_all_lab(detailed=None):
             }
         )
         return {"SUCCESS": "Success"}
+
+
+def pre_proc_samples_received_per_lab():
+    """Fetch the samples received per laboratory data from LIMS and save it"""
+    raw_data = core.utils.rest_api.get_summarize_data("")
+    if "ERROR" in raw_data:
+        return raw_data
+
+    data = {"x": [], "y": []}
+    for key, value in raw_data["laboratory"].items():
+        data["x"].append(key)
+        data["y"].append(value)
+    dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
+        {
+            "graphic_name": "samples_received_per_lab",
+            "graphic_data": data,
+        }
+    )
+    return {"SUCCESS": "Success"}
+
+
+def pre_proc_samples_received_per_ccaa():
+    """Fetch the received samples per ccaa data from LIMS and save it"""
+    raw_data = core.utils.rest_api.get_summarize_data("")
+    if "ERROR" in raw_data:
+        return raw_data
+
+    data = {"x": [], "y": []}
+    for key, value in raw_data["region"].items():
+        data["x"].append(key)
+        data["y"].append(value)
+    dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
+        {
+            "graphic_name": "samples_received_per_ccaa",
+            "graphic_data": data,
+        }
+    )
+    return {"SUCCESS": "Success"}
+
+
+def pre_proc_intranet_gisaid_data():
+    """Get the list of the accesion values for gisaid data to show in intranet"""
+    gisaid_acc = core.utils.public_db.get_public_accession_from_sample_lab(
+        "gisaid_accession_id", None
+    )
+    gisaid_data = defaultdict(list)
+    for acc in gisaid_acc:
+        lab_name = acc[0]
+        gisaid_data[lab_name].append(acc[1:])
+    dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
+        {
+            "graphic_name": "intranet_gisaid_data",
+            "graphic_data": gisaid_data,
+        }
+    )
+    return {"SUCCESS": "Success"}
+
+
+def pre_proc_intranet_ena_data():
+    """Get the list of the accesion values for ena data to show in intranet"""
+    ena_acc = core.utils.public_db.get_public_accession_from_sample_lab(
+        "ena_sample_accession", None
+    )
+    ena_data = defaultdict(list)
+    for acc in ena_acc:
+        lab_name = acc[0]
+        ena_data[lab_name].append(acc[1:])
+    dashboard.models.GraphicJsonFile.objects.create_new_graphic_json(
+        {
+            "graphic_name": "intranet_ena_data",
+            "graphic_data": ena_data,
+        }
+    )
+    return {"SUCCESS": "Success"}

dashboard/utils/var_lineage_variation_over_time_graph.py

@@ -74,7 +74,10 @@ def create_lineages_variations_graphic():
                     dbc.Col(period_text, md=6),
                     dbc.Col(
                         dcc.Graph(
-                            id="lineageGraph", figure="", config={"displaylogo": False}
+                            id="lineageGraph",
+                            figure="",
+                            config={"displaylogo": False},
+                            style={"padding-top": "15px"},
                         ),
                         md=12,
                     ),
@@ -115,7 +118,6 @@ def update_graph(start_date, end_date):
         # samples_per_week = samples_df.groupby(["samples", pd.Grouper(key="Collection date", freq="W-MON")]).sum().reset_index().sort_values("Collection date")
         # samples_df["Collection date"] = samples_df.index
         lineages = sub_data_df["Lineage"].unique().tolist()
-
         # group samples in variants per weeks
         data_week_df = (
             sub_data_df.groupby(
@@ -141,7 +143,7 @@ def update_graph(start_date, end_date):
         graph_df[lineages] = graph_df[lineages].astype(int)
         # Do the percentage calculation
         value_per_df = (graph_df.div(graph_df.sum(axis=1), axis=0) * 100).round(2)
-        # value_per_df = value_per_df
+
         # Create figure with secondary y-axis
         fig = make_subplots(specs=[[{"secondary_y": True}]])
         if sub_data_df.empty:
@@ -164,20 +166,20 @@ def update_graph(start_date, end_date):
                 legend_yanchor="top",
                 legend_orientation="h",  # show entries horizontally
                 legend_x=0.5,  # put legend in center of x-axis
-                legend_y=-0.25,
+                legend_y=-0.30,
                 bargap=0,  # gap between bars of adjacent location coordinates.
                 bargroupgap=0,  # gap between bars of the same location coordinate.
                 margin_l=10,
                 margin_r=10,
                 margin_b=40,
-                margin_t=30,
+                margin_t=40,
                 height=600,
             )
             return fig
         fig.add_trace(
             go.Scatter(
                 x=value_per_df.index,
-                y=samples_df["samples"],
+                y=data_week_df["samples"],
                 mode="lines",
                 line_color="#0066cc",
                 line_width=2,
@@ -219,13 +221,13 @@ def update_graph(start_date, end_date):
             legend_yanchor="top",
             legend_orientation="h",  # show entries horizontally
             legend_x=0.5,  # put legend in center of x-axis
-            legend_y=-0.25,
+            legend_y=-0.30,
             bargap=0,  # gap between bars of adjacent location coordinates.
             bargroupgap=0,  # gap between bars of the same location coordinate.
             margin_l=10,
             margin_r=10,
             margin_b=40,
-            margin_t=30,
+            margin_t=40,
             height=600,
         )
         return fig