extend changes in database regarding models metadatavalue and sample_…

…state_history

.gitignore

@@ -8,10 +8,10 @@ migrations/
 /static/
 virtualenv/
 manage.py
-install_settings.txt
+install_*.txt
 relecov_platform/
 documents/
-initial_settings.txt
+initial_*.txt
 
 # VS code
 .vscode/

conf/template_settings.py

@@ -18,7 +18,7 @@
 ALLOWED_HOSTS = ["localhost", "127.0.0.1", "localserverip", "dns_url"]
 
 # Application definition
-
+# FIXME: The project should be renamed. "Pathoweb-'${project_name}' ?
 INSTALLED_APPS = [
     "django.contrib.admin",
     "django.contrib.auth",
@@ -76,7 +76,7 @@
         "USER": "djangouser",
         "PASSWORD": "djangopass",
         "PORT": "djangoport",
-        "NAME": "relecov",
+        "NAME": "pathoweb",
         "HOST": "djangohost",
     },
 }

core/api/utils/metadata_values.py

@@ -28,7 +28,9 @@ def store_metadata_values(s_data, schema_obj, analysis_date):
         sequencing_sample_id__iexact=s_data["sequencing_sample_id"]
     ).last()
     for field, value in s_data.items():
-
+        # This condition avoids error when assessing bioinformatics fields
+        if 'schema_' in field :
+            continue
         property_name = core.models.SchemaProperties.objects.filter(
             schemaID=schema_obj, property__iexact=field
         ).last()

core/models.py

@@ -341,7 +341,7 @@ def get_id(self):
         return "%s" % (self.pk)
 
     def get_b_process_field_id(self):
-        return "%s" % (self.bioinfo_analysis_fieldID)
+        return "%s" % (self.schema_property)
 
     objects = MetadataValuesManager()
 
@@ -693,8 +693,9 @@ def get_ena_info(self):
         return self.ena_obj.get_ena_data()
 
     def get_state(self):
-        if self.state:
-            return "%s" % (self.state.get_state())
+        latest_state = SampleStateHistory.objects.filter(sample=self, is_current=True).last()
+        if latest_state and latest_state.state:
+            return "%s" % (latest_state.state.get_state())
         return None
 
     def get_user(self):

core/utils/bioinfo_analysis.py

@@ -2,54 +2,91 @@
 import core.models
 import core.utils.samples
 import core.utils.schema
+from django.db.models import Max, F, Subquery, OuterRef
 
-
-# TODO: Replace the outdated DateUpdateState with the new SampleStateHistory
 def get_bio_analysis_stats_from_lab(lab_name=None):
     """Get the number of samples that are analized and compare with the number
     of recieved samples. If no lab name is given it matches all labs
     """
     bio_stats = {}
     if lab_name is None:
-        # get stats from all lab
-        bioqry = core.models.DateUpdateState.objects.filter(
-            stateID__state__iexact="Bioinfo"
+        bioqry = core.models.SampleStateHistory.objects.filter(
+            state__state__iexact="Bioinfo"
         )
-        bio_stats["analized"] = bioqry.values("sampleID").distinct().count()
+        bio_stats["analized"] = bioqry.values("sample_id").distinct().count()
         bio_stats["received"] = core.models.Sample.objects.all().count()
     else:
         sample_objs = core.models.Sample.objects.filter(
             collecting_institution__iexact=lab_name
         )
-        samples_bioquery = bioqry.filter(sampleID__in=sample_objs)
-        bio_stats["analized"] = samples_bioquery.values("sampleID").distinct().count()
+        bioqry = core.models.SampleStateHistory.objects.filter(
+            state__state__iexact="Bioinfo"
+        )
+        samples_bioquery = bioqry.filter(sample__in=sample_objs)
+        bio_stats["analized"] = samples_bioquery.values("sample_id").distinct().count()
         bio_stats["received"] = len(sample_objs)
     return bio_stats
 
-
 def get_bioinfo_analysis_data_from_sample(sample_id):
-    """Get the bioinfo analysis for the sample"""
+    """Get the latest bioinfo analysis data for the sample, ensuring unique values"""
+
+    # Retrieve the sample object
     sample_obj = core.utils.samples.get_sample_obj_from_id(sample_id)
     if not sample_obj:
         return None
+
     # Get the schema ID for filtering Fields
     schema_obj = sample_obj.get_schema_obj()
     bio_anlys_data = []
-    bioan_fields = core.models.BioinfoAnalysisField.objects.filter(schemaID=schema_obj)
-    if not bioan_fields.exists():
+
+    # Ensure schema_obj is valid
+    if not schema_obj:
+        return None
+
+    # Get all MetadataValues related to the schema and sample
+    bioan_fields_qs = core.models.MetadataValues.objects.filter(
+        schema_property__schemaID=schema_obj,
+        sample=sample_obj  # Ensure we only fetch for this specific sample
+    )
+
+    # Get the latest analysis_date for this sample
+    latest_analysis_date = bioan_fields_qs.aggregate(Max("analysis_date"))["analysis_date__max"]
+
+    # If no data exists, return None
+    if not latest_analysis_date:
         return None
-    for bio_field in bioan_fields:
-        samples_bio = core.models.BioinfoAnalysisValue.objects.filter(
-            bioinfo_analysis_fieldID=bio_field, sample=sample_obj
+
+    # Find the latest generated_at for each unique value and schema_property_id
+    latest_bioan_fields = bioan_fields_qs.filter(
+        analysis_date=latest_analysis_date,
+        generated_at=Subquery(
+            bioan_fields_qs.filter(
+                analysis_date=latest_analysis_date,
+                schema_property=OuterRef("schema_property"),
+                value=OuterRef("value")
+            ).order_by("-generated_at").values("generated_at")[:1]
         )
-        if samples_bio.exists():
-            value = samples_bio.last().get_value()
-        else:
-            value = ""
-        bio_anlys_data.append([bio_field.get_label(), value])
+    )
+
+    # If still empty, return None
+    if not latest_bioan_fields.exists():
+        return None
+
+    # Iterate through the unique latest bioan_fields queryset
+    for bio_field in latest_bioan_fields:
+        value = bio_field.get_value() if bio_field else ""
+
+        bio_anlys_data.append([
+            bio_field.schema_property.get_label(),
+            value
+        ])
+
     return bio_anlys_data
 
 
+
+# FIXME: Replace BioinfoAnalysisField wit MetadataValue
+# TODO: I think this function belongs to dashboard app
 def get_bioinfo_analyis_fields_utilization(schema_obj=None):
     """Get the level of utilization for the bioinfo analysis fields.
     If schema is not given, the function get the latest default schema

core/utils/samples.py

@@ -315,7 +315,6 @@ def delete_temporary_sample_table(user_obj):
     return True
 
 
-# TODO: Replace the outdated DateUpdateState with the new SampleStateHistory
 def get_lab_last_actions(lab_name=None):
     """Get the last action performed on the samples for a specific lab.
     If no lab is given it returns the info for all labs
@@ -334,12 +333,11 @@ def get_lab_last_actions(lab_name=None):
             ).last()
             lab_data = [lab[0]]
             for action in action_list:
-                if core.models.DateUpdateState.objects.filter(
-                    sampleID=sam_obj, stateID__state__exact=action
-                ).exists():
+                if core.models.SampleStateHistory.objects.filter(sample=sam_obj, state__state__exact=action).exist():
                     lab_data.append(
-                        core.models.DateUpdateState.objects.filter(
-                            sampleID=sam_obj, stateID__state__exact=action
+                        core.models.SampleStateHistory.objects.filter(
+                            sample=sam_obj, 
+                            state__state__exact=action
                         )
                         .last()
                         .get_date()
@@ -353,11 +351,11 @@ def get_lab_last_actions(lab_name=None):
         last_sample_obj = core.models.Sample.objects.filter(
             collecting_institution__iexact=lab_name
         ).last()
-        action_objs = core.models.DateUpdateState.objects.filter(
-            sampleID=last_sample_obj
+        action_objs = core.models.SampleStateHistory.objects.filter(
+            sample=last_sample_obj
         )
         for action_obj in action_objs:
-            s_state = action_obj.get_state_name()
+            s_state = action_obj.get_state()
             if s_state in action_list:
                 actions[s_state] = action_obj.get_date()
         return actions
@@ -392,7 +390,6 @@ def get_public_database_fields(schema_obj, db_type):
     return None
 
 
-# TODO: Replace the outdated DateUpdateState with the new SampleStateHistory
 def get_sample_display_data(sample_id, user):
     """Check if user is allowed to see the data and if true collect all info
     from sample to display
@@ -423,14 +420,14 @@ def get_sample_display_data(sample_id, user):
         )
     )
     # Fetch actions done on the sample
-    if core.models.DateUpdateState.objects.filter(sampleID=sample_obj).exists():
+    if core.models.SampleStateHistory.objects.filter(sample=sample_obj).exists():
         actions = []
-        actions_date_objs = core.models.DateUpdateState.objects.filter(
-            sampleID=sample_obj
-        ).order_by("-date")
+        actions_date_objs = core.models.SampleStateHistory.objects.filter(
+            sample=sample_obj
+        ).order_by("-changed_at")
         for action_date_obj in actions_date_objs:
             actions.append(
-                [action_date_obj.get_state_display_name(), action_date_obj.get_date()]
+                [action_date_obj.get_state(), action_date_obj.get_date()]
             )
         s_data["actions"] = actions
 
@@ -444,11 +441,11 @@ def get_sample_display_data(sample_id, user):
             # iskylims_data is a list with one element. Then get the first element
             for key, i_data in iskylims_data[0].items():
                 if key == "Project values":
-                    for p_key, p_data in iskylims_data["Project values"].items():
+                    for p_key, p_data in iskylims_data[0]["Project values"].items():
                         s_data["iskylims_p_data"].append([p_key, p_data])
                 else:
                     s_data["iskylims_basic"].append([key, i_data])
-            s_data["iskylims_project"] = iskylims_data["sample_project"]
+            s_data["iskylims_project"] = iskylims_data[0]["sample_project"]
     return s_data
 
 
@@ -529,18 +526,25 @@ def get_sample_per_date_per_lab(lab_name):
     and number of samples
     """
     samples_per_date = OrderedDict()
-
+    sample_qs = get_sample_objs_per_lab(lab_name)
+    sequence_date_obj = core.models.SchemaProperties.objects.get(property='sequencing_date')
     s_dates = (
-        core.models.Sample.objects.filter(collecting_institution__iexact=lab_name)
-        .values_list("sequencing_date", flat=True)
+        core.models.MetadataValues.objects.filter(
+            sample__in=sample_qs,
+            schema_property__id=sequence_date_obj.pk
+        )
+        .values_list("value", flat=True)
         .distinct()
-        .order_by("sequencing_date")
+        .order_by("value")
     )
+
     for s_date in s_dates:
-        date = datetime.strftime(s_date, "%d-%B-%Y")
-        samples_per_date[date] = core.models.Sample.objects.filter(
-            collecting_institution__iexact=lab_name, sequencing_date=s_date
-        ).count()
+        if s_date:
+            date = datetime.strptime(s_date, "%Y-%m-%d").strftime("%d-%B-%Y")
+            samples_per_date[date] = core.models.MetadataValues.objects.filter(
+                sample__in=sample_qs, value=s_date, schema_property__id=1405
+            ).count()
+
     return samples_per_date
 
 
@@ -719,7 +723,7 @@ def save_excel_form_in_samba_folder(m_file, user_name):
     return
 
 
-# TODO: Replace the outdated DateUpdateState with the new SampleStateHistory
+# FIXME: Replace the outdated DateUpdateState with the new SampleStateHistory
 def search_samples(sample_name, lab_name, sample_state, s_date, user):
     """Search the samples that match with the query conditions"""
     sample_list = []

core/views.py

@@ -65,6 +65,8 @@ def sample_display(request, sample_id):
     sample_data["ena"] = core.utils.public_db.get_public_information_from_sample(
         "ena", sample_id
     )
+    # TODO: Reduce search waiting time by optimizing DB queries
+    # FIXME: Some tables in the template appear abnormally. Fix it and discuss the strategy followed in get_bioinfo_analysis_data_from_sample
     sample_data["bioinfo"] = (
         core.utils.bioinfo_analysis.get_bioinfo_analysis_data_from_sample(sample_id)
     )
@@ -169,7 +171,7 @@ def search_sample(request):
         )
     return render(request, "core/searchSample.html", {"search_data": search_data})
 
-
+# FIXME: This needs a template or error message when user != admin tryies to access.
 @login_required
 def metadata_visualization(request):
     if request.user.username != "admin":
@@ -360,7 +362,9 @@ def metadata_form(request):
         m_batch_form = core.utils.samples.create_form_for_batch(
             schema_obj, request.user
         )
-        sample_saved = core.utils.samples.get_sample_pre_recorded(request.user)
+        sample_saved = core.utils.samples.get_sample_pre_recorded(
+            request.user
+        )
         return render(
             request,
             "core/metadataForm.html",