TASK-7481 - Sample Index Configuration completely empty on new studies

.github/workflows/test-analysis.yml

@@ -97,10 +97,11 @@ jobs:
             MODULE="opencga"
           fi
           TAGS=$(echo ${{ inputs.test_profile }} | sed -e 's/run\([^,]*\)Tests/\1/g'  | tr ',' '_' | tr '[:upper:]' '[:lower:]' )
-          echo "TESTS_LOG_FILE_NAME=junit_${{ inputs.hadoop }}_${TAGS}_${MODULE}.log" >> $GITHUB_OUTPUT
+          echo "TESTS_LOG_FILE_NAME=junit_${{ inputs.hadoop }}_${TAGS}_${MODULE}.log.gz" >> $GITHUB_OUTPUT
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}  |& tee ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }} |& grep -P '^\[[^\]]*(INFO|WARNING|ERROR)' --colour=never --line-buffered
+        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}  |& tee >(gzip > ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }} ) |& grep -P '^\[[^\]]*(INFO|WARNING|ERROR)' --colour=never --line-buffered
       - name: Upload Junit test logs
+        if: success() || failure()
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.BuildJunitLogFileName.outputs.TESTS_LOG_FILE_NAME }}

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java

@@ -37,6 +37,7 @@
 import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.common.BatchUtils;
 import org.opencb.opencga.core.config.storage.CellBaseConfiguration;
+import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
 import org.opencb.opencga.core.models.cohort.Cohort;
 import org.opencb.opencga.core.models.cohort.CohortStatus;
 import org.opencb.opencga.core.models.cohort.CohortUpdateParams;
@@ -48,6 +49,7 @@
 import org.opencb.opencga.core.models.project.Project;
 import org.opencb.opencga.core.models.project.ProjectOrganism;
 import org.opencb.opencga.core.models.sample.*;
+import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager;
@@ -421,8 +423,26 @@ protected boolean synchronizeCohorts(StudyMetadata study, String sessionId) thro
         return modified;
     }
 
+    private boolean synchronizeSampleIndexConfiguration(StudyMetadata studyMetadata, String token) throws CatalogException {
+        Study study = catalogManager.getStudyManager().get(studyMetadata.getName(), new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(
+                StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION.key())), token).first();
+        SampleIndexConfiguration sampleIndexFromCatalog = study.getInternal().getConfiguration().getVariantEngine().getSampleIndex();
+        SampleIndexConfiguration sampleIndexFromStorage = studyMetadata.getSampleIndexConfigurationLatest().getConfiguration();
+        if (!sampleIndexFromStorage.equals(sampleIndexFromCatalog)) {
+            logger.info("Update sample index configuration from storage for study {}", studyMetadata.getName());
+            catalogManager.getStudyManager().setVariantEngineConfigurationSampleIndex(studyMetadata.getName(), sampleIndexFromStorage, token);
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String token) throws CatalogException {
         boolean modified = false;
+
+        // FIXME: This method call should be relocated
+        modified |= synchronizeSampleIndexConfiguration(study, token);
+
         Map<String, Integer> fileNameMap = new HashMap<>();
         Map<Integer, String> filePathMap = new HashMap<>();
         Set<Integer> virtualFiles = new HashSet<>();

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/AbstractVariantOperationManagerTest.java

@@ -168,7 +168,7 @@ public final void setUpAbstract() throws Exception {
         outputPath = "data/index/";
         studyId2 = catalogManager.getStudyManager().create(projectId, "s2", "s2", "s2", "Study " + "2", null, null,
                 null, Collections.singletonMap(VariantStatsAnalysis.STATS_AGGREGATION_CATALOG, getAggregation()),
-                new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first().getId();
+                        new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionId).first().getId();
         outputId2 = catalogManager.getFileManager().createFolder(studyId2, Paths.get("data", "index").toString(),
                 true, null, QueryOptions.empty(), sessionId).first().getId();
 

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManagerTest.java

@@ -34,10 +34,12 @@
 import org.opencb.opencga.catalog.db.api.CohortDBAdaptor;
 import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
 import org.opencb.opencga.catalog.db.api.SampleDBAdaptor;
+import org.opencb.opencga.catalog.db.api.StudyDBAdaptor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.utils.Constants;
 import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.common.UriUtils;
+import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
 import org.opencb.opencga.core.models.cohort.CohortStatus;
 import org.opencb.opencga.core.models.file.File;
 import org.opencb.opencga.core.models.file.FileInternalVariantIndex;
@@ -172,7 +174,20 @@ public void testIndex() throws Exception {
         QueryOptions queryOptions = new QueryOptions(VariantStorageOptions.ANNOTATE.key(), false)
                 .append(VariantStorageOptions.STATS_CALCULATE.key(), false);
 
+        QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.INTERNAL_CONFIGURATION.key()));
+        SampleIndexConfiguration sampleIndex = catalogManager.getStudyManager().get(studyId, options, sessionId)
+                .first().getInternal().getConfiguration().getVariantEngine().getSampleIndex();
+        assertNotNull(sampleIndex);
+
+        catalogManager.getStudyManager().setVariantEngineConfigurationSampleIndex(studyId, null, sessionId);
+        sampleIndex = catalogManager.getStudyManager().get(studyId, options, sessionId)
+                .first().getInternal().getConfiguration().getVariantEngine().getSampleIndex();
+        assertNull(sampleIndex);
+
         indexFile(getFile(0), queryOptions, outputId);
+        sampleIndex = catalogManager.getStudyManager().get(studyId, options, sessionId)
+                .first().getInternal().getConfiguration().getVariantEngine().getSampleIndex();
+        assertNotNull(sampleIndex);
     }
 
     @Test

opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_4_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java → opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java

@@ -1,22 +1,24 @@
-package org.opencb.opencga.app.migrations.v3.v3_4_0.storage;
+package org.opencb.opencga.app.migrations.v4.v4_0_0.storage;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.opencb.opencga.app.migrations.StorageMigrationTool;
 import org.opencb.opencga.catalog.migration.Migration;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
+import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.storage.core.metadata.models.StudyMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.function.Supplier;
 
 @Migration(id = "ensure_sample_index_configuration_is_defined",
         description = "Ensure that the SampleIndexConfiguration object is correctly defined. #TASK-6765", version = "4.0.0",
         language = Migration.MigrationLanguage.JAVA,
         domain = Migration.MigrationDomain.STORAGE,
-        patch = 1,
+        patch = 2,
         date = 20240910)
 public class EnsureSampleIndexConfigurationIsAlwaysDefined extends StorageMigrationTool {
 
@@ -25,7 +27,9 @@ protected void run() throws Exception {
 
         for (String variantStorageProject : getVariantStorageProjects()) {
             VariantStorageEngine engine = getVariantStorageEngineByProject(variantStorageProject);
+            logger.info("Checking project '" + variantStorageProject + "'");
             if (engine.getMetadataManager().exists()) {
+                logger.info("Project '" + variantStorageProject + "' exists in variant storage. Checking studies.");
                 for (Integer studyId : engine.getMetadataManager().getStudyIds()) {
                     StudyMetadata studyMetadata = engine.getMetadataManager().getStudyMetadata(studyId);
                     if (CollectionUtils.isEmpty(studyMetadata.getSampleIndexConfigurations())) {
@@ -42,11 +46,32 @@ protected void run() throws Exception {
                             }
                         });
                     }
+                    Study study = catalogManager.getStudyManager().get(studyMetadata.getName(), null, token).first();
+                    SampleIndexConfiguration sampleIndexConfFromStorage = studyMetadata.getSampleIndexConfigurationLatest().getConfiguration();
+                    SampleIndexConfiguration sampleIndexConfFromCatalog = orDefault(() -> study.getInternal()
+                            .getConfiguration().getVariantEngine().getSampleIndex(), null);
+
+                    if (sampleIndexConfFromStorage.equals(sampleIndexConfFromCatalog)) {
+                        logger.info("SampleIndexConfiguration for study '" + studyMetadata.getName() + "' (" + studyId + ") is up to date.");
+                    } else {
+                        logger.info("Updating SampleIndexConfiguration for study '" + studyMetadata.getName() + "' (" + studyId + ")");
+                        catalogManager.getStudyManager().setVariantEngineConfigurationSampleIndex(studyMetadata.getName(),
+                                sampleIndexConfFromStorage, token);
+                    }
                 }
+            } else {
+                logger.info("Project '" + variantStorageProject + "' does not exist in variant storage. Skipping.");
             }
         }
     }
 
+    public static <T> T orDefault(Supplier<T> supplier, T defaultValue) {
+        try {
+            return supplier.get();
+        } catch (Exception e) {
+            return defaultValue;
+        }
+    }
 
     public static SampleIndexConfiguration preFileDataConfiguration() {
         // If missing, it was assuming cellbase v5

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java

@@ -424,7 +424,7 @@ public OpenCGAResult<Study> create(String projectStr, Study study, QueryOptions
 
             authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId);
             String cellbaseVersion;
-            if (project.getCellbase() == null || StringUtils.isEmpty(project.getCellbase().getUrl())) {
+            if (project.getCellbase() != null && StringUtils.isNotEmpty(project.getCellbase().getUrl())) {
                 CellBaseValidator cellBaseValidator = new CellBaseValidator(
                         project.getCellbase(),
                         project.getOrganism().getScientificName(),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java

@@ -664,9 +664,16 @@ private MigrationRun run(String organizationId, Class<? extends MigrationTool> r
         }
         migrationTool.setup(configuration, catalogManager, dbAdaptorFactory, migrationRun, organizationId, appHome, params, token);
 
+        String userId;
+        try {
+            userId = catalogManager.getUserManager().validateToken(token).getUserId();
+        } catch (CatalogException e) {
+            throw new MigrationException(e);
+        }
         StopWatch stopWatch = StopWatch.createStarted();
         String path = Paths.get("JOBS")
-                .resolve("opencga")
+                .resolve(organizationId)
+                .resolve(userId)
                 .resolve(TimeUtils.getDay())
                 .resolve(annotation.id()).toString();
         String jobId = "migration"