diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/DataMigration.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/DataMigration.java index b3d41ae7cf..2a27a436e1 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/DataMigration.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/DataMigration.java @@ -23,4 +23,8 @@ public enum MigrationState { private MigrationState state; private Timestamp timestamp; + + private Integer totalDocuments; + private Integer successfulDocuments; + private Integer failedDocuments; } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java index fac4ccb32b..73783f6eb8 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java @@ -20,6 +20,7 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration; @@ -73,14 +74,14 @@ private static class MigrationConfig migrateFromEsToPg(final ElasticsearchService elasticService) throws IOException { // check if there is a target index to migrate from if (!elasticService.indexExists(index)) { - return; + return Pair.of(0, 0); } if (elasticService.count(index) == 0) { - return; + return Pair.of(0, 0); } // check if the data has already been migrated @@ -92,6 +93,9 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep final String SORT_FIELD = "createdOn"; + Integer totalSuccess = 0; + Integer totalFailed = 0; + while (true) { final SearchRequest.Builder reqBuilder = new SearchRequest.Builder() .index(index) @@ -132,13 +136,15 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep for (final T asset : assets) { try { service.getRepository().save(asset); + totalSuccess++; } catch (final Exception e) { log.warn("Failed to insert id: {}", asset.getId(), e); failed += 1; + totalFailed++; } } if (failed == assets.size()) { - throw new RuntimeException("All assets faield to insert"); + throw new RuntimeException("All assets failed to insert"); } } @@ -150,6 +156,8 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep // over and over lastPagesLastId = lastId; } + + return Pair.of(totalSuccess, totalFailed); } } @@ -180,12 +188,15 @@ void runMigrations() { continue; } - migration.migrateFromEsToPg(elasticService); + final Pair res = migration.migrateFromEsToPg(elasticService); migrationRecord = new DataMigration(); migrationRecord.setTableName(tableName); migrationRecord.setTimestamp(new Timestamp(System.currentTimeMillis())); migrationRecord.setState(MigrationState.SUCCESS); + migrationRecord.setTotalDocuments(res.getLeft() + res.getRight()); + migrationRecord.setSuccessfulDocuments(res.getLeft()); + migrationRecord.setFailedDocuments(res.getRight()); migrationRepository.save(migrationRecord); log.info("Migrated ES index {} to PG table {} successfully", migration.getIndex(), tableName);