Add document counts to migrations (#3781)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: David Gauldie <[email protected]>

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/DataMigration.java

@@ -23,4 +23,8 @@ public enum MigrationState {
 
  private MigrationState state;
  private Timestamp timestamp;
+
+ private Integer totalDocuments;
+ private Integer successfulDocuments;
+ private Integer failedDocuments;
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java

@@ -20,6 +20,7 @@
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration;
@@ -73,14 +74,14 @@ private static class MigrationConfig<T extends TerariumAsset, R extends PSCrudSo
  this.index = index;
  }
 
- void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOException {
+ Pair<Integer, Integer> migrateFromEsToPg(final ElasticsearchService elasticService) throws IOException {
  // check if there is a target index to migrate from
  if (!elasticService.indexExists(index)) {
- return;
+ return Pair.of(0, 0);
  }
 
  if (elasticService.count(index) == 0) {
- return;
+ return Pair.of(0, 0);
  }
 
  // check if the data has already been migrated
@@ -92,6 +93,9 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep
 
  final String SORT_FIELD = "createdOn";
 
+ Integer totalSuccess = 0;
+ Integer totalFailed = 0;
+
  while (true) {
  final SearchRequest.Builder reqBuilder = new SearchRequest.Builder()
  .index(index)
@@ -132,13 +136,15 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep
  for (final T asset : assets) {
  try {
  service.getRepository().save(asset);
+ totalSuccess++;
  } catch (final Exception e) {
  log.warn("Failed to insert id: {}", asset.getId(), e);
  failed += 1;
+ totalFailed++;
  }
  }
  if (failed == assets.size()) {
- throw new RuntimeException("All assets faield to insert");
+ throw new RuntimeException("All assets failed to insert");
  }
  }
 
@@ -150,6 +156,8 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep
  // over and over
  lastPagesLastId = lastId;
  }
+
+ return Pair.of(totalSuccess, totalFailed);
  }
  }
 
@@ -180,12 +188,15 @@ void runMigrations() {
  continue;
  }
 
- migration.migrateFromEsToPg(elasticService);
+ final Pair<Integer, Integer> res = migration.migrateFromEsToPg(elasticService);
 
  migrationRecord = new DataMigration();
  migrationRecord.setTableName(tableName);
  migrationRecord.setTimestamp(new Timestamp(System.currentTimeMillis()));
  migrationRecord.setState(MigrationState.SUCCESS);
+ migrationRecord.setTotalDocuments(res.getLeft() + res.getRight());
+ migrationRecord.setSuccessfulDocuments(res.getLeft());
+ migrationRecord.setFailedDocuments(res.getRight());
  migrationRepository.save(migrationRecord);
 
  log.info("Migrated ES index {} to PG table {} successfully", migration.getIndex(), tableName);