Funman tracking n result (#3447)

Co-authored-by: mwdchang <[email protected]>
Co-authored-by: Charles Coleman <[email protected]>

packages/client/hmi-client/src/types/Types.ts

@@ -1376,6 +1376,7 @@ export enum SimulationType {
  Simulation = "SIMULATION",
  Calibration = "CALIBRATION",
  Optimization = "OPTIMIZATION",
+ Validation = "VALIDATION",
 }
 
 export enum ProgressState {

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/funman/FunmanController.java

@@ -8,21 +8,23 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.annotation.PostConstruct;
-import java.util.UUID;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.annotation.Secured;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.server.ResponseStatusException;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.ProgressState;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.Simulation;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.SimulationType;
 import software.uncharted.terarium.hmiserver.models.funman.FunmanPostQueriesRequest;
 import software.uncharted.terarium.hmiserver.models.task.TaskRequest;
 import software.uncharted.terarium.hmiserver.models.task.TaskRequest.TaskType;
-import software.uncharted.terarium.hmiserver.models.task.TaskResponse;
 import software.uncharted.terarium.hmiserver.proxies.funman.FunmanProxy;
 import software.uncharted.terarium.hmiserver.security.Roles;
 import software.uncharted.terarium.hmiserver.service.CurrentUserService;
+import software.uncharted.terarium.hmiserver.service.data.SimulationService;
 import software.uncharted.terarium.hmiserver.service.tasks.TaskService;
 import software.uncharted.terarium.hmiserver.service.tasks.TaskService.TaskMode;
 import software.uncharted.terarium.hmiserver.service.tasks.ValidateModelConfigHandler;
@@ -38,6 +40,7 @@ public class FunmanController {
  private final CurrentUserService currentUserService;
 
  private final ValidateModelConfigHandler validateModelConfigHandler;
+ private final SimulationService simulationService;
 
  @PostConstruct
  void init() {
@@ -57,14 +60,14 @@ void init() {
  mediaType = "application/json",
  schema =
  @io.swagger.v3.oas.annotations.media.Schema(
- implementation = TaskResponse.class))),
+ implementation = Simulation.class))),
  @ApiResponse(responseCode = "400", description = "Invalid input or bad request", content = @Content),
  @ApiResponse(
  responseCode = "500",
  description = "There was an issue dispatching the request",
  content = @Content)
  })
- public ResponseEntity<TaskResponse> createValidationRequest(@RequestBody final JsonNode input) {
+ public ResponseEntity<Simulation> createValidationRequest(@RequestBody final JsonNode input) {
 
  try {
  final TaskRequest taskRequest = new TaskRequest();
@@ -73,18 +76,26 @@ public ResponseEntity<TaskResponse> createValidationRequest(@RequestBody final J
  taskRequest.setUserId(currentUserService.get().getId());
  taskRequest.setInput(objectMapper.writeValueAsBytes(input));
 
- // TODO:
- // - create Simulation for tracking
- // - mark Simulation as pending
- final UUID uuid = UUID.randomUUID();
+ final Simulation sim = new Simulation();
+ sim.setType(SimulationType.VALIDATION);
+ sim.setStatus(ProgressState.RUNNING);
+ sim.setDescription("funman model configuration validation");
+ sim.setExecutionPayload(objectMapper.convertValue(input, JsonNode.class));
+
+ // Create new simulatin object to proxy the funman validation process
+ Simulation newSimulation = simulationService.createAsset(sim);
+
  final ValidateModelConfigHandler.Properties props = new ValidateModelConfigHandler.Properties();
- props.setSimulationId(uuid);
+ props.setSimulationId(newSimulation.getId());
  taskRequest.setAdditionalProperties(props);
+ taskService.runTask(TaskMode.ASYNC, taskRequest);
 
- return ResponseEntity.ok().body(taskService.runTask(TaskMode.ASYNC, taskRequest));
+ return ResponseEntity.ok(newSimulation);
  } catch (final ResponseStatusException e) {
+ e.printStackTrace();
  throw e;
  } catch (final Exception e) {
+ e.printStackTrace();
  final String error = "Unable to dispatch task request";
  throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
  }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/simulation/Simulation.java

@@ -30,13 +30,16 @@ public class Simulation extends TerariumAsset {
 
  @JsonAlias("execution_payload")
  @Convert(converter = JpaConverterJson.class)
+ @Column(columnDefinition = "text")
  private JsonNode executionPayload;
 
  @TSOptional
+ @Column(length = 1000)
  private String description;
 
  @JsonAlias("result_files")
  @TSOptional
+ @Column(length = 1000)
  @Schema(accessMode = Schema.AccessMode.READ_ONLY)
  @ElementCollection
  private List<String> resultFiles;
@@ -49,6 +52,7 @@ public class Simulation extends TerariumAsset {
 
  @TSOptional
  @Schema(accessMode = Schema.AccessMode.READ_ONLY)
+ @Column(columnDefinition = "text")
  private String statusMessage;
 
  @JsonAlias("start_time")

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/simulation/SimulationType.java

@@ -10,5 +10,7 @@ public enum SimulationType {
  @JsonAlias({"calibrate", "calibration"})
  CALIBRATION,
  @JsonAlias({"optimize", "optimization"})
- OPTIMIZATION
+ OPTIMIZATION,
+ @JsonAlias({"validation"})
+ VALIDATION
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/tasks/ValidateModelConfigHandler.java

@@ -1,18 +1,31 @@
 package software.uncharted.terarium.hmiserver.service.tasks;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.Optional;
 import java.util.UUID;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
 import org.springframework.stereotype.Component;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.ProgressState;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.Simulation;
 import software.uncharted.terarium.hmiserver.models.task.TaskResponse;
+import software.uncharted.terarium.hmiserver.service.data.SimulationService;
 
 @Component
 @RequiredArgsConstructor
 @Slf4j
 public class ValidateModelConfigHandler extends TaskResponseHandler {
  public static final String NAME = "funman_task:validate_modelconfig";
 
+ private final ObjectMapper objectMapper;
+ private final SimulationService simulationService;
+
  @Override
  public String getName() {
  return NAME;
@@ -25,17 +38,33 @@ public static class Properties {
 
  @Override
  public TaskResponse onSuccess(final TaskResponse resp) {
+ final String resultFilename = "validation.json";
  try {
+ // Parse validation result
  final Properties props = resp.getAdditionalProperties(Properties.class);
  final UUID simulationId = props.getSimulationId();
+ Optional<Simulation> sim = simulationService.getAsset(simulationId);
+ if (sim.isEmpty()) {
+ log.error("Cannot find Simulation " + simulationId + " for task " + resp.getId());
+ throw new Error("Cannot find Simulation " + simulationId + " for task " + resp.getId());
+ }
+
+ // Retrive final result json
+ final JsonNode result = objectMapper.readValue(resp.getOutput(), JsonNode.class);
+
+ // Upload final result into S3
+ final byte[] bytes = objectMapper.writeValueAsBytes(result.get("response"));
+ final HttpEntity fileEntity = new ByteArrayEntity(bytes, ContentType.APPLICATION_OCTET_STREAM);
+ simulationService.uploadFile(simulationId, resultFilename, fileEntity, ContentType.TEXT_PLAIN);
+
+ // Mark simulation as completed, update result file
+ sim.get().setStatus(ProgressState.COMPLETE);
+ final ArrayList<String> resultFiles = new ArrayList<String>();
+ resultFiles.add(resultFilename);
+ sim.get().setResultFiles(resultFiles);
 
- // TODO:
- // - Retrive final result json
- // - Upload final result into S3
- // - Mark simulation as completed, update result file
- System.out.println("");
- System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! " + simulationId.toString());
- System.out.println("");
+ // Save
+ simulationService.updateAsset(sim.get());
  } catch (final Exception e) {
  throw new RuntimeException(e);
  }

packages/server/src/main/resources/application-local.properties

@@ -6,6 +6,7 @@ spring.datasource.password=postgres
 spring.datasource.username=postgres
 spring.datasource.initialize=true
 spring.jpa.hibernate.ddl-auto=update
+spring.flyway.enabled=true
 
 ########################################################################################################################
 # Elasticsearch configuration

packages/server/src/main/resources/application.properties

@@ -7,6 +7,7 @@ spring.datasource.username=${terarium.db.username}
 spring.datasource.initialize=false
 spring.jpa.hibernate.ddl-auto=none
 spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
+spring.flyway.enabled=false
 
 ########################################################################################################################
 # Elasticsearch configuration
@@ -148,6 +149,7 @@ skema-unified.url=https://skema-unified.staging.terarium.ai
 terarium.dataservice.url=https://data-service.staging.terarium.ai
 xdd-dev-service.url=https://xdddev.chtc.io
 xdd-prod-service.url=https://xdd.wisc.edu
+cosmos-service.url=https://cosmos.staging.terarium.ai
 funman-service.url=https://funman.staging.terarium.ai
 simulation-service.url=https://sciml-service.staging.terarium.ai
 ciemss-service.url=https://pyciemss.staging.terarium.ai

packages/server/src/main/resources/db/migration/V3__Simulation_funman_tracking.sql

@@ -0,0 +1,9 @@
+BEGIN;
+ALTER TABLE simulation DROP CONSTRAINT simulation_type_check;
+ALTER TABLE simulation ADD CONSTRAINT simulation_type_check CHECK(type = ANY(ARRAY['ENSEMBLE', 'SIMULATION', 'CALIBRATION', 'VALIDATION']));
+
+ALTER TABLE simulation ALTER COLUMN execution_payload TYPE text;
+ALTER TABLE simulation ALTER COLUMN description TYPE VARCHAR(1000);
+ALTER TABLE simulation ALTER COLUMN result_files TYPE VARCHAR(1000)[];
+ALTER TABLE simulation ALTER COLUMN status_message TYPE text;
+COMMIT;