Import / Export related fixes (#3745)

Co-authored-by: kbirk <[email protected]>

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java

@@ -259,7 +259,7 @@ private Dataset extractColumnsAsNeededAndSave(final Dataset dataset, final Schem
  // columns are set. No need to extract
  return dataset;
  }
- if (dataset.getFileNames() == null || dataset.getFileNames().isEmpty()) {
+ if (dataset.getFileNames() != null || dataset.getFileNames().isEmpty()) {
  // no file names to extract columns from
  return dataset;
  }
@@ -679,9 +679,7 @@ public ResponseEntity<Void> uploadData( // HttpServletRequest request,
  throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
  }
 
- if (updatedDataset.get().getFileNames() == null) {
- updatedDataset.get().setFileNames(new ArrayList<>(List.of(filename)));
- } else {
+ if (!updatedDataset.get().getFileNames().contains(filename)) {
  updatedDataset.get().getFileNames().add(filename);
  }
 
@@ -772,9 +770,7 @@ private ResponseEntity<ResponseStatus> uploadCSVAndUpdateColumns(
  updateHeaders(updatedDataset.get(), Arrays.asList(headers));
 
  // add the filename to existing file names
- if (updatedDataset.get().getFileNames() == null) {
- updatedDataset.get().setFileNames(new ArrayList<>(List.of(filename)));
- } else if (!updatedDataset.get().getFileNames().contains(filename)) {
+ if (!updatedDataset.get().getFileNames().contains(filename)) {
  updatedDataset.get().getFileNames().add(filename);
  }
 

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DocumentController.java

@@ -556,7 +556,9 @@ public ResponseEntity<Void> createDocumentFromXDD(@RequestBody final AddDocument
  DocumentAsset documentAsset = createDocumentAssetFromXDDDocument(
  document, userId, extractionResponse.getSuccess().getData(), summaries, permission);
  if (filename != null) {
- documentAsset.getFileNames().add(filename);
+ if (!documentAsset.getFileNames().contains(filename)) {
+ documentAsset.getFileNames().add(filename);
+ }
  documentAsset = documentAssetService
  .updateAsset(documentAsset, permission)
  .orElseThrow();
@@ -809,7 +811,6 @@ private DocumentAsset createDocumentAssetFromXDDDocument(
  .getMetadata()
  .put("description", extraction.getProperties().getCaption());
  documentAsset.getAssets().add(documentExtraction);
- documentAsset.getFileNames().add(documentExtraction.getFileName());
  }
  }
  }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java

@@ -22,6 +22,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.annotation.Secured;
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -32,7 +33,9 @@
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.server.ResponseStatusException;
 import software.uncharted.terarium.hmiserver.models.TerariumAsset;
 import software.uncharted.terarium.hmiserver.models.dataservice.AssetType;
@@ -567,31 +570,78 @@ public ResponseEntity<ProjectExport> exportProject(@PathVariable("id") final UUI
  @ApiResponses(
  value = {
  @ApiResponse(
- responseCode = "200",
+ responseCode = "201",
  description = "The project export",
  content = {
  @Content(
  mediaType = "application/json",
  schema =
- @io.swagger.v3.oas.annotations.media.Schema(
- implementation = ProjectExport.class))
+ @io.swagger.v3.oas.annotations.media.Schema(implementation = Project.class))
  }),
+ @ApiResponse(
+ responseCode = "400",
+ description = "An error occurred when trying to parse the import file",
+ content = @Content),
+ @ApiResponse(
+ responseCode = "500",
+ description = "There was a rebac issue when creating the project",
+ content = @Content),
  @ApiResponse(
  responseCode = "503",
  description = "An error occurred when trying to communicate with either the postgres or spicedb"
  + " databases",
  content = @Content)
  })
- @PostMapping("/import")
+ @PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
  @Secured(Roles.USER)
- public ResponseEntity<Project> importProject(@RequestBody final ProjectExport projectExport) {
+ public ResponseEntity<Project> importProject(@RequestPart("file") final MultipartFile input) {
+
+ ProjectExport projectExport;
  try {
- return ResponseEntity.ok(cloneService.importProject(projectExport));
+ projectExport = objectMapper.readValue(input.getInputStream(), ProjectExport.class);
+ } catch (final Exception e) {
+ log.error("Error parsing project export", e);
+ throw new ResponseStatusException(HttpStatus.BAD_REQUEST, messages.get("projects.import-parse-failure"));
+ }
+
+ final String userId = currentUserService.get().getId();
+ final String userName = userService.getById(userId).getName();
+
+ Project project;
+ try {
+ project = cloneService.importProject(userId, userName, projectExport);
  } catch (final Exception e) {
  log.error("Error importing project", e);
  throw new ResponseStatusException(
  HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable"));
  }
+
+ try {
+ project = projectService.createProject(project);
+ } catch (final Exception e) {
+ log.error("Error creating project", e);
+ throw new ResponseStatusException(
+ HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable"));
+ }
+
+ try {
+ final RebacProject rebacProject = new RebacProject(project.getId(), reBACService);
+ final RebacGroup rebacAskemAdminGroup = new RebacGroup(ReBACService.ASKEM_ADMIN_GROUP_ID, reBACService);
+ final RebacUser rebacUser = new RebacUser(userId, reBACService);
+
+ rebacUser.createCreatorRelationship(rebacProject);
+ rebacAskemAdminGroup.createWriterRelationship(rebacProject);
+ } catch (final Exception e) {
+ log.error("Error setting user's permissions for project", e);
+ throw new ResponseStatusException(
+ HttpStatus.SERVICE_UNAVAILABLE, messages.get("rebac.service-unavailable"));
+ } catch (final RelationshipAlreadyExistsException e) {
+ log.error("Error the user is already the creator of this project", e);
+ throw new ResponseStatusException(
+ HttpStatus.INTERNAL_SERVER_ERROR, messages.get("rebac.relationship-already-exists"));
+ }
+
+ return ResponseEntity.status(HttpStatus.CREATED).body(project);
  }
 
  // --------------------------------------------------------------------------

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/gollm/GoLLMController.java

@@ -106,7 +106,7 @@ public ResponseEntity<TaskResponse> createModelCardTask(
  @RequestParam(name = "document-id", required = true) final UUID documentId,
  @RequestParam(name = "mode", required = false, defaultValue = "ASYNC") final TaskMode mode,
  @RequestParam(name = "project-id", required = false) final UUID projectId) {
- Schema.Permission permission =
+ final Schema.Permission permission =
  projectService.checkPermissionCanRead(currentUserService.get().getId(), projectId);
 
  try {
@@ -184,7 +184,7 @@ public ResponseEntity<TaskResponse> createConfigureModelTask(
  @RequestParam(name = "workflow-id", required = false) final UUID workflowId,
  @RequestParam(name = "node-id", required = false) final UUID nodeId,
  @RequestParam(name = "project-id", required = false) final UUID projectId) {
- Schema.Permission permission =
+ final Schema.Permission permission =
  projectService.checkPermissionCanRead(currentUserService.get().getId(), projectId);
 
  try {
@@ -281,7 +281,7 @@ public ResponseEntity<TaskResponse> createConfigFromDatasetTask(
  @RequestParam(name = "node-id", required = false) final UUID nodeId,
  @RequestParam(name = "project-id", required = false) final UUID projectId,
  @RequestBody(required = false) final ConfigFromDatasetBody body) {
- Schema.Permission permission =
+ final Schema.Permission permission =
  projectService.checkPermissionCanRead(currentUserService.get().getId(), projectId);
 
  try {
@@ -393,7 +393,7 @@ public ResponseEntity<TaskResponse> createCompareModelsTask(
  @RequestParam(name = "workflow-id", required = false) final UUID workflowId,
  @RequestParam(name = "node-id", required = false) final UUID nodeId,
  @RequestParam(name = "project-id", required = false) final UUID projectId) {
- Schema.Permission permission =
+ final Schema.Permission permission =
  projectService.checkPermissionCanWrite(currentUserService.get().getId(), projectId);
  try {
  final List<String> modelCards = new ArrayList<>();

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/TerariumAsset.java

@@ -58,7 +58,8 @@ public TerariumAsset clone() {
 
  protected TerariumAsset cloneSuperFields(final TerariumAsset asset) {
 
- // TODO this should be a part of the clone method, and this should implement Cloneable
+ // TODO this should be a part of the clone method, and this should implement
+ // Cloneable
 
  super.cloneSuperFields(asset);
 

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/AssetExport.java

@@ -1,5 +1,6 @@
 package software.uncharted.terarium.hmiserver.models.dataservice;
 
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import java.util.HashMap;
 import java.util.Map;
 import lombok.Data;
@@ -8,6 +9,7 @@
 
 @Data
 @Accessors(chain = true)
+@JsonDeserialize(using = AssetExportDeserializer.class)
 public class AssetExport {
  AssetType type;
  TerariumAsset asset;

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/AssetExportDeserializer.java

@@ -0,0 +1,37 @@
+package software.uncharted.terarium.hmiserver.models.dataservice;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import software.uncharted.terarium.hmiserver.models.TerariumAsset;
+
+public class AssetExportDeserializer extends JsonDeserializer<AssetExport> {
+
+ @Override
+ public AssetExport deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException {
+ ObjectMapper mapper = (ObjectMapper) jp.getCodec();
+ JsonNode node = mapper.readTree(jp);
+
+ final String assetTypeStr = node.get("type").asText();
+ final AssetType assetType = AssetType.getAssetType(assetTypeStr, mapper);
+
+ TerariumAsset asset = mapper.treeToValue(node.get("asset"), assetType.getAssetClass());
+
+ Map<String, FileExport> files = new HashMap<>();
+ if (node.has("files")) {
+ files = mapper.convertValue(node.get("files"), new TypeReference<Map<String, FileExport>>() {});
+ }
+
+ AssetExport export = new AssetExport();
+ export.setType(assetType);
+ export.setAsset(asset);
+ export.setFiles(files);
+ return export;
+ }
+}

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/AssetType.java

@@ -7,6 +7,14 @@
 import org.springframework.http.HttpStatus;
 import org.springframework.web.server.ResponseStatusException;
 import software.uncharted.terarium.hmiserver.annotations.TSModel;
+import software.uncharted.terarium.hmiserver.models.TerariumAsset;
+import software.uncharted.terarium.hmiserver.models.dataservice.code.Code;
+import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset;
+import software.uncharted.terarium.hmiserver.models.dataservice.document.DocumentAsset;
+import software.uncharted.terarium.hmiserver.models.dataservice.model.Model;
+import software.uncharted.terarium.hmiserver.models.dataservice.model.ModelConfiguration;
+import software.uncharted.terarium.hmiserver.models.dataservice.simulation.Simulation;
+import software.uncharted.terarium.hmiserver.models.dataservice.workflow.Workflow;
 
 @RequiredArgsConstructor
 @TSModel
@@ -49,4 +57,29 @@ public static AssetType getAssetType(final String assetTypeName, final ObjectMap
  HttpStatus.BAD_REQUEST, "Failed to convert an AssetTypeName into an AssetType");
  }
  }
+
+ public Class<? extends TerariumAsset> getAssetClass() {
+ switch (this) {
+ case ARTIFACT:
+ return Artifact.class;
+ case CODE:
+ return Code.class;
+ case DATASET:
+ return Dataset.class;
+ case DOCUMENT:
+ return DocumentAsset.class;
+ case MODEL:
+ return Model.class;
+ case MODEL_CONFIGURATION:
+ return ModelConfiguration.class;
+ case PUBLICATION:
+ throw new IllegalArgumentException("Publication assets are not supported");
+ case SIMULATION:
+ return Simulation.class;
+ case WORKFLOW:
+ return Workflow.class;
+ default:
+ throw new IllegalArgumentException("Unrecognized asset type: " + this);
+ }
+ }
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/ContentTypeDeserializer.java

@@ -0,0 +1,31 @@
+package software.uncharted.terarium.hmiserver.models.dataservice;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import org.apache.http.entity.ContentType;
+
+public class ContentTypeDeserializer extends JsonDeserializer<ContentType> {
+
+ @Override
+ public ContentType deserialize(final JsonParser jp, final DeserializationContext ctxt)
+ throws IOException, JsonProcessingException {
+
+ final ObjectMapper mapper = (ObjectMapper) jp.getCodec();
+ final JsonNode node = mapper.readTree(jp);
+
+ final String mimeTypeStr = node.get("mimeType").asText();
+ if (!node.has("charset")) {
+ return ContentType.create(mimeTypeStr);
+ }
+
+ final String charsetStr = node.get("charset").asText();
+ final Charset charset = Charset.forName(charsetStr);
+ return ContentType.create(mimeTypeStr, charset);
+ }
+}

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/FileExport.java

@@ -1,5 +1,6 @@
 package software.uncharted.terarium.hmiserver.models.dataservice;
 
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import lombok.Data;
 import lombok.experimental.Accessors;
 import org.apache.http.entity.ContentType;
@@ -8,6 +9,8 @@
 @Accessors(chain = true)
 public class FileExport {
 
- byte[] bytes;
+ @JsonDeserialize(using = ContentTypeDeserializer.class)
  ContentType contentType;
+
+ byte[] bytes;
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/ProjectAssetRepository.java

@@ -14,7 +14,7 @@ public interface ProjectAssetRepository extends PSCrudRepository<ProjectAsset, U
 
  ProjectAsset findByProjectIdAndAssetId(@NotNull UUID projectId, @NotNull UUID assetId);
 
- List<ProjectAsset> findAllByProjectId(@NotNull UUID projectId);
+ List<ProjectAsset> findAllByProjectIdAndDeletedOnIsNullAndTemporaryFalse(@NotNull UUID projectId);
 
  List<ProjectAsset> findAllByProjectIdAndAssetTypeInAndDeletedOnIsNullAndTemporaryFalse(
  @NotNull UUID projectId, Collection<@NotNull AssetType> assetType);

packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/TerariumAssetCloneService.java

@@ -174,6 +174,8 @@ public ProjectExport exportProject(final UUID projectId) throws IOException {
 
  final ProjectExport projectExport = new ProjectExport();
  projectExport.setProject(project.clone());
+ projectExport.getProject().setUserId(null); // clear the user id
+ projectExport.getProject().setUserName(null); // clear the user name
  projectExport.setAssets(exportedAssets);
  return projectExport.clone();
  }
@@ -186,10 +188,15 @@ public ProjectExport exportProject(final UUID projectId) throws IOException {
  * @throws IOException
  */
  @SuppressWarnings({"unchecked", "rawtypes"})
- public Project importProject(final ProjectExport export) throws IOException {
+ public Project importProject(final String userId, final String userName, final ProjectExport export)
+ throws IOException {
 
  final ProjectExport projectExport = export.clone(); // clone in case it has been imported already
 
+ // set the current user id
+ projectExport.getProject().setUserId(userId);
+ projectExport.getProject().setUserName(userName);
+
  // create the project
  final Project project = projectService.createProject(projectExport.getProject());
 

packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ProjectAssetService.java

@@ -140,7 +140,7 @@ public Optional<ProjectAsset> getProjectAssetByProjectIdAndAssetId(
 
  @Observed(name = "function_profile")
  public List<ProjectAsset> getProjectAssets(final UUID projectId, final Schema.Permission hasReadPermission) {
- return projectAssetRepository.findAllByProjectId(projectId);
+ return projectAssetRepository.findAllByProjectIdAndDeletedOnIsNullAndTemporaryFalse(projectId);
  }
 
  @Observed(name = "function_profile")

packages/server/src/main/resources/messages.properties

@@ -34,6 +34,7 @@ projects.unable-to-delete = We were unable to delete the project. Please try aga
 projects.unable-to-update = We were unable to update the project. Please try again later.
 projects.unable-to-get-permissions = Terarium was unable to determine read/write permissions. For safety reasons, we are unable to return results. Please try again later.
 projects.name-required = You must supply a name for the project.
+projects.import-parse-failure = We were unable to parse the import file. Please ensure that the file is in the correct format and try again.
 
 rebac.relationship-already-exists = You already have permissions set to access this resource. Please contact the system administrator to upgrade permissions.
 rebac.service-unavailable = The service that Terarium uses for security and authorization is temporarily unavailable. Please try again later.