Revert "3346 task migrate dataset to postgres from es (#3518)" (#3544)

Co-authored-by: dgauldie <[email protected]>

packages/client/hmi-client/src/services/project.ts

@@ -8,7 +8,14 @@ import { b64EncodeUnicode } from '@/utils/binary';
 import DatasetIcon from '@/assets/svg/icons/dataset.svg?component';
 import { Component } from 'vue';
 import * as EventService from '@/services/event';
-import { AssetType, EventType, PermissionRelationships, Project } from '@/types/Types';
+import {
+ AssetType,
+ Code,
+ EventType,
+ PermissionRelationships,
+ Project,
+ ProjectAsset
+} from '@/types/Types';
 
 /**
  * Create a project
@@ -23,9 +30,14 @@ async function create(
  userId: Project['userId'] = ''
 ): Promise<Project | null> {
  try {
- const response = await API.post(
- `/projects?name=${name}&description=${description}&userId=${userId}`
- );
+ const project: Project = {
+ name,
+ description,
+ userId,
+ projectAssets: [] as ProjectAsset[],
+ codeAssets: [] as Code[]
+ };
+ const response = await API.post(`/projects`, project);
  const { status, data } = response;
  if (status !== 201) return null;
  return data ?? null;

packages/client/hmi-client/src/types/Types.ts

@@ -165,7 +165,6 @@ export interface Dataset extends TerariumAsset {
  metadata?: any;
  source?: string;
  grounding?: Grounding;
- project?: Project;
 }
 
 export interface DatasetColumn {
@@ -405,7 +404,6 @@ export interface Project extends TerariumAsset {
  */
  projectAssets: ProjectAsset[];
  codeAssets: Code[];
- datasetAssets: Dataset[];
  metadata?: { [index: string]: string };
  publicProject?: boolean;
  userPermission?: string;

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java

@@ -148,7 +148,7 @@ public ResponseEntity<List<Dataset>> getDatasets(
  if (query == null) {
  return ResponseEntity.ok(datasetService.getAssets(page, pageSize));
  } else {
- return ResponseEntity.ok(datasetService.searchAssets(page, pageSize, query));
+ return ResponseEntity.ok(datasetService.getAssets(page, pageSize, query));
  }
 
  } catch (final IOException e) {
@@ -221,7 +221,7 @@ public ResponseEntity<Dataset> getDataset(@PathVariable("id") final UUID id) {
  }
  return dataset.map(ResponseEntity::ok)
  .orElseGet(() -> ResponseEntity.notFound().build());
- } catch (final Exception e) {
+ } catch (final IOException e) {
  final String error = "Unable to get dataset";
  log.error(error, e);
  throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
@@ -476,7 +476,7 @@ public ResponseEntity<PresignedURL> getDownloadURL(
  if (dataset.isEmpty()) {
  throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, "Dataset not found");
  }
- } catch (final Exception e) {
+ } catch (final IOException e) {
  final String error = "Unable to get dataset";
  log.error(error, e);
  throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java

@@ -36,7 +36,6 @@
 import software.uncharted.terarium.hmiserver.models.dataservice.AssetType;
 import software.uncharted.terarium.hmiserver.models.dataservice.ResponseDeleted;
 import software.uncharted.terarium.hmiserver.models.dataservice.code.Code;
-import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset;
 import software.uncharted.terarium.hmiserver.models.dataservice.project.Project;
 import software.uncharted.terarium.hmiserver.models.dataservice.project.ProjectAsset;
 import software.uncharted.terarium.hmiserver.models.permissions.PermissionGroup;
@@ -46,7 +45,6 @@
 import software.uncharted.terarium.hmiserver.service.CurrentUserService;
 import software.uncharted.terarium.hmiserver.service.UserService;
 import software.uncharted.terarium.hmiserver.service.data.CodeService;
-import software.uncharted.terarium.hmiserver.service.data.DatasetService;
 import software.uncharted.terarium.hmiserver.service.data.ITerariumAssetService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectAssetService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectService;
@@ -80,29 +78,10 @@ public class ProjectController {
 
  final CodeService codeService;
 
- final DatasetService datasetService;
-
  final UserService userService;
 
  final ObjectMapper objectMapper;
 
- static final String WELCOME_MESSAGE =
- """
- <div>
- <h2>Hey there!</h2>
- <p>This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:</p>
- <br>
- <ul>
- <li><strong>Upload stuff:</strong> Upload documents, models, code or datasets with the green button in the bottom left corner.</li>
- <li><strong>Explore and add:</strong> Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.</li>
- <li><strong>Build a model:</strong> Create a model that fits just what you need.</li>
- <li><strong>Create a workflow:</strong> Connect resources with operators so you can focus on the science and not the plumbing.</li>
- </ul>
- <br>
- <p>Feel free to erase this text and make it your own.</p>
- </div>
- """;
-
  // --------------------------------------------------------------------------
  // Basic Project Operations
  // --------------------------------------------------------------------------
@@ -381,15 +360,26 @@ public ResponseEntity<ResponseDeleted> deleteProject(@PathVariable("id") final U
  })
  @PostMapping
  @Secured(Roles.USER)
- public ResponseEntity<Project> createProject(
- @RequestParam("name") final String name,
- @RequestParam("description") final String description,
- @RequestParam("userId") final String userId) {
- Project project = (Project)
- new Project().setUserId(userId).setDescription(description).setName(name);
-
- project.setOverviewContent(WELCOME_MESSAGE.getBytes());
-
+ public ResponseEntity<Project> createProject(@RequestBody Project project) {
+ if (project.getOverviewContent() == null) {
+ final String welcomeMessage =
+ """
+ <div>
+ <h2>Hey there!</h2>
+ <p>This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:</p>
+ <br>
+ <ul>
+ <li><strong>Upload stuff:</strong> Upload documents, models, code or datasets with the green button in the bottom left corner.</li>
+ <li><strong>Explore and add:</strong> Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.</li>
+ <li><strong>Build a model:</strong> Create a model that fits just what you need.</li>
+ <li><strong>Create a workflow:</strong> Connect resources with operators so you can focus on the science and not the plumbing.</li>
+ </ul>
+ <br>
+ <p>Feel free to erase this text and make it your own.</p>
+ </div>
+ """;
+ project.setOverviewContent(welcomeMessage.getBytes());
+ }
  project = projectService.createProject(project);
 
  try {
@@ -517,22 +507,6 @@ public ResponseEntity<ProjectAsset> createAsset(
 
  code.get().setProject(project.get());
  codeService.updateAsset(code.get());
- } else if (assetType.equals(AssetType.DATASET)) {
-
- final Optional<Dataset> dataset = datasetService.getAsset(assetId);
- if (dataset.isEmpty()) {
- throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Dataset Asset does not exist");
- }
-
- if (project.get().getDatasetAssets() == null)
- project.get().setDatasetAssets(new ArrayList<>());
- if (project.get().getDatasetAssets().contains(dataset.get())) {
- throw new ResponseStatusException(
- HttpStatus.CONFLICT, "Dataset Asset already exists on project");
- }
-
- dataset.get().setProject(project.get());
- datasetService.updateAsset(dataset.get());
  }
 
  // double check that this asset is not already a part of this project, and if it
@@ -596,26 +570,20 @@ public ResponseEntity<ResponseDeleted> deleteAsset(
  final RebacProject rebacProject = new RebacProject(projectId, reBACService);
  if (rebacUser.canWrite(rebacProject)) {
 
- /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead
- projects will directly hold a reference to the assets associated with them. During this
- transition we need to properly create the relationships when users add assets to their
- projects. However the exact API may not look like this in the end, and in fact may be
- directly in the controllers for these assets and not in this ProjectController
- */
  if (assetType.equals(AssetType.CODE)) {
 
+ /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead
+ projects will directly hold a reference to the assets associated with them. During this
+ transition we need to properly create the relationships when users add assets to their
+ projects. However the exact API may not look like this in the end, and in fact may be
+ directly in the controllers for these assets and not in this ProjectController
+ */
+
  final Optional<Code> deletedCode = codeService.deleteAsset(assetId);
  if (deletedCode.isEmpty() || deletedCode.get().getDeletedOn() == null) {
  throw new ResponseStatusException(
  HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete code asset");
  }
- } else if (assetType.equals(AssetType.DATASET)) {
-
- final Optional<Dataset> deletedDataset = datasetService.deleteAsset(assetId);
- if (deletedDataset.isEmpty() || deletedDataset.get().getDeletedOn() == null) {
- throw new ResponseStatusException(
- HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete dataset asset");
- }
  }
 
  final boolean deleted = projectAssetService.deleteByAssetId(projectId, assetType, assetId);

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java

@@ -2,14 +2,10 @@
 
 import java.io.Serial;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import lombok.Data;
 import lombok.experimental.Accessors;
-import org.hibernate.annotations.JdbcTypeCode;
-import org.hibernate.type.SqlTypes;
 import software.uncharted.terarium.hmiserver.annotations.TSModel;
 import software.uncharted.terarium.hmiserver.annotations.TSOptional;
 
@@ -23,29 +19,9 @@ public class Grounding implements Serializable {
  private static final long serialVersionUID = 302308407252037615L;
 
  /** Ontological identifier per DKG */
- @JdbcTypeCode(SqlTypes.JSON)
  private List<Identifier> identifiers;
 
  /** (Optional) Additional context that informs the grounding */
  @TSOptional
- @JdbcTypeCode(SqlTypes.JSON)
  private Map<String, Object> context;
-
- @Override
- public Grounding clone() {
-
- final Grounding clone = new Grounding();
- if (this.identifiers != null) {
- clone.identifiers = new ArrayList<>();
- clone.identifiers.addAll(this.identifiers);
- }
- if (this.context != null) {
- clone.context = new HashMap<>();
- for (final String key : this.context.keySet()) {
- clone.context.put(key, context.get(key));
- }
- }
-
- return clone;
- }
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java

@@ -1,17 +1,9 @@
 package software.uncharted.terarium.hmiserver.models.dataservice.dataset;
 
 import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.databind.JsonNode;
-import jakarta.persistence.Column;
-import jakarta.persistence.Convert;
-import jakarta.persistence.ElementCollection;
-import jakarta.persistence.Entity;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
 import java.io.Serial;
 import java.sql.Timestamp;
-import java.util.ArrayList;
 import java.util.List;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
@@ -20,113 +12,59 @@
 import software.uncharted.terarium.hmiserver.annotations.TSOptional;
 import software.uncharted.terarium.hmiserver.models.TerariumAsset;
 import software.uncharted.terarium.hmiserver.models.dataservice.Grounding;
-import software.uncharted.terarium.hmiserver.models.dataservice.JsonConverter;
-import software.uncharted.terarium.hmiserver.models.dataservice.ObjectConverter;
-import software.uncharted.terarium.hmiserver.models.dataservice.project.Project;
 
 /** Represents a dataset document from TDS */
 @EqualsAndHashCode(callSuper = true)
 @Data
 @Accessors(chain = true)
 @TSModel
-@Entity
 public class Dataset extends TerariumAsset {
 
  @Serial
  private static final long serialVersionUID = 6927286281160755696L;
-
  /** UserId of the user who created the dataset */
  @TSOptional
- @Column(length = 255)
  private String userId;
 
  /** ESGF id of the dataset. This will be null for datasets that are not from ESGF */
  @TSOptional
- @Column(length = 255)
  private String esgfId;
 
  /** (Optional) data source date */
  @TSOptional
  @JsonAlias("data_source_date")
- @Column(columnDefinition = "TIMESTAMP WITH TIME ZONE")
  private Timestamp dataSourceDate;
 
  /** (Optional) list of file names associated with the dataset */
  @TSOptional
  @JsonAlias("file_names")
- @ElementCollection
- @Column(length = 1024)
  private List<String> fileNames;
 
- @ManyToOne
- @JoinColumn(name = "project_id")
- @JsonBackReference
- @TSOptional
- private Project project;
-
+ /**
+ * (Optional) Url from which the dataset can be downloaded/fetched TODO: IS THIS NEEDED? IS THIS FROM OLD TDS?
+ * https://github.com/DARPA-ASKEM/terarium/issues/3194
+ */
  @TSOptional
  @JsonAlias("dataset_url")
- @Column(length = 1024)
  private String datasetUrl;
 
  /** (Optional) List of urls from which the dataset can be downloaded/fetched. Used for ESGF datasets */
  @TSOptional
- @Column(length = 1024)
- @ElementCollection
  private List<String> datasetUrls;
 
  /** Information regarding the columns that make up the dataset */
  @TSOptional
- @Convert(converter = ObjectConverter.class)
- @Column(columnDefinition = "text")
  private List<DatasetColumn> columns;
 
  /** (Optional) Unformatted metadata about the dataset */
  @TSOptional
- @Convert(converter = JsonConverter.class)
- @Column(columnDefinition = "text")
  private JsonNode metadata;
 
  /** (Optional) Source of dataset */
  @TSOptional
- @Column(columnDefinition = "text")
  private String source;
 
  /** (Optional) Grounding of ontological concepts related to the dataset as a whole */
  @TSOptional
- @Convert(converter = ObjectConverter.class)
- @Column(columnDefinition = "text")
  private Grounding grounding;
-
- @Override
- public Dataset clone() {
- final Dataset clone = new Dataset();
- super.cloneSuperFields(clone);
-
- clone.userId = this.userId;
- clone.esgfId = this.esgfId;
- clone.dataSourceDate = this.dataSourceDate;
- if (fileNames != null) {
- clone.fileNames = new ArrayList<>();
- clone.fileNames.addAll(fileNames);
- }
- clone.datasetUrl = this.datasetUrl;
- if (datasetUrls != null) {
- clone.datasetUrls = new ArrayList<>();
- clone.datasetUrls.addAll(datasetUrls);
- }
-
- if (columns != null) {
- clone.columns = new ArrayList<>();
- for (final DatasetColumn column : columns) {
- clone.columns.add(column.clone());
- }
- }
-
- if (this.metadata != null) clone.metadata = this.metadata.deepCopy();
- clone.source = this.source;
- if (this.grounding != null) clone.grounding = this.grounding.clone();
-
- return clone;
- }
 }