3346 task migrate dataset to postgres from es

packages/client/hmi-client/src/services/project.ts

@@ -8,14 +8,7 @@ import { b64EncodeUnicode } from '@/utils/binary';
 import DatasetIcon from '@/assets/svg/icons/dataset.svg?component';
 import { Component } from 'vue';
 import * as EventService from '@/services/event';
-import {
-	AssetType,
-	Code,
-	EventType,
-	PermissionRelationships,
-	Project,
-	ProjectAsset
-} from '@/types/Types';
+import { AssetType, EventType, PermissionRelationships, Project } from '@/types/Types';
 
 /**
  * Create a project
@@ -30,14 +23,9 @@ async function create(
 	userId: Project['userId'] = ''
 ): Promise<Project | null> {
 	try {
-		const project: Project = {
-			name,
-			description,
-			userId,
-			projectAssets: [] as ProjectAsset[],
-			codeAssets: [] as Code[]
-		};
-		const response = await API.post(`/projects`, project);
+		const response = await API.post(
+			`/projects?name=${name}&description=${description}&userId=${userId}`
+		);
 		const { status, data } = response;
 		if (status !== 201) return null;
 		return data ?? null;

packages/client/hmi-client/src/types/Types.ts

@@ -165,6 +165,7 @@ export interface Dataset extends TerariumAsset {
     metadata?: any;
     source?: string;
     grounding?: Grounding;
+    project?: Project;
 }
 
 export interface DatasetColumn {
@@ -404,6 +405,7 @@ export interface Project extends TerariumAsset {
      */
     projectAssets: ProjectAsset[];
     codeAssets: Code[];
+    datasetAssets: Dataset[];
     metadata?: { [index: string]: string };
     publicProject?: boolean;
     userPermission?: string;

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java

@@ -148,7 +148,7 @@ public ResponseEntity<List<Dataset>> getDatasets(
 			if (query == null) {
 				return ResponseEntity.ok(datasetService.getAssets(page, pageSize));
 			} else {
-				return ResponseEntity.ok(datasetService.getAssets(page, pageSize, query));
+				return ResponseEntity.ok(datasetService.searchAssets(page, pageSize, query));
 			}
 
 		} catch (final IOException e) {
@@ -221,7 +221,7 @@ public ResponseEntity<Dataset> getDataset(@PathVariable("id") final UUID id) {
 			}
 			return dataset.map(ResponseEntity::ok)
 					.orElseGet(() -> ResponseEntity.notFound().build());
-		} catch (final IOException e) {
+		} catch (final Exception e) {
 			final String error = "Unable to get dataset";
 			log.error(error, e);
 			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);
@@ -476,7 +476,7 @@ public ResponseEntity<PresignedURL> getDownloadURL(
 			if (dataset.isEmpty()) {
 				throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, "Dataset not found");
 			}
-		} catch (final IOException e) {
+		} catch (final Exception e) {
 			final String error = "Unable to get dataset";
 			log.error(error, e);
 			throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error);

packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java

@@ -36,6 +36,7 @@
 import software.uncharted.terarium.hmiserver.models.dataservice.AssetType;
 import software.uncharted.terarium.hmiserver.models.dataservice.ResponseDeleted;
 import software.uncharted.terarium.hmiserver.models.dataservice.code.Code;
+import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset;
 import software.uncharted.terarium.hmiserver.models.dataservice.project.Project;
 import software.uncharted.terarium.hmiserver.models.dataservice.project.ProjectAsset;
 import software.uncharted.terarium.hmiserver.models.permissions.PermissionGroup;
@@ -45,6 +46,7 @@
 import software.uncharted.terarium.hmiserver.service.CurrentUserService;
 import software.uncharted.terarium.hmiserver.service.UserService;
 import software.uncharted.terarium.hmiserver.service.data.CodeService;
+import software.uncharted.terarium.hmiserver.service.data.DatasetService;
 import software.uncharted.terarium.hmiserver.service.data.ITerariumAssetService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectAssetService;
 import software.uncharted.terarium.hmiserver.service.data.ProjectService;
@@ -78,10 +80,29 @@ public class ProjectController {
 
 	final CodeService codeService;
 
+	final DatasetService datasetService;
+
 	final UserService userService;
 
 	final ObjectMapper objectMapper;
 
+	static final String WELCOME_MESSAGE =
+			"""
+		<div>
+			<h2>Hey there!</h2>
+			<p>This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:</p>
+			<br>
+				<ul>
+					<li><strong>Upload stuff:</strong> Upload documents, models, code or datasets with the green button in the bottom left corner.</li>
+					<li><strong>Explore and add:</strong> Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.</li>
+					<li><strong>Build a model:</strong> Create a model that fits just what you need.</li>
+					<li><strong>Create a workflow:</strong> Connect resources with operators so you can focus on the science and not the plumbing.</li>
+				</ul>
+			<br>
+			<p>Feel free to erase this text and make it your own.</p>
+		</div>
+		""";
+
 	// --------------------------------------------------------------------------
 	// Basic Project Operations
 	// --------------------------------------------------------------------------
@@ -360,26 +381,15 @@ public ResponseEntity<ResponseDeleted> deleteProject(@PathVariable("id") final U
 			})
 	@PostMapping
 	@Secured(Roles.USER)
-	public ResponseEntity<Project> createProject(@RequestBody Project project) {
-		if (project.getOverviewContent() == null) {
-			final String welcomeMessage =
-					"""
-					<div>
-						<h2>Hey there!</h2>
-						<p>This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:</p>
-						<br>
-							<ul>
-								<li><strong>Upload stuff:</strong> Upload documents, models, code or datasets with the green button in the bottom left corner.</li>
-								<li><strong>Explore and add:</strong> Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.</li>
-								<li><strong>Build a model:</strong> Create a model that fits just what you need.</li>
-								<li><strong>Create a workflow:</strong> Connect resources with operators so you can focus on the science and not the plumbing.</li>
-							</ul>
-						<br>
-						<p>Feel free to erase this text and make it your own.</p>
-					</div>
-					""";
-			project.setOverviewContent(welcomeMessage.getBytes());
-		}
+	public ResponseEntity<Project> createProject(
+			@RequestParam("name") final String name,
+			@RequestParam("description") final String description,
+			@RequestParam("userId") final String userId) {
+		Project project = (Project)
+				new Project().setUserId(userId).setDescription(description).setName(name);
+
+		project.setOverviewContent(WELCOME_MESSAGE.getBytes());
+
 		project = projectService.createProject(project);
 
 		try {
@@ -507,6 +517,22 @@ public ResponseEntity<ProjectAsset> createAsset(
 
 						code.get().setProject(project.get());
 						codeService.updateAsset(code.get());
+					} else if (assetType.equals(AssetType.DATASET)) {
+
+						final Optional<Dataset> dataset = datasetService.getAsset(assetId);
+						if (dataset.isEmpty()) {
+							throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Dataset Asset does not exist");
+						}
+
+						if (project.get().getDatasetAssets() == null)
+							project.get().setDatasetAssets(new ArrayList<>());
+						if (project.get().getDatasetAssets().contains(dataset.get())) {
+							throw new ResponseStatusException(
+									HttpStatus.CONFLICT, "Dataset Asset already exists on project");
+						}
+
+						dataset.get().setProject(project.get());
+						datasetService.updateAsset(dataset.get());
 					}
 
 					// double check that this asset is not already a part of this project, and if it
@@ -570,20 +596,26 @@ public ResponseEntity<ResponseDeleted> deleteAsset(
 			final RebacProject rebacProject = new RebacProject(projectId, reBACService);
 			if (rebacUser.canWrite(rebacProject)) {
 
+				/* TODO: 	At the end of the Postgres migration we will be getting rid of ProjectAsset and instead
+										projects will directly hold a reference to the assets associated with them.  During this
+										transition we need to properly create the relationships when users add assets to their
+										projects. However the exact API may not look like this in the end, and in fact may be
+										directly in the controllers for these assets and not in this ProjectController
+				*/
 				if (assetType.equals(AssetType.CODE)) {
 
-					/* TODO: 	At the end of the Postgres migration we will be getting rid of ProjectAsset and instead
-											projects will directly hold a reference to the assets associated with them.  During this
-											transition we need to properly create the relationships when users add assets to their
-											projects. However the exact API may not look like this in the end, and in fact may be
-											directly in the controllers for these assets and not in this ProjectController
-					*/
-
 					final Optional<Code> deletedCode = codeService.deleteAsset(assetId);
 					if (deletedCode.isEmpty() || deletedCode.get().getDeletedOn() == null) {
 						throw new ResponseStatusException(
 								HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete code asset");
 					}
+				} else if (assetType.equals(AssetType.DATASET)) {
+
+					final Optional<Dataset> deletedDataset = datasetService.deleteAsset(assetId);
+					if (deletedDataset.isEmpty() || deletedDataset.get().getDeletedOn() == null) {
+						throw new ResponseStatusException(
+								HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete dataset asset");
+					}
 				}
 
 				final boolean deleted = projectAssetService.deleteByAssetId(projectId, assetType, assetId);

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java

@@ -2,10 +2,14 @@
 
 import java.io.Serial;
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import lombok.Data;
 import lombok.experimental.Accessors;
+import org.hibernate.annotations.JdbcTypeCode;
+import org.hibernate.type.SqlTypes;
 import software.uncharted.terarium.hmiserver.annotations.TSModel;
 import software.uncharted.terarium.hmiserver.annotations.TSOptional;
 
@@ -19,9 +23,29 @@ public class Grounding implements Serializable {
 	private static final long serialVersionUID = 302308407252037615L;
 
 	/** Ontological identifier per DKG */
+	@JdbcTypeCode(SqlTypes.JSON)
 	private List<Identifier> identifiers;
 
 	/** (Optional) Additional context that informs the grounding */
 	@TSOptional
+	@JdbcTypeCode(SqlTypes.JSON)
 	private Map<String, Object> context;
+
+	@Override
+	public Grounding clone() {
+
+		final Grounding clone = new Grounding();
+		if (this.identifiers != null) {
+			clone.identifiers = new ArrayList<>();
+			clone.identifiers.addAll(this.identifiers);
+		}
+		if (this.context != null) {
+			clone.context = new HashMap<>();
+			for (final String key : this.context.keySet()) {
+				clone.context.put(key, context.get(key));
+			}
+		}
+
+		return clone;
+	}
 }

packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java

@@ -1,70 +1,129 @@
 package software.uncharted.terarium.hmiserver.models.dataservice.dataset;
 
 import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.databind.JsonNode;
+import jakarta.persistence.Column;
+import jakarta.persistence.ElementCollection;
+import jakarta.persistence.Entity;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
 import java.io.Serial;
 import java.sql.Timestamp;
+import java.util.ArrayList;
 import java.util.List;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.Accessors;
+import org.hibernate.annotations.JdbcTypeCode;
+import org.hibernate.type.SqlTypes;
 import software.uncharted.terarium.hmiserver.annotations.TSModel;
 import software.uncharted.terarium.hmiserver.annotations.TSOptional;
 import software.uncharted.terarium.hmiserver.models.TerariumAsset;
 import software.uncharted.terarium.hmiserver.models.dataservice.Grounding;
+import software.uncharted.terarium.hmiserver.models.dataservice.project.Project;
 
 /** Represents a dataset document from TDS */
 @EqualsAndHashCode(callSuper = true)
 @Data
 @Accessors(chain = true)
 @TSModel
+@Entity
 public class Dataset extends TerariumAsset {
 
 	@Serial
 	private static final long serialVersionUID = 6927286281160755696L;
+
 	/** UserId of the user who created the dataset */
 	@TSOptional
+	@Column(length = 255)
 	private String userId;
 
 	/** ESGF id of the dataset. This will be null for datasets that are not from ESGF */
 	@TSOptional
+	@Column(length = 255)
 	private String esgfId;
 
 	/** (Optional) data source date */
 	@TSOptional
 	@JsonAlias("data_source_date")
+	@Column(columnDefinition = "TIMESTAMP WITH TIME ZONE")
 	private Timestamp dataSourceDate;
 
 	/** (Optional) list of file names associated with the dataset */
 	@TSOptional
 	@JsonAlias("file_names")
+	@ElementCollection
+	@Column(length = 1024)
 	private List<String> fileNames;
 
-	/**
-	 * (Optional) Url from which the dataset can be downloaded/fetched TODO: IS THIS NEEDED? IS THIS FROM OLD TDS?
-	 * https://github.com/DARPA-ASKEM/terarium/issues/3194
-	 */
+	@ManyToOne
+	@JoinColumn(name = "project_id")
+	@JsonBackReference
+	@TSOptional
+	private Project project;
+
 	@TSOptional
 	@JsonAlias("dataset_url")
+	@Column(length = 1024)
 	private String datasetUrl;
 
 	/** (Optional) List of urls from which the dataset can be downloaded/fetched. Used for ESGF datasets */
 	@TSOptional
+	@Column(length = 1024)
+	@ElementCollection
 	private List<String> datasetUrls;
 
 	/** Information regarding the columns that make up the dataset */
 	@TSOptional
+	@JdbcTypeCode(SqlTypes.JSON)
 	private List<DatasetColumn> columns;
 
 	/** (Optional) Unformatted metadata about the dataset */
 	@TSOptional
+	@JdbcTypeCode(SqlTypes.JSON)
+	@Column(columnDefinition = "text")
 	private JsonNode metadata;
 
 	/** (Optional) Source of dataset */
 	@TSOptional
+	@Column(columnDefinition = "text")
 	private String source;
 
 	/** (Optional) Grounding of ontological concepts related to the dataset as a whole */
 	@TSOptional
+	@JdbcTypeCode(SqlTypes.JSON)
 	private Grounding grounding;
+
+	@Override
+	public Dataset clone() {
+		final Dataset clone = new Dataset();
+		super.cloneSuperFields(clone);
+
+		clone.userId = this.userId;
+		clone.esgfId = this.esgfId;
+		clone.dataSourceDate = this.dataSourceDate;
+		if (fileNames != null) {
+			clone.fileNames = new ArrayList<>();
+			clone.fileNames.addAll(fileNames);
+		}
+		clone.datasetUrl = this.datasetUrl;
+		if (datasetUrls != null) {
+			clone.datasetUrls = new ArrayList<>();
+			clone.datasetUrls.addAll(datasetUrls);
+		}
+
+		if (columns != null) {
+			clone.columns = new ArrayList<>();
+			for (final DatasetColumn column : columns) {
+				clone.columns.add(column.clone());
+			}
+		}
+
+		if (this.metadata != null) clone.metadata = this.metadata.deepCopy();
+		clone.source = this.source;
+		if (this.grounding != null) clone.grounding = this.grounding.clone();
+
+		return clone;
+	}
 }