diff --git a/packages/client/hmi-client/src/services/project.ts b/packages/client/hmi-client/src/services/project.ts index e192d4b0cb..35f39e1474 100644 --- a/packages/client/hmi-client/src/services/project.ts +++ b/packages/client/hmi-client/src/services/project.ts @@ -8,14 +8,7 @@ import { b64EncodeUnicode } from '@/utils/binary'; import DatasetIcon from '@/assets/svg/icons/dataset.svg?component'; import { Component } from 'vue'; import * as EventService from '@/services/event'; -import { - AssetType, - Code, - EventType, - PermissionRelationships, - Project, - ProjectAsset -} from '@/types/Types'; +import { AssetType, EventType, PermissionRelationships, Project } from '@/types/Types'; /** * Create a project @@ -30,14 +23,9 @@ async function create( userId: Project['userId'] = '' ): Promise { try { - const project: Project = { - name, - description, - userId, - projectAssets: [] as ProjectAsset[], - codeAssets: [] as Code[] - }; - const response = await API.post(`/projects`, project); + const response = await API.post( + `/projects?name=${name}&description=${description}&userId=${userId}` + ); const { status, data } = response; if (status !== 201) return null; return data ?? null; diff --git a/packages/client/hmi-client/src/types/Types.ts b/packages/client/hmi-client/src/types/Types.ts index 4e77eff771..66c062a9fb 100644 --- a/packages/client/hmi-client/src/types/Types.ts +++ b/packages/client/hmi-client/src/types/Types.ts @@ -165,6 +165,7 @@ export interface Dataset extends TerariumAsset { metadata?: any; source?: string; grounding?: Grounding; + project?: Project; } export interface DatasetColumn { @@ -404,6 +405,7 @@ export interface Project extends TerariumAsset { */ projectAssets: ProjectAsset[]; codeAssets: Code[]; + datasetAssets: Dataset[]; metadata?: { [index: string]: string }; publicProject?: boolean; userPermission?: string; diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java index 7342250487..356ba3d2c2 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java @@ -148,7 +148,7 @@ public ResponseEntity> getDatasets( if (query == null) { return ResponseEntity.ok(datasetService.getAssets(page, pageSize)); } else { - return ResponseEntity.ok(datasetService.getAssets(page, pageSize, query)); + return ResponseEntity.ok(datasetService.searchAssets(page, pageSize, query)); } } catch (final IOException e) { @@ -221,7 +221,7 @@ public ResponseEntity getDataset(@PathVariable("id") final UUID id) { } return dataset.map(ResponseEntity::ok) .orElseGet(() -> ResponseEntity.notFound().build()); - } catch (final IOException e) { + } catch (final Exception e) { final String error = "Unable to get dataset"; log.error(error, e); throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); @@ -476,7 +476,7 @@ public ResponseEntity getDownloadURL( if (dataset.isEmpty()) { throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, "Dataset not found"); } - } catch (final IOException e) { + } catch (final Exception e) { final String error = "Unable to get dataset"; log.error(error, e); throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java index 9de7226258..4f2a84fe0f 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java @@ -36,6 +36,7 @@ import software.uncharted.terarium.hmiserver.models.dataservice.AssetType; import software.uncharted.terarium.hmiserver.models.dataservice.ResponseDeleted; import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; +import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; import software.uncharted.terarium.hmiserver.models.dataservice.project.Project; import software.uncharted.terarium.hmiserver.models.dataservice.project.ProjectAsset; import software.uncharted.terarium.hmiserver.models.permissions.PermissionGroup; @@ -45,6 +46,7 @@ import software.uncharted.terarium.hmiserver.service.CurrentUserService; import software.uncharted.terarium.hmiserver.service.UserService; import software.uncharted.terarium.hmiserver.service.data.CodeService; +import software.uncharted.terarium.hmiserver.service.data.DatasetService; import software.uncharted.terarium.hmiserver.service.data.ITerariumAssetService; import software.uncharted.terarium.hmiserver.service.data.ProjectAssetService; import software.uncharted.terarium.hmiserver.service.data.ProjectService; @@ -78,10 +80,29 @@ public class ProjectController { final CodeService codeService; + final DatasetService datasetService; + final UserService userService; final ObjectMapper objectMapper; + static final String WELCOME_MESSAGE = + """ +
+

Hey there!

+

This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:

+
+
    +
  • Upload stuff: Upload documents, models, code or datasets with the green button in the bottom left corner.
  • +
  • Explore and add: Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.
  • +
  • Build a model: Create a model that fits just what you need.
  • +
  • Create a workflow: Connect resources with operators so you can focus on the science and not the plumbing.
  • +
+
+

Feel free to erase this text and make it your own.

+
+ """; + // -------------------------------------------------------------------------- // Basic Project Operations // -------------------------------------------------------------------------- @@ -360,26 +381,15 @@ public ResponseEntity deleteProject(@PathVariable("id") final U }) @PostMapping @Secured(Roles.USER) - public ResponseEntity createProject(@RequestBody Project project) { - if (project.getOverviewContent() == null) { - final String welcomeMessage = - """ -
-

Hey there!

-

This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:

-
-
    -
  • Upload stuff: Upload documents, models, code or datasets with the green button in the bottom left corner.
  • -
  • Explore and add: Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.
  • -
  • Build a model: Create a model that fits just what you need.
  • -
  • Create a workflow: Connect resources with operators so you can focus on the science and not the plumbing.
  • -
-
-

Feel free to erase this text and make it your own.

-
- """; - project.setOverviewContent(welcomeMessage.getBytes()); - } + public ResponseEntity createProject( + @RequestParam("name") final String name, + @RequestParam("description") final String description, + @RequestParam("userId") final String userId) { + Project project = (Project) + new Project().setUserId(userId).setDescription(description).setName(name); + + project.setOverviewContent(WELCOME_MESSAGE.getBytes()); + project = projectService.createProject(project); try { @@ -507,6 +517,22 @@ public ResponseEntity createAsset( code.get().setProject(project.get()); codeService.updateAsset(code.get()); + } else if (assetType.equals(AssetType.DATASET)) { + + final Optional dataset = datasetService.getAsset(assetId); + if (dataset.isEmpty()) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Dataset Asset does not exist"); + } + + if (project.get().getDatasetAssets() == null) + project.get().setDatasetAssets(new ArrayList<>()); + if (project.get().getDatasetAssets().contains(dataset.get())) { + throw new ResponseStatusException( + HttpStatus.CONFLICT, "Dataset Asset already exists on project"); + } + + dataset.get().setProject(project.get()); + datasetService.updateAsset(dataset.get()); } // double check that this asset is not already a part of this project, and if it @@ -570,20 +596,26 @@ public ResponseEntity deleteAsset( final RebacProject rebacProject = new RebacProject(projectId, reBACService); if (rebacUser.canWrite(rebacProject)) { + /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead + projects will directly hold a reference to the assets associated with them. During this + transition we need to properly create the relationships when users add assets to their + projects. However the exact API may not look like this in the end, and in fact may be + directly in the controllers for these assets and not in this ProjectController + */ if (assetType.equals(AssetType.CODE)) { - /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead - projects will directly hold a reference to the assets associated with them. During this - transition we need to properly create the relationships when users add assets to their - projects. However the exact API may not look like this in the end, and in fact may be - directly in the controllers for these assets and not in this ProjectController - */ - final Optional deletedCode = codeService.deleteAsset(assetId); if (deletedCode.isEmpty() || deletedCode.get().getDeletedOn() == null) { throw new ResponseStatusException( HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete code asset"); } + } else if (assetType.equals(AssetType.DATASET)) { + + final Optional deletedDataset = datasetService.deleteAsset(assetId); + if (deletedDataset.isEmpty() || deletedDataset.get().getDeletedOn() == null) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete dataset asset"); + } } final boolean deleted = projectAssetService.deleteByAssetId(projectId, assetType, assetId); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java index b1784161c7..6065e5f016 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java @@ -2,10 +2,14 @@ import java.io.Serial; import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import software.uncharted.terarium.hmiserver.annotations.TSModel; import software.uncharted.terarium.hmiserver.annotations.TSOptional; @@ -19,9 +23,29 @@ public class Grounding implements Serializable { private static final long serialVersionUID = 302308407252037615L; /** Ontological identifier per DKG */ + @JdbcTypeCode(SqlTypes.JSON) private List identifiers; /** (Optional) Additional context that informs the grounding */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) private Map context; + + @Override + public Grounding clone() { + + final Grounding clone = new Grounding(); + if (this.identifiers != null) { + clone.identifiers = new ArrayList<>(); + clone.identifiers.addAll(this.identifiers); + } + if (this.context != null) { + clone.context = new HashMap<>(); + for (final String key : this.context.keySet()) { + clone.context.put(key, context.get(key)); + } + } + + return clone; + } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java index a3b53e1ae7..837f3e4df1 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java @@ -1,70 +1,129 @@ package software.uncharted.terarium.hmiserver.models.dataservice.dataset; import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; import java.io.Serial; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import software.uncharted.terarium.hmiserver.annotations.TSModel; import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.TerariumAsset; import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; +import software.uncharted.terarium.hmiserver.models.dataservice.project.Project; /** Represents a dataset document from TDS */ @EqualsAndHashCode(callSuper = true) @Data @Accessors(chain = true) @TSModel +@Entity public class Dataset extends TerariumAsset { @Serial private static final long serialVersionUID = 6927286281160755696L; + /** UserId of the user who created the dataset */ @TSOptional + @Column(length = 255) private String userId; /** ESGF id of the dataset. This will be null for datasets that are not from ESGF */ @TSOptional + @Column(length = 255) private String esgfId; /** (Optional) data source date */ @TSOptional @JsonAlias("data_source_date") + @Column(columnDefinition = "TIMESTAMP WITH TIME ZONE") private Timestamp dataSourceDate; /** (Optional) list of file names associated with the dataset */ @TSOptional @JsonAlias("file_names") + @ElementCollection + @Column(length = 1024) private List fileNames; - /** - * (Optional) Url from which the dataset can be downloaded/fetched TODO: IS THIS NEEDED? IS THIS FROM OLD TDS? - * https://github.com/DARPA-ASKEM/terarium/issues/3194 - */ + @ManyToOne + @JoinColumn(name = "project_id") + @JsonBackReference + @TSOptional + private Project project; + @TSOptional @JsonAlias("dataset_url") + @Column(length = 1024) private String datasetUrl; /** (Optional) List of urls from which the dataset can be downloaded/fetched. Used for ESGF datasets */ @TSOptional + @Column(length = 1024) + @ElementCollection private List datasetUrls; /** Information regarding the columns that make up the dataset */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) private List columns; /** (Optional) Unformatted metadata about the dataset */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) + @Column(columnDefinition = "text") private JsonNode metadata; /** (Optional) Source of dataset */ @TSOptional + @Column(columnDefinition = "text") private String source; /** (Optional) Grounding of ontological concepts related to the dataset as a whole */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) private Grounding grounding; + + @Override + public Dataset clone() { + final Dataset clone = new Dataset(); + super.cloneSuperFields(clone); + + clone.userId = this.userId; + clone.esgfId = this.esgfId; + clone.dataSourceDate = this.dataSourceDate; + if (fileNames != null) { + clone.fileNames = new ArrayList<>(); + clone.fileNames.addAll(fileNames); + } + clone.datasetUrl = this.datasetUrl; + if (datasetUrls != null) { + clone.datasetUrls = new ArrayList<>(); + clone.datasetUrls.addAll(datasetUrls); + } + + if (columns != null) { + clone.columns = new ArrayList<>(); + for (final DatasetColumn column : columns) { + clone.columns.add(column.clone()); + } + } + + if (this.metadata != null) clone.metadata = this.metadata.deepCopy(); + clone.source = this.source; + if (this.grounding != null) clone.grounding = this.grounding.clone(); + + return clone; + } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java index 9b66ce54a3..d95d861a8d 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java @@ -1,10 +1,17 @@ package software.uncharted.terarium.hmiserver.models.dataservice.dataset; import com.fasterxml.jackson.annotation.JsonAlias; +import jakarta.persistence.Column; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import software.uncharted.terarium.hmiserver.annotations.TSModel; import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; @@ -16,34 +23,73 @@ public class DatasetColumn { /** Name of the column */ + @Column(length = 255) private String name; /** * Datatype. One of: unknown, boolean, string, char, integer, int, float, double, timestamp, datetime, date, time */ @JsonAlias("data_type") + @Enumerated(EnumType.STRING) private ColumnType dataType; /** (Optional) String that describes the formatting of the value */ @TSOptional @JsonAlias("format_str") + @Column(length = 255) private String formatStr; /** Column annotations from the MIT data profiling tool */ + @Column(columnDefinition = "text") private List annotations; /** (Optional) Unformatted metadata about the dataset */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) private Map metadata; /** (Optional) Grounding of ontological concepts related to the column */ @TSOptional + @JdbcTypeCode(SqlTypes.JSON) private Grounding grounding; @TSOptional + @Column(columnDefinition = "text") private String description; - enum ColumnType { + public void updateMetadata(final Map metadata) { + if (this.metadata == null) { + this.metadata = metadata; + } else { + this.metadata.putAll(metadata); + } + } + + @Override + public DatasetColumn clone() { + final DatasetColumn clone = new DatasetColumn(); + + clone.name = this.name; + clone.dataType = this.dataType; + clone.formatStr = this.formatStr; + if (this.annotations != null) { + clone.annotations = new ArrayList<>(); + clone.annotations.addAll(this.annotations); + } + + if (this.metadata != null) { + clone.metadata = new HashMap<>(); + clone.metadata.putAll(this.metadata); + } + + if (this.grounding != null) clone.grounding = this.grounding.clone(); + + clone.description = this.description; + + return clone; + } + + public enum ColumnType { @JsonAlias("unknown") UNKNOWN, @JsonAlias("boolean") @@ -69,12 +115,4 @@ enum ColumnType { @JsonAlias("time") TIME } - - public void updateMetadata(final Map metadata) { - if (this.metadata == null) { - this.metadata = metadata; - } else { - this.metadata.putAll(metadata); - } - } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java index b941226793..4680a3c14b 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java @@ -21,6 +21,7 @@ import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.TerariumAsset; import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; +import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; @EqualsAndHashCode(callSuper = true) @Data @@ -65,6 +66,13 @@ public class Project extends TerariumAsset { @JsonManagedReference private List codeAssets = new ArrayList<>(); + @OneToMany(mappedBy = "project") + @Where(clause = "deleted_on IS NULL") + @Schema(accessMode = Schema.AccessMode.READ_ONLY) + @ToString.Exclude + @JsonManagedReference + private List datasetAssets = new ArrayList<>(); + @TSOptional @Transient @Schema(accessMode = Schema.AccessMode.READ_ONLY, defaultValue = "{}") diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java new file mode 100644 index 0000000000..a0b2f89b51 --- /dev/null +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java @@ -0,0 +1,7 @@ +package software.uncharted.terarium.hmiserver.repository.data; + +import java.util.UUID; +import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; +import software.uncharted.terarium.hmiserver.repository.PSCrudSoftDeleteRepository; + +public interface DatasetRepository extends PSCrudSoftDeleteRepository {} diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java index 7ac35464c6..78401fc81b 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java @@ -48,6 +48,7 @@ public class DataMigrationESToPG { private final WorkflowService workflowService; private final SimulationService simulationService; private final CodeService codeService; + private final DatasetService datasetService; @PersistenceContext private EntityManager entityManager; @@ -141,7 +142,8 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep return List.of( new MigrationConfig<>(workflowService, elasticConfig.getWorkflowIndex()), new MigrationConfig<>(simulationService, elasticConfig.getSimulationIndex()), - new MigrationConfig<>(codeService, elasticConfig.getCodeIndex())); + new MigrationConfig<>(codeService, elasticConfig.getCodeIndex()), + new MigrationConfig<>(datasetService, elasticConfig.getDatasetIndex())); // TODO: Write a script to properly sync the old ProjectAsset to the new PG data } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java index 07df6c10f7..0fce792841 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java @@ -5,19 +5,21 @@ import software.uncharted.terarium.hmiserver.configuration.Config; import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration; import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; +import software.uncharted.terarium.hmiserver.repository.data.DatasetRepository; import software.uncharted.terarium.hmiserver.service.elasticsearch.ElasticsearchService; import software.uncharted.terarium.hmiserver.service.s3.S3ClientService; @Service -public class DatasetService extends S3BackedAssetService { +public class DatasetService extends TerariumAssetServiceWithSearch { public DatasetService( - final ElasticsearchConfiguration elasticConfig, final Config config, + final ElasticsearchConfiguration elasticConfig, final ElasticsearchService elasticService, final ProjectAssetService projectAssetService, - final S3ClientService s3ClientService) { - super(elasticConfig, config, elasticService, projectAssetService, s3ClientService, Dataset.class); + final S3ClientService s3ClientService, + final DatasetRepository repository) { + super(config, elasticConfig, elasticService, projectAssetService, s3ClientService, repository, Dataset.class); } @Override @@ -31,4 +33,9 @@ protected String getAssetPath() { protected String getAssetIndex() { return elasticConfig.getDatasetIndex(); } + + @Override + public String getAssetAlias() { + return elasticConfig.getDatasetAlias(); + } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java index b6a99fe3ad..b0f90cc8fc 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java @@ -143,7 +143,8 @@ public List getAssets(final Integer page, final Integer pageSize) throws @Observed(name = "function_profile") public Model createAsset(final Model asset) throws IOException { // Make sure that the model framework is set to lowercase - asset.getHeader().setSchemaName(asset.getHeader().getSchemaName().toLowerCase()); + if (asset.getHeader() != null && asset.getHeader().getSchemaName() != null) + asset.getHeader().setSchemaName(asset.getHeader().getSchemaName().toLowerCase()); // Set default value for model parameters (0.0) if (asset.getSemantics() != null diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java index 51add8e8bf..deb855f4f1 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java @@ -26,12 +26,10 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import software.uncharted.terarium.hmiserver.TerariumApplicationTests; -import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration; import software.uncharted.terarium.hmiserver.configuration.MockUser; import software.uncharted.terarium.hmiserver.models.dataservice.PresignedURL; import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; import software.uncharted.terarium.hmiserver.service.data.DatasetService; -import software.uncharted.terarium.hmiserver.service.elasticsearch.ElasticsearchService; public class DatasetControllerTests extends TerariumApplicationTests { @@ -41,20 +39,14 @@ public class DatasetControllerTests extends TerariumApplicationTests { @Autowired private DatasetService datasetService; - @Autowired - private ElasticsearchService elasticService; - - @Autowired - private ElasticsearchConfiguration elasticConfig; - @BeforeEach public void setup() throws IOException { - elasticService.createOrEnsureIndexIsEmpty(elasticConfig.getDatasetIndex()); + datasetService.setupIndexAndAliasAndEnsureEmpty(); } @AfterEach public void teardown() throws IOException { - elasticService.deleteIndex(elasticConfig.getDatasetIndex()); + datasetService.teardownIndexAndAlias(); } @Test diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java index 8ca53ffa28..c0146dac17 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java @@ -60,12 +60,9 @@ public void teardown() throws IOException { @WithUserDetails(MockUser.URSULA) public void testItCanCreateProject() throws Exception { - final Project project = (Project) new Project().setName("test-name"); - - mockMvc.perform(MockMvcRequestBuilders.post("/projects") + mockMvc.perform(MockMvcRequestBuilders.post("/projects?name=test&userId=abc123&description=desc") .with(csrf()) - .contentType("application/json") - .content(objectMapper.writeValueAsString(project))) + .contentType("application/json")) .andExpect(status().isCreated()); } diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java new file mode 100644 index 0000000000..40eb2e219a --- /dev/null +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java @@ -0,0 +1,252 @@ +package software.uncharted.terarium.hmiserver.service.data; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.test.context.support.WithUserDetails; +import software.uncharted.terarium.hmiserver.TerariumApplicationTests; +import software.uncharted.terarium.hmiserver.configuration.MockUser; +import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; +import software.uncharted.terarium.hmiserver.models.dataservice.Identifier; +import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; +import software.uncharted.terarium.hmiserver.models.dataservice.dataset.DatasetColumn; + +@Slf4j +public class DatasetServiceTests extends TerariumApplicationTests { + + @Autowired + private ObjectMapper mapper; + + @Autowired + private DatasetService datasetService; + + @BeforeEach + public void setup() throws IOException { + datasetService.setupIndexAndAliasAndEnsureEmpty(); + } + + @AfterEach + public void teardown() throws IOException { + datasetService.teardownIndexAndAlias(); + } + + static Dataset createDataset() throws Exception { + return createDataset("A"); + } + + static Dataset createDataset(final String key) throws Exception { + + final Grounding grounding = new Grounding(); + grounding.setContext(new HashMap<>()); + grounding.getContext().put("hello", "world-" + key); + grounding.getContext().put("foo", "bar-" + key); + grounding.setIdentifiers(new ArrayList<>()); + grounding.getIdentifiers().add(new Identifier("curie", "maria")); + + final DatasetColumn column1 = new DatasetColumn() + .setName("Title") + .setDataType(DatasetColumn.ColumnType.STRING) + .setDescription("hello world") + .setGrounding(grounding); + final DatasetColumn column2 = new DatasetColumn() + .setName("Value") + .setDataType(DatasetColumn.ColumnType.FLOAT) + .setDescription("3.1415926") + .setGrounding(grounding); + + final Dataset dataset = new Dataset(); + dataset.setName("test-dataset-name-" + key); + dataset.setDescription("test-dataset-description-" + key); + dataset.setColumns(new ArrayList<>()); + dataset.getColumns().add(column1); + dataset.getColumns().add(column2); + dataset.setGrounding(grounding); + dataset.setPublicAsset(true); + + return dataset; + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanCreateDataset() throws Exception { + + final Dataset before = (Dataset) createDataset().setId(UUID.randomUUID()); + final Dataset after = datasetService.createAsset(before); + + Assertions.assertEquals(before.getId(), after.getId()); + Assertions.assertNotNull(after.getId()); + Assertions.assertNotNull(after.getCreatedOn()); + Assertions.assertEquals(after.getColumns().size(), 2); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCantCreateDuplicates() throws Exception { + + final Dataset dataset = (Dataset) createDataset().setId(UUID.randomUUID()); + + datasetService.createAsset(dataset); + + try { + datasetService.createAsset(dataset); + Assertions.fail("Should have thrown an exception"); + } catch (final IllegalArgumentException e) { + Assertions.assertTrue(e.getMessage().contains("already exists")); + } + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanGetDatasets() throws Exception { + + datasetService.createAsset(createDataset("0")); + datasetService.createAsset(createDataset("1")); + datasetService.createAsset(createDataset("2")); + + final List datasets = datasetService.getAssets(0, 3); + + Assertions.assertEquals(3, datasets.size()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanGetDataset() throws Exception { + + final Dataset dataset = datasetService.createAsset(createDataset()); + + final Dataset fetchedDataset = datasetService.getAsset(dataset.getId()).get(); + + Assertions.assertEquals(dataset, fetchedDataset); + Assertions.assertEquals(dataset.getId(), fetchedDataset.getId()); + Assertions.assertEquals(dataset.getCreatedOn(), fetchedDataset.getCreatedOn()); + Assertions.assertEquals(dataset.getUpdatedOn(), fetchedDataset.getUpdatedOn()); + Assertions.assertEquals(dataset.getDeletedOn(), fetchedDataset.getDeletedOn()); + Assertions.assertEquals(dataset.getGrounding(), fetchedDataset.getGrounding()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanUpdateDataset() throws Exception { + + final Dataset dataset = datasetService.createAsset(createDataset()); + dataset.setName("new name"); + + final Dataset updatedDataset = datasetService.updateAsset(dataset).orElseThrow(); + + Assertions.assertEquals(dataset, updatedDataset); + Assertions.assertNotNull(updatedDataset.getUpdatedOn()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanDeleteDataset() throws Exception { + + final Dataset dataset = datasetService.createAsset(createDataset()); + + datasetService.deleteAsset(dataset.getId()); + + final Optional deleted = datasetService.getAsset(dataset.getId()); + + Assertions.assertTrue(deleted.isEmpty()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanCloneDataset() throws Exception { + + Dataset dataset = createDataset(); + dataset = datasetService.createAsset(dataset); + + final Dataset cloned = datasetService.cloneAsset(dataset.getId()); + + Assertions.assertNotEquals(dataset.getId(), cloned.getId()); + Assertions.assertEquals(dataset.getGrounding(), cloned.getGrounding()); + Assertions.assertEquals(dataset.getColumns(), cloned.getColumns()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanExportAndImportDataset() throws Exception { + + Dataset dataset = createDataset(); + dataset = datasetService.createAsset(dataset); + + final byte[] exported = datasetService.exportAsset(dataset.getId()); + + final Dataset imported = datasetService.importAsset(exported); + + Assertions.assertNotEquals(dataset.getId(), imported.getId()); + Assertions.assertEquals(dataset.getName(), imported.getName()); + Assertions.assertEquals(dataset.getDescription(), imported.getDescription()); + Assertions.assertEquals(dataset.getGrounding(), imported.getGrounding()); + Assertions.assertEquals(dataset.getColumns(), imported.getColumns()); + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanSearchAssets() throws Exception { + + final int NUM = 32; + + List datasets = new ArrayList<>(); + for (int i = 0; i < NUM; i++) { + datasets.add(createDataset(String.valueOf(i))); + } + datasets = datasetService.createAssets(datasets); + + final List results = datasetService.searchAssets(0, NUM, null); + + Assertions.assertEquals(NUM, results.size()); + + for (int i = 0; i < results.size(); i++) { + Assertions.assertEquals(datasets.get(i).getName(), results.get(i).getName()); + Assertions.assertEquals( + datasets.get(i).getDescription(), results.get(i).getDescription()); + Assertions.assertEquals( + datasets.get(i).getGrounding(), results.get(i).getGrounding()); + Assertions.assertEquals( + datasets.get(i).getCreatedOn().toInstant().getEpochSecond(), + results.get(i).getCreatedOn().toInstant().getEpochSecond()); + Assertions.assertEquals( + datasets.get(i).getUpdatedOn().toInstant().getEpochSecond(), + results.get(i).getUpdatedOn().toInstant().getEpochSecond()); + Assertions.assertEquals( + datasets.get(i).getDeletedOn(), results.get(i).getDeletedOn()); + } + } + + @Test + @WithUserDetails(MockUser.URSULA) + public void testItCanSyncToNewIndex() throws Exception { + + final int NUM = 32; + + final List datasets = new ArrayList<>(); + for (int i = 0; i < NUM; i++) { + datasets.add(createDataset(String.valueOf(i))); + } + datasetService.createAssets(datasets); + + final String currentIndex = datasetService.getCurrentAssetIndex(); + + Assertions.assertEquals(NUM, datasetService.searchAssets(0, NUM, null).size()); + + datasetService.syncAllAssetsToNewIndex(true); + + final String newIndex = datasetService.getCurrentAssetIndex(); + + Assertions.assertEquals(NUM, datasetService.searchAssets(0, NUM, null).size()); + + Assertions.assertNotEquals(currentIndex, newIndex); + } +}