From 815e539e952b645fa50ce35fe3c2113c6bb1fee5 Mon Sep 17 00:00:00 2001 From: Kevin Birk Date: Tue, 23 Jul 2024 15:34:06 -0400 Subject: [PATCH] Add PG indexes (#4230) --- .../dataservice/ProjectController.java | 13 +--- .../dataservice/project/ProjectAsset.java | 11 ++++ .../notification/NotificationEvent.java | 9 +++ .../notification/NotificationGroup.java | 8 +++ .../PSCrudSoftDeleteRepository.java | 3 + .../repository/data/FrameworkRepository.java | 12 +--- .../repository/data/ProjectRepository.java | 65 ++++++++++--------- .../service/data/ProjectService.java | 17 +++-- .../hmiserver/utils/rebac/ReBACService.java | 24 ++++++- ...Remove_Project_Asset_Aggregation_Index.sql | 3 + 10 files changed, 102 insertions(+), 63 deletions(-) create mode 100644 packages/server/src/main/resources/db/migration/V15__Remove_Project_Asset_Aggregation_Index.sql diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java index 7c543852da..4ed8135631 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java @@ -12,7 +12,6 @@ import jakarta.transaction.Transactional; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -171,18 +170,8 @@ public ResponseEntity> getProjects( } projects.forEach(project -> { - final List assetTypes = Arrays.asList( - AssetType.DATASET, - AssetType.MODEL, - AssetType.DOCUMENT, - AssetType.WORKFLOW - ); - final RebacProject rebacProject = new RebacProject(project.getId(), reBACService); - final Schema.Permission permission = projectService.checkPermissionCanRead( - currentUserService.get().getId(), - project.getId() - ); + projectService.checkPermissionCanRead(currentUserService.get().getId(), project.getId()); // Set the user permission for the project. If we are unable to get the user // permission, we remove the project. diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/ProjectAsset.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/ProjectAsset.java index 2664636bbc..f1a9f28f1a 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/ProjectAsset.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/ProjectAsset.java @@ -4,7 +4,9 @@ import jakarta.persistence.Entity; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; +import jakarta.persistence.Index; import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; import jakarta.validation.constraints.NotNull; import java.io.Serial; import java.util.UUID; @@ -21,6 +23,15 @@ @Accessors(chain = true) @TSModel @Entity +@Table( + name = "project_asset", + indexes = { + @Index(name = "idx_asset_id", columnList = "assetId"), + @Index(name = "idx_asset_type", columnList = "assetType"), + @Index(name = "idx_project_id", columnList = "project_id"), + @Index(name = "idx_project_asset_count", columnList = "project_id, assetType, deletedOn") + } +) public class ProjectAsset extends TerariumAsset { @Serial diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationEvent.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationEvent.java index 9c67c2f473..4b4ac62e4f 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationEvent.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationEvent.java @@ -9,7 +9,9 @@ import jakarta.persistence.Entity; import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; +import jakarta.persistence.Index; import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; import jakarta.validation.constraints.NotNull; import java.io.Serial; import java.sql.Timestamp; @@ -26,6 +28,13 @@ @EqualsAndHashCode(callSuper = true) @TSModel @Entity +@Table( + name = "notification_event", + indexes = { + @Index(name = "idx_notification_group_id", columnList = "notification_group_id"), + @Index(name = "idx_acknowledged_on", columnList = "acknowledgedOn") + } +) public class NotificationEvent extends TerariumEntity { @Serial diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationGroup.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationGroup.java index 597b10705f..14ae3ea4dc 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationGroup.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/notification/NotificationGroup.java @@ -5,8 +5,10 @@ import jakarta.persistence.CascadeType; import jakarta.persistence.Entity; import jakarta.persistence.FetchType; +import jakarta.persistence.Index; import jakarta.persistence.OneToMany; import jakarta.persistence.OrderBy; +import jakarta.persistence.Table; import jakarta.validation.constraints.NotNull; import java.io.Serial; import java.util.ArrayList; @@ -24,6 +26,12 @@ @EqualsAndHashCode(callSuper = true) @TSModel @Entity +@Table( + name = "notification_group", + indexes = { + @Index(name = "idx_user_id", columnList = "userId"), @Index(name = "idx_created_on", columnList = "createdOn") + } +) public class NotificationGroup extends TerariumEntity { @Serial diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/PSCrudSoftDeleteRepository.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/PSCrudSoftDeleteRepository.java index 8b5d428947..a302fcefac 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/PSCrudSoftDeleteRepository.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/PSCrudSoftDeleteRepository.java @@ -5,6 +5,7 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.repository.NoRepositoryBean; +import software.uncharted.terarium.hmiserver.models.dataservice.model.ModelFramework; @NoRepositoryBean public interface PSCrudSoftDeleteRepository extends PSCrudRepository { @@ -13,4 +14,6 @@ public interface PSCrudSoftDeleteRepository extends PSCrudRepository getByIdAndDeletedOnIsNull(final ID id); Page findAllByPublicAssetIsTrueAndTemporaryIsFalseAndDeletedOnIsNull(final Pageable pageable); + + List findAllByDeletedOnIsNull(); } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/FrameworkRepository.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/FrameworkRepository.java index ba1dc7c2c6..0a39aaec70 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/FrameworkRepository.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/FrameworkRepository.java @@ -1,17 +1,9 @@ package software.uncharted.terarium.hmiserver.repository.data; -import java.util.List; -import java.util.Optional; import java.util.UUID; import org.springframework.stereotype.Repository; import software.uncharted.terarium.hmiserver.models.dataservice.model.ModelFramework; -import software.uncharted.terarium.hmiserver.repository.PSCrudRepository; +import software.uncharted.terarium.hmiserver.repository.PSCrudSoftDeleteRepository; @Repository -public interface FrameworkRepository extends PSCrudRepository { - List findAllByDeletedOnIsNull(); - - List findAllByIdInAndDeletedOnIsNull(final List ids); - - Optional getByIdAndDeletedOnIsNull(final UUID id); -} +public interface FrameworkRepository extends PSCrudSoftDeleteRepository {} diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/ProjectRepository.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/ProjectRepository.java index cc47e4b4b2..a17f6f3881 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/ProjectRepository.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/ProjectRepository.java @@ -18,38 +18,39 @@ public interface ProjectRepository extends PSCrudRepository, JpaS Optional getByIdAndDeletedOnIsNull(final UUID id); @Query( - "select " + - " p.id as id, " + - " p.createdOn as createdOn, " + - " p.updatedOn as updatedOn, " + - " p.deletedOn as deletedOn, " + - " p.description as description, " + - " p.fileNames as fileNames, " + - " p.name as name, " + - " p.overviewContent as overviewContent, " + - " p.publicAsset as publicAsset, " + - " p.temporary as temporary, " + - " p.thumbnail as thumbnail, " + - " p.userId as userId, " + - " p2.assetCount as assetCount, " + - " p2.assetType as assetType " + - "from " + - " Project p " + - "left join (" + - "select " + - " pa.project.id as projectId, " + - " pa.assetType as assetType, " + - " count(*) as assetCount " + - "from " + - " ProjectAsset pa " + - "where " + - " pa.deletedOn is null " + - "group by pa.project.id, pa.assetType " + - ") as p2 " + - "on p.id = p2.projectId " + - "where " + - " p.id in (:ids) " + - " and p.deletedOn is null" + """ + select + p.id as id, + p.createdOn as createdOn, + p.updatedOn as updatedOn, + p.deletedOn as deletedOn, + p.description as description, + p.fileNames as fileNames, + p.name as name, + p.overviewContent as overviewContent, + p.publicAsset as publicAsset, + p.temporary as temporary, + p.thumbnail as thumbnail, + p.userId as userId, + p2.assetCount as assetCount, + p2.assetType as assetType + from + Project p + left join ( + select + pa.project.id as projectId, + pa.assetType as assetType, + count(*) as assetCount + from + ProjectAsset pa + where + pa.deletedOn is null + group by pa.project.id, pa.assetType) as p2 + on p.id = p2.projectId + where + p.id in (:ids) + and p.deletedOn is null + """ ) List findByIdsWithAssets(@Param("ids") final List ids); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ProjectService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ProjectService.java index 6d6c473c70..f12ec3fa21 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ProjectService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ProjectService.java @@ -48,14 +48,14 @@ public List getProjects(final List ids) { @Observed(name = "function_profile") public List getActiveProjects(final List ids) { - Map projectMap = new HashMap<>(); - List projectAggregates = projectRepository.findByIdsWithAssets(ids); - for (ProjectAndAssetAggregate aggregate : projectAggregates) { + final Map projectMap = new HashMap<>(); + final List projectAggregates = projectRepository.findByIdsWithAssets(ids); + for (final ProjectAndAssetAggregate aggregate : projectAggregates) { if (projectMap.containsKey(aggregate.getId())) { - Project project = projectMap.get(aggregate.getId()); + final Project project = projectMap.get(aggregate.getId()); addAssetCount(project, aggregate.getAssetType(), aggregate.getAssetCount()); } else { - Project project = new Project(); + final Project project = new Project(); project.setId(aggregate.getId()); project.setCreatedOn(aggregate.getCreatedOn()); project.setUpdatedOn(aggregate.getUpdatedOn()); @@ -76,7 +76,7 @@ public List getActiveProjects(final List ids) { return new ArrayList<>(projectMap.values()); } - private void addAssetCount(Project project, String assetTypeName, Integer assetCount) { + private void addAssetCount(final Project project, final String assetTypeName, final Integer assetCount) { if (AssetType.DATASET.name().equals(assetTypeName)) { project.getMetadata().put("datasets-count", assetCount.toString()); } @@ -129,6 +129,7 @@ public boolean delete(final UUID id) { return true; } + @Observed(name = "function_profile") public boolean isProjectPublic(final UUID id) { final Optional isPublic = projectRepository.findPublicAssetByIdNative(id); if (isPublic.isEmpty()) { @@ -137,6 +138,7 @@ public boolean isProjectPublic(final UUID id) { return isPublic.get(); } + @Observed(name = "function_profile") public Schema.Permission checkPermissionCanReadOrNone(final String userId, final UUID projectId) throws ResponseStatusException { try { @@ -152,6 +154,7 @@ public Schema.Permission checkPermissionCanReadOrNone(final String userId, final return Schema.Permission.NONE; } + @Observed(name = "function_profile") public Schema.Permission checkPermissionCanRead(final String userId, final UUID projectId) throws ResponseStatusException { try { @@ -167,6 +170,7 @@ public Schema.Permission checkPermissionCanRead(final String userId, final UUID throw new ResponseStatusException(HttpStatus.FORBIDDEN, messages.get("rebac.unauthorized-update")); } + @Observed(name = "function_profile") public Schema.Permission checkPermissionCanWrite(final String userId, final UUID projectId) throws ResponseStatusException { try { @@ -182,6 +186,7 @@ public Schema.Permission checkPermissionCanWrite(final String userId, final UUID throw new ResponseStatusException(HttpStatus.FORBIDDEN, messages.get("rebac.unauthorized-update")); } + @Observed(name = "function_profile") public Schema.Permission checkPermissionCanAdministrate(final String userId, final UUID projectId) throws ResponseStatusException { try { diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/utils/rebac/ReBACService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/utils/rebac/ReBACService.java index fce6c7a1ee..fb7632a542 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/utils/rebac/ReBACService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/utils/rebac/ReBACService.java @@ -10,6 +10,7 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.inprocess.InProcessChannelBuilder; +import io.micrometer.observation.annotation.Observed; import jakarta.annotation.PostConstruct; import jakarta.ws.rs.core.Response; import java.util.ArrayList; @@ -226,10 +227,12 @@ private PermissionGroup createGroup(final String parentId, final String name) { } } + @Observed(name = "function_profile") public PermissionGroup createGroup(final String name) { return this.createGroup(null, name); } + @Observed(name = "function_profile") public PermissionUser getUser(final String id) { final UsersResource usersResource = keycloak.realm(REALM_NAME).users(); final UserResource userResource = usersResource.get(id); @@ -242,6 +245,7 @@ public PermissionUser getUser(final String id) { ); } + @Observed(name = "function_profile") public List getUsers() { final List response = new ArrayList<>(); final UsersResource usersResource = keycloak.realm(REALM_NAME).users(); @@ -277,6 +281,7 @@ public List getUsers() { return response; } + @Observed(name = "function_profile") public List getRoles() { final List response = new ArrayList<>(); @@ -306,6 +311,7 @@ public List getRoles() { return response; } + @Observed(name = "function_profile") public List getGroups() { final List response = new ArrayList<>(); @@ -318,6 +324,7 @@ public List getGroups() { return response; } + @Observed(name = "function_profile") public PermissionGroup getGroup(final String id) { final GroupResource groupResource = keycloak.realm(REALM_NAME).groups().group(id); final GroupRepresentation groupRepresentation = groupResource.toRepresentation(); @@ -332,12 +339,14 @@ public PermissionGroup getGroup(final String id) { /** * Determines if user `who` has `permission` on resource `what` * - * @param who User requesting access + * @param who User requesting access * @param permission Granted permission - * @param what Resource being questioned + * @param what Resource being questioned * @return true if resource grants permission for user, otherwise false - * @throws Exception some sort of ReBAC error, most likely SpiceDB is unavailable + * @throws Exception some sort of ReBAC error, most likely SpiceDB is + * unavailable */ + @Observed(name = "function_profile") public boolean can(final SchemaObject who, final Schema.Permission permission, final SchemaObject what) throws Exception { final ReBACFunctions rebac = new ReBACFunctions(channel, spiceDbBearerToken); @@ -347,16 +356,19 @@ public boolean can(final SchemaObject who, final Schema.Permission permission, f return rebac.checkPermission(who, permission, what, getCurrentConsistency()); } + @Observed(name = "function_profile") public boolean isMemberOf(final SchemaObject who, final SchemaObject what) throws Exception { final ReBACFunctions rebac = new ReBACFunctions(channel, spiceDbBearerToken); return rebac.checkPermission(who, Schema.Permission.MEMBERSHIP, what, getCurrentConsistency()); } + @Observed(name = "function_profile") public boolean isCreator(final SchemaObject who, final SchemaObject what) throws Exception { final ReBACFunctions rebac = new ReBACFunctions(channel, spiceDbBearerToken); return rebac.hasRelationship(who, Schema.Relationship.CREATOR, what, getCurrentConsistency()); } + @Observed(name = "function_profile") public void createRelationship( final SchemaObject who, final SchemaObject what, @@ -366,6 +378,7 @@ public void createRelationship( CURRENT_ZED_TOKEN = rebac.createRelationship(who, relationship, what); } + @Observed(name = "function_profile") public void removeRelationship( final SchemaObject who, final SchemaObject what, @@ -383,11 +396,13 @@ private Consistency getCurrentConsistency() { return Consistency.newBuilder().setAtLeastAsFresh(zedToken).build(); } + @Observed(name = "function_profile") public List getRelationships(final SchemaObject what) throws Exception { final ReBACFunctions rebac = new ReBACFunctions(channel, spiceDbBearerToken); return rebac.getRelationship(what, getCurrentConsistency()); } + @Observed(name = "function_profile") public ResponseEntity deleteRoleFromUser(final String roleName, final String userId) { final UsersResource usersResource = keycloak.realm(REALM_NAME).users(); final UserResource userResource = usersResource.get(userId); @@ -431,6 +446,7 @@ public ResponseEntity deleteRoleFromUser(final String roleName, final Stri } } + @Observed(name = "function_profile") public ResponseEntity addRoleToUser(final String roleName, final String userId) { final UsersResource usersResource = keycloak.realm(REALM_NAME).users(); final UserResource userResource = usersResource.get(userId); @@ -480,12 +496,14 @@ public ResponseEntity addRoleToUser(final String roleName, final String us } } + @Observed(name = "function_profile") public List lookupResources(final SchemaObject who, final Schema.Permission permission, final Schema.Type type) throws Exception { final ReBACFunctions rebac = new ReBACFunctions(channel, spiceDbBearerToken); return rebac.lookupResources(type, permission, who, getCurrentConsistency()); } + @Observed(name = "function_profile") public static boolean isServiceUser(final String id) { return API_SERVICE_USER_ID != null && API_SERVICE_USER_ID.equals(id); } diff --git a/packages/server/src/main/resources/db/migration/V15__Remove_Project_Asset_Aggregation_Index.sql b/packages/server/src/main/resources/db/migration/V15__Remove_Project_Asset_Aggregation_Index.sql new file mode 100644 index 0000000000..18305954c1 --- /dev/null +++ b/packages/server/src/main/resources/db/migration/V15__Remove_Project_Asset_Aggregation_Index.sql @@ -0,0 +1,3 @@ + +-- Remove this, due to flyway baseline, we need indexes created in the code / entity +DROP INDEX IF EXISTS pa_count;