Skip to content

Commit

Permalink
EPMRPP-92306 || Change query for item retrieving for export data. (#1019
Browse files Browse the repository at this point in the history 
)

* EPMRPP-92306 || Add a min req query for export

* EPMRPP-92306 || Add clipped fetcher

* EPMRPP-92306 || Add clipped fetcher

* EPMRPP-92306 || Add clipped fetcher
  • Loading branch information
@pbortnik
pbortnik committed Jul 11, 2024
1 parent 2834dd5 commit c771cec
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 42 deletions.
2 changes: 2 additions & 0 deletions src/main/java/com/epam/ta/reportportal/dao/TestItemRepositoryCustom.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,8 @@ Page<TestItemHistory> loadItemsHistoryPage(boolean isLatest, Queryable launchFil

List<TestItem> findItemsForAnalyze(Long launchId);

List<TestItem> selectTestItemsProjection(Long launchId);

/**
* Select all {@link TestItem#getItemId()} of {@link TestItem} with attached {@link Issue} and
* {@link TestItem#getLaunchId()} equal to provided `launchId`
Expand Down
60 changes: 38 additions & 22 deletions src/main/java/com/epam/ta/reportportal/dao/TestItemRepositoryCustomImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import static com.epam.ta.reportportal.dao.util.RecordMappers.ISSUE_TYPE_RECORD_MAPPER;
import static com.epam.ta.reportportal.dao.util.RecordMappers.NESTED_STEP_RECORD_MAPPER;
import static com.epam.ta.reportportal.dao.util.RecordMappers.TEST_ITEM_RECORD_MAPPER;
import static com.epam.ta.reportportal.dao.util.ResultFetchers.TEST_ITEM_CLIPPED_FETCHER;
import static com.epam.ta.reportportal.dao.util.ResultFetchers.TEST_ITEM_FETCHER;
import static com.epam.ta.reportportal.dao.util.ResultFetchers.TEST_ITEM_RETRY_FETCHER;
import static com.epam.ta.reportportal.jooq.Tables.ATTACHMENT;
Expand Down Expand Up @@ -184,13 +185,13 @@ public Set<Statistics> accumulateStatisticsByFilterNotFromBaseline(Queryable tar
Queryable baselineFilter) {
final QueryBuilder targetBuilder =
QueryBuilder.newBuilder(targetFilter,
collectJoinFields(targetFilter));
collectJoinFields(targetFilter));
final QueryBuilder baselineBuilder =
QueryBuilder.newBuilder(baselineFilter,
collectJoinFields(baselineFilter));
collectJoinFields(baselineFilter));
final SelectQuery<? extends Record> contentQuery =
getQueryWithBaseline(targetBuilder,
baselineBuilder).build();
baselineBuilder).build();

return dsl.fetch(DSL.with(FILTERED_QUERY)
.as(contentQuery)
Expand Down Expand Up @@ -235,7 +236,7 @@ public Page<TestItem> findByFilter(boolean isLatest, Queryable launchFilter,

Set<String> joinFields =
QueryUtils.collectJoinFields(testItemFilter,
testItemPageable.getSort());
testItemPageable.getSort());

return PageableExecutionUtils.getPage(
TEST_ITEM_FETCHER.apply(dsl.fetch(QueryBuilder.newBuilder(testItemFilter, joinFields)
Expand Down Expand Up @@ -273,13 +274,13 @@ public Page<TestItem> findAllNotFromBaseline(Queryable targetFilter, Queryable b

final QueryBuilder targetPagingBuilder =
QueryBuilder.newBuilder(targetFilter,
collectJoinFields(targetFilter, pageable.getSort()));
collectJoinFields(targetFilter, pageable.getSort()));
final QueryBuilder baselinePagingBuilder =
QueryBuilder.newBuilder(baselineFilter,
collectJoinFields(baselineFilter));
collectJoinFields(baselineFilter));
final SelectQuery<? extends Record> pagingQuery =
getQueryWithBaseline(targetPagingBuilder,
baselinePagingBuilder).build();
baselinePagingBuilder).build();

return PageableExecutionUtils.getPage(TEST_ITEM_FETCHER.apply(dsl.fetch(contentQuery)),
pageable,
Expand Down Expand Up @@ -320,7 +321,7 @@ public Page<TestItemHistory> loadItemsHistoryPage(Queryable filter, Pageable pag
Field<?> historyGroupingField = usingHash ? TEST_ITEM.TEST_CASE_HASH : TEST_ITEM.UNIQUE_ID;
Page<String> historyBaseline =
loadHistoryBaseline(filteringQuery, historyGroupingField,
LAUNCH.PROJECT_ID.eq(projectId),
LAUNCH.PROJECT_ID.eq(projectId),
pageable);

List<TestItemHistory> itemHistories = historyBaseline.getContent().stream().map(value -> {
Expand Down Expand Up @@ -626,7 +627,7 @@ public List<Long> findTestItemIdsByLaunchId(Long launchId, Pageable pageable) {
public List<TestItem> selectItemsInStatusByLaunch(Long launchId, StatusEnum... statuses) {
List<JStatusEnum> jStatuses =
Arrays.stream(statuses).map(it -> JStatusEnum.valueOf(it.name()))
.collect(toList());
.collect(toList());
return commonTestItemDslSelect().where(
TEST_ITEM.LAUNCH_ID.eq(launchId).and(TEST_ITEM_RESULTS.STATUS.in(jStatuses)))
.fetch(TEST_ITEM_RECORD_MAPPER);
Expand All @@ -636,7 +637,7 @@ public List<TestItem> selectItemsInStatusByLaunch(Long launchId, StatusEnum... s
public List<TestItem> selectItemsInStatusByParent(Long itemId, StatusEnum... statuses) {
List<JStatusEnum> jStatuses =
Arrays.stream(statuses).map(it -> JStatusEnum.valueOf(it.name()))
.collect(toList());
.collect(toList());
return commonTestItemDslSelect().where(
TEST_ITEM.PARENT_ID.eq(itemId).and(TEST_ITEM_RESULTS.STATUS.in(jStatuses)))
.fetch(TEST_ITEM_RECORD_MAPPER);
Expand All @@ -646,7 +647,7 @@ public List<TestItem> selectItemsInStatusByParent(Long itemId, StatusEnum... sta
public Boolean hasItemsInStatusByLaunch(Long launchId, StatusEnum... statuses) {
List<JStatusEnum> jStatuses =
Arrays.stream(statuses).map(it -> JStatusEnum.valueOf(it.name()))
.collect(toList());
.collect(toList());
return dsl.fetchExists(dsl.selectOne()
.from(TEST_ITEM)
.join(TEST_ITEM_RESULTS)
Expand Down Expand Up @@ -759,6 +760,22 @@ public List<TestItem> findItemsForAnalyze(Long launchId) {
.fetch(TEST_ITEM_RECORD_MAPPER);
}

@Override
public List<TestItem> selectTestItemsProjection(Long launchId) {
return TEST_ITEM_CLIPPED_FETCHER.apply(dsl.fetch(dsl.select()
.from(TEST_ITEM)
.join(TEST_ITEM_RESULTS)
.on(TEST_ITEM.ITEM_ID.eq(TEST_ITEM_RESULTS.RESULT_ID))
.leftJoin(ISSUE)
.on(ISSUE.ISSUE_ID.eq(TEST_ITEM_RESULTS.RESULT_ID))
.leftJoin(STATISTICS)
.on(TEST_ITEM.ITEM_ID.eq(STATISTICS.ITEM_ID))
.leftJoin(STATISTICS_FIELD)
.on(STATISTICS.STATISTICS_FIELD_ID.eq(STATISTICS_FIELD.SF_ID))
.where(TEST_ITEM.LAUNCH_ID.eq(launchId)).and(TEST_ITEM.HAS_STATS.eq(true))
.orderBy(TEST_ITEM.START_TIME.asc())));
}

@Override
public List<Long> selectIdsWithIssueByLaunch(Long launchId) {
return dsl.select(TEST_ITEM.ITEM_ID)
Expand All @@ -776,7 +793,7 @@ public Boolean hasItemsInStatusAddedLately(Long launchId, Duration period,
StatusEnum... statuses) {
List<JStatusEnum> jStatuses =
Arrays.stream(statuses).map(it -> JStatusEnum.valueOf(it.name()))
.collect(toList());
.collect(toList());
return dsl.fetchExists(dsl.selectOne()
.from(TEST_ITEM)
.join(TEST_ITEM_RESULTS)
Expand All @@ -791,7 +808,7 @@ public Boolean hasItemsInStatusAddedLately(Long launchId, Duration period,
public Boolean hasLogs(Long launchId, Duration period, StatusEnum... statuses) {
List<JStatusEnum> jStatuses =
Arrays.stream(statuses).map(it -> JStatusEnum.valueOf(it.name()))
.collect(toList());
.collect(toList());
return dsl.fetchExists(dsl.selectOne()
.from(TEST_ITEM)
.join(TEST_ITEM_RESULTS)
Expand Down Expand Up @@ -1099,11 +1116,11 @@ public Page<TestItem> findByFilter(Queryable filter, Pageable pageable) {
Set<String> joinFields = QueryUtils.collectJoinFields(filter, pageable.getSort());
List<TestItem> items =
TEST_ITEM_FETCHER.apply(
dsl.fetch(QueryBuilder.newBuilder(filter, joinFields)
.with(pageable)
.wrap()
.withWrapperSort(pageable.getSort())
.build()));
dsl.fetch(QueryBuilder.newBuilder(filter, joinFields)
.with(pageable)
.wrap()
.withWrapperSort(pageable.getSort())
.build()));

return PageableExecutionUtils.getPage(items, pageable,
() -> dsl.fetchCount(QueryBuilder.newBuilder(filter, joinFields).build()));
Expand All @@ -1114,7 +1131,6 @@ public List<NestedStep> findAllNestedStepsByIds(Collection<Long> ids, Queryable
boolean excludePassedLogs) {
JTestItem nested = TEST_ITEM.as(NESTED);


CommonTableExpression<?> logsCte = DSL.name("logsCTE")
.as(QueryBuilder.newBuilder(logFilter, QueryUtils.collectJoinFields(logFilter)).build());

Expand All @@ -1132,7 +1148,7 @@ public List<NestedStep> findAllNestedStepsByIds(Collection<Long> ids, Queryable
.join(nested)
.on(LOG.ITEM_ID.eq(nested.ITEM_ID))
.join(logsCte)
.on(LOG.ID.eq(logsCte.field( ID).cast(Long.class)))
.on(LOG.ID.eq(logsCte.field(ID).cast(Long.class)))
.join(ATTACHMENT)
.on(LOG.ATTACHMENT_ID.eq(ATTACHMENT.ID))
.where(nested.HAS_STATS.isFalse()
Expand Down Expand Up @@ -1178,7 +1194,7 @@ private Condition hasContentQuery(JTestItem nested, CommonTableExpression logsCt
return DSL.exists(dsl.select()
.from(LOG)
.join(logsCte)
.on(LOG.ID.eq(logsCte.field( ID).cast(Long.class)))
.on(LOG.ID.eq(logsCte.field(ID).cast(Long.class)))
.join(TEST_ITEM_RESULTS)
.on(LOG.ITEM_ID.eq(TEST_ITEM_RESULTS.RESULT_ID))
.where(LOG.ITEM_ID.eq(TEST_ITEM.ITEM_ID)))
Expand All @@ -1188,7 +1204,7 @@ private Condition hasContentQuery(JTestItem nested, CommonTableExpression logsCt
return DSL.exists(dsl.select()
.from(LOG)
.join(logsCte)
.on(LOG.ID.eq(logsCte.field( ID).cast(Long.class)))
.on(LOG.ID.eq(logsCte.field(ID).cast(Long.class)))
.where(LOG.ITEM_ID.eq(TEST_ITEM.ITEM_ID)))
.orExists(dsl.select().from(nested)
.where(nested.PARENT_ID.eq(TEST_ITEM.ITEM_ID).and(nested.HAS_STATS.isFalse())));
Expand Down
52 changes: 32 additions & 20 deletions src/main/java/com/epam/ta/reportportal/dao/util/ResultFetchers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import static com.epam.ta.reportportal.jooq.Tables.INTEGRATION;
import static com.epam.ta.reportportal.jooq.Tables.LAUNCH;
import static com.epam.ta.reportportal.jooq.Tables.LOG;
import static com.epam.ta.reportportal.jooq.Tables.OWNED_ENTITY;
import static com.epam.ta.reportportal.jooq.Tables.PARAMETER;
import static com.epam.ta.reportportal.jooq.Tables.PROJECT_ATTRIBUTE;
import static com.epam.ta.reportportal.jooq.tables.JProject.PROJECT;
Expand All @@ -42,6 +43,8 @@
import static com.epam.ta.reportportal.jooq.tables.JUsers.USERS;
import static java.util.Optional.ofNullable;

import com.epam.reportportal.rules.exception.ErrorType;
import com.epam.reportportal.rules.exception.ReportPortalException;
import com.epam.ta.reportportal.commons.ReportPortalUser;
import com.epam.ta.reportportal.commons.querygen.FilterCondition;
import com.epam.ta.reportportal.entity.activity.Activity;
Expand All @@ -62,32 +65,21 @@
import com.epam.ta.reportportal.entity.user.User;
import com.epam.ta.reportportal.entity.user.UserRole;
import com.epam.ta.reportportal.entity.widget.Widget;
import com.epam.reportportal.rules.exception.ReportPortalException;

import com.epam.reportportal.rules.exception.ErrorType;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import org.jooq.Record;
import org.jooq.Result;
import org.springframework.data.domain.Sort;
import org.springframework.util.CollectionUtils;

import java.util.*;
import java.util.function.Function;

import static com.epam.ta.reportportal.dao.constant.LogRepositoryConstants.PAGE_NUMBER;
import static com.epam.ta.reportportal.dao.constant.LogRepositoryConstants.TYPE;
import static com.epam.ta.reportportal.dao.constant.LogRepositoryConstants.LOG_LEVEL;
import static com.epam.ta.reportportal.dao.constant.WidgetContentRepositoryConstants.ID;
import static com.epam.ta.reportportal.dao.util.RecordMappers.*;
import static com.epam.ta.reportportal.jooq.Tables.*;
import static com.epam.ta.reportportal.jooq.tables.JProject.PROJECT;
import static com.epam.ta.reportportal.jooq.tables.JProjectUser.PROJECT_USER;
import static com.epam.ta.reportportal.jooq.tables.JTestItem.TEST_ITEM;
import static com.epam.ta.reportportal.jooq.tables.JUsers.USERS;
import static java.util.Optional.ofNullable;

/**
* Fetches results from db by JOOQ queries into Java objects.
*
Expand Down Expand Up @@ -179,6 +171,26 @@ private ResultFetchers() {
return new ArrayList<>(testItems.values());
};

/**
* Fetches records from db results into list of {@link TestItem} objects.
*/
public static final Function<Result<? extends Record>, List<TestItem>> TEST_ITEM_CLIPPED_FETCHER = records -> {
Map<Long, TestItem> testItems = Maps.newLinkedHashMap();
records.forEach(record -> {
Long id = record.get(TEST_ITEM.ITEM_ID);
TestItem testItem;
if (!testItems.containsKey(id)) {
testItem = RecordMappers.TEST_ITEM_RECORD_MAPPER.map(record);
} else {
testItem = testItems.get(id);
}
testItem.getItemResults().getStatistics()
.add(RecordMappers.STATISTICS_RECORD_MAPPER.map(record));
testItems.put(id, testItem);
});
return new ArrayList<>(testItems.values());
};

/**
* Fetches records from db results into list of {@link TestItem} objects.
*/
Expand Down Expand Up @@ -229,8 +241,8 @@ private ResultFetchers() {
};

/**
* Fetches records from db results into list of
* {@link com.epam.ta.reportportal.entity.integration.Integration} objects.
* Fetches records from db results into list of {@link com.epam.ta.reportportal.entity.integration.Integration}
* objects.
*/
public static final Function<Result<? extends Record>, List<Integration>> INTEGRATION_FETCHER = records -> {
Map<Integer, Integration> integrations = Maps.newLinkedHashMap();
Expand Down

0 comments on commit c771cec

Please sign in to comment.