Skip to content

Commit

Permalink
[API] Allow to load all pipelines and metadata if applied (#3808)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ekazachkova
ekazachkova authored Dec 5, 2024
1 parent 6641750 commit 750fefe
Show file tree
Hide file tree
Showing 8 changed files with 131 additions and 30 deletions.
6 changes: 4 additions & 2 deletions api/src/main/java/com/epam/pipeline/acl/pipeline/PipelineApiService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import com.epam.pipeline.entity.pipeline.DocumentGenerationProperty;
import com.epam.pipeline.entity.pipeline.Pipeline;
import com.epam.pipeline.entity.pipeline.PipelineRun;
import com.epam.pipeline.entity.pipeline.PipelineWithMetadata;
import com.epam.pipeline.entity.pipeline.Revision;
import com.epam.pipeline.exception.git.GitClientException;
import com.epam.pipeline.manager.cluster.InstanceOfferManager;
Expand Down Expand Up @@ -134,8 +135,9 @@ public List<Pipeline> loadAllPipelines(boolean loadVersions) {

@PostFilter("hasRole('ADMIN') OR hasPermission(filterObject, 'READ')")
@AclMaskList
public List<Pipeline> filterPipelines(final boolean loadVersions, final EntityFilterVO filter) {
return pipelineManager.loadAllPipelines(loadVersions, filter);
public List<PipelineWithMetadata> filterPipelines(final boolean loadVersions, final boolean loadMetadata,
final EntityFilterVO filter) {
return pipelineManager.loadAllPipelines(loadVersions, loadMetadata, filter);
}

@PreAuthorize(ADMIN_ONLY)
Expand Down
8 changes: 5 additions & 3 deletions api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import com.epam.pipeline.entity.pipeline.DocumentGenerationProperty;
import com.epam.pipeline.entity.pipeline.Pipeline;
import com.epam.pipeline.entity.pipeline.PipelineRun;
import com.epam.pipeline.entity.pipeline.PipelineWithMetadata;
import com.epam.pipeline.entity.pipeline.Revision;
import com.epam.pipeline.exception.git.GitClientException;
import com.epam.pipeline.acl.pipeline.PipelineApiService;
Expand Down Expand Up @@ -177,10 +178,11 @@ public Result<List<Pipeline>> loadAllPipelines(
@ApiResponses(
value = {@ApiResponse(code = HTTP_STATUS_OK, message = API_STATUS_DESCRIPTION)
})
public Result<List<Pipeline>> filterPipelines(
public Result<List<PipelineWithMetadata>> filterPipelines(
@RequestBody final EntityFilterVO filter,
@RequestParam(defaultValue = "false") final Boolean loadVersion) {
return Result.success(pipelineApiService.filterPipelines(loadVersion, filter));
@RequestParam(defaultValue = "false") final Boolean loadVersion,
@RequestParam(defaultValue = "false") final boolean loadMetadata) {
return Result.success(pipelineApiService.filterPipelines(loadVersion, loadMetadata, filter));
}

@GetMapping(value = "/pipeline/permissions")
Expand Down
27 changes: 25 additions & 2 deletions api/src/main/java/com/epam/pipeline/dao/MetadataTagsUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,37 @@

package com.epam.pipeline.dao;

import com.epam.pipeline.controller.vo.EntityFilterVO;
import joptsimple.internal.Strings;
import org.apache.commons.collections4.MapUtils;

import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public interface MetadataTagsUtils {
Pattern FILTER_PATTERN = Pattern.compile("@FILTER@");
Pattern LOAD_METADATA_PATTERN = Pattern.compile("@WITH_METADATA@");
String AND = " AND ";
String WHERE = " WHERE ";

static String buildTagsFilterWhereClause(final EntityFilterVO filter, final String query) {
final String replacement = !Objects.isNull(filter) && MapUtils.isNotEmpty(filter.getTags())
? WHERE + buildTagsFilterClause(filter.getTags())
: Strings.EMPTY;
return FILTER_PATTERN.matcher(query).replaceFirst(replacement);
}

static String buildTagsFilterClause(final String query, final Map<String, List<String>> tags) {
return FILTER_PATTERN.matcher(query).replaceFirst(tags.entrySet().stream()
return FILTER_PATTERN.matcher(query).replaceFirst(buildTagsFilterClause(tags));
}

static String buildTagsFilterClause(final Map<String, List<String>> tags) {
return tags.entrySet().stream()
.map(entry -> tagFilterQuery(entry.getKey(), entry.getValue()))
.collect(Collectors.joining(" AND ")));
.collect(Collectors.joining(AND));
}

static String tagFilterQuery(final String key, final List<String> values) {
Expand All @@ -36,4 +55,8 @@ static String tagFilterQuery(final String key, final List<String> values) {
.map(value -> String.format("'%s'", value))
.collect(Collectors.joining(",")));
}

static String buildWithMetadataQuery(final String query, final boolean loadMetadata) {
return LOAD_METADATA_PATTERN.matcher(query).replaceFirst(loadMetadata ? ", m.data" : Strings.EMPTY);
}
}
49 changes: 37 additions & 12 deletions api/src/main/java/com/epam/pipeline/dao/pipeline/PipelineDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,20 @@
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.regex.Pattern;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;

import com.epam.pipeline.controller.vo.EntityFilterVO;
import com.epam.pipeline.dao.DaoHelper;
import com.epam.pipeline.dao.MetadataTagsUtils;
import com.epam.pipeline.dao.metadata.MetadataDao;
import com.epam.pipeline.entity.pipeline.Folder;
import com.epam.pipeline.entity.pipeline.Pipeline;
import com.epam.pipeline.entity.pipeline.PipelineType;
import com.epam.pipeline.entity.pipeline.PipelineWithMetadata;
import com.epam.pipeline.entity.pipeline.RepositoryType;
import com.epam.pipeline.entity.pipeline.run.RunVisibilityPolicy;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -88,10 +90,12 @@ public List<Pipeline> loadAllPipelines() {
PipelineParameters.getRowMapper());
}

public List<Pipeline> loadAllPipelines(final EntityFilterVO filter) {
return getNamedParameterJdbcTemplate().query(
MetadataTagsUtils.buildTagsFilterClause(loadPipelinesFiltersQuery, filter.getTags()),
PipelineParameters.getRowMapper());
public List<PipelineWithMetadata> loadPipelinesWithMetadata(final boolean loadMetadata,
final EntityFilterVO filter) {
final String query = MetadataTagsUtils.buildTagsFilterWhereClause(filter,
MetadataTagsUtils.buildWithMetadataQuery(loadPipelinesFiltersQuery, loadMetadata));
return getNamedParameterJdbcTemplate()
.query(query, PipelineParameters.getRowMapperWithMetadata(loadMetadata));
}

public Pipeline loadPipeline(Long id) {
Expand Down Expand Up @@ -168,7 +172,8 @@ enum PipelineParameters {
CONFIG,
VISIBILITY,
CODE_PATH,
DOCS_PATH;
DOCS_PATH,
DATA;

static MapSqlParameterSource getParameters(Pipeline pipeline) {
MapSqlParameterSource params = new MapSqlParameterSource();
Expand Down Expand Up @@ -203,10 +208,18 @@ static MapSqlParameterSource getParameters(Pipeline pipeline) {

static RowMapper<Pipeline> getRowMapper() {
return (rs, rowNum) -> {
Pipeline pipeline = basicInitPipeline(rs);
Long folderId = rs.getLong(FOLDER_ID.name());
if (!rs.wasNull()) {
pipeline.setParentFolderId(folderId);
final Pipeline pipeline = new Pipeline();
initPipeline(rs, pipeline);
return pipeline;
};
}

static RowMapper<PipelineWithMetadata> getRowMapperWithMetadata(final boolean loadMetadata) {
return (rs, rowNum) -> {
final PipelineWithMetadata pipeline = new PipelineWithMetadata();
initPipeline(rs, pipeline);
if (loadMetadata) {
pipeline.setData(MetadataDao.MetadataParameters.parseData(rs.getString(DATA.name())));
}
return pipeline;
};
Expand Down Expand Up @@ -234,8 +247,15 @@ private static Pipeline getPipeline(final ResultSet rs, final Folder folder) {
}
}

private static Pipeline basicInitPipeline(ResultSet rs) throws SQLException {
Pipeline pipeline = new Pipeline();
private static void initPipeline(final ResultSet rs, final Pipeline pipeline) throws SQLException {
basicInitPipeline(rs, pipeline);
final Long folderId = rs.getLong(FOLDER_ID.name());
if (!rs.wasNull()) {
pipeline.setParentFolderId(folderId);
}
}

private static Pipeline basicInitPipeline(final ResultSet rs, final Pipeline pipeline) throws SQLException {
pipeline.setId(rs.getLong(PIPELINE_ID.name()));
pipeline.setName(rs.getString(PIPELINE_NAME.name()));
pipeline.setDescription(rs.getString(DESCRIPTION.name()));
Expand All @@ -255,6 +275,11 @@ private static Pipeline basicInitPipeline(ResultSet rs) throws SQLException {
return pipeline;
}

private static Pipeline basicInitPipeline(ResultSet rs) throws SQLException {
Pipeline pipeline = new Pipeline();
return basicInitPipeline(rs, pipeline);
}

private static RunVisibilityPolicy getRunVisibility(final String rawVisibility) {
return StringUtils.isNotBlank(rawVisibility) ? RunVisibilityPolicy.valueOf(rawVisibility) : null;
}
Expand Down
29 changes: 29 additions & 0 deletions api/src/main/java/com/epam/pipeline/entity/pipeline/PipelineWithMetadata.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.epam.pipeline.entity.pipeline;

import com.epam.pipeline.entity.metadata.PipeConfValue;
import lombok.Data;
import lombok.EqualsAndHashCode;

import java.util.Map;

@EqualsAndHashCode(callSuper = true)
@Data
public class PipelineWithMetadata extends Pipeline {
private Map<String, PipeConfValue> data;
}
14 changes: 10 additions & 4 deletions api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.epam.pipeline.entity.pipeline.Folder;
import com.epam.pipeline.entity.pipeline.Pipeline;
import com.epam.pipeline.entity.pipeline.PipelineType;
import com.epam.pipeline.entity.pipeline.PipelineWithMetadata;
import com.epam.pipeline.entity.pipeline.RepositoryType;
import com.epam.pipeline.entity.pipeline.Revision;
import com.epam.pipeline.entity.security.acl.AclClass;
Expand Down Expand Up @@ -61,6 +62,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

@Service
@AclSync
Expand Down Expand Up @@ -290,10 +292,14 @@ public List<Pipeline> loadAllPipelines(boolean loadVersions) {
return result;
}

public List<Pipeline> loadAllPipelines(final boolean loadVersions, final EntityFilterVO filter) {
final List<Pipeline> result = Objects.isNull(filter) || MapUtils.isEmpty(filter.getTags())
? pipelineDao.loadAllPipelines()
: pipelineDao.loadAllPipelines(filter);
public List<PipelineWithMetadata> loadAllPipelines(final boolean loadVersions, final boolean loadMetadata,
final EntityFilterVO filter) {
final List<PipelineWithMetadata> result =
Objects.isNull(filter) || MapUtils.isEmpty(filter.getTags()) || loadMetadata
? pipelineDao.loadPipelinesWithMetadata(loadMetadata, filter)
: pipelineDao.loadAllPipelines().stream()
.map(pipeline -> (PipelineWithMetadata) pipeline)
.collect(Collectors.toList());
if (loadVersions) {
result.forEach(this::setCurrentVersion);
}
Expand Down
6 changes: 4 additions & 2 deletions api/src/main/resources/dao/pipeline-dao.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,12 @@
p.visibility,
p.code_path,
p.docs_path
@WITH_METADATA@
FROM
pipeline.pipeline p
LEFT JOIN pipeline.metadata m ON p.pipeline_id = m.entity_id
WHERE m.entity_class = 'PIPELINE' AND @FILTER@
LEFT JOIN pipeline.metadata m ON
p.pipeline_id = m.entity_id AND m.entity_class = 'PIPELINE'
@FILTER@
ORDER BY pipeline_id
]]>
</value>
Expand Down
22 changes: 17 additions & 5 deletions api/src/test/java/com/epam/pipeline/dao/pipeline/PipelineDaoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.epam.pipeline.entity.metadata.PipeConfValue;
import com.epam.pipeline.entity.pipeline.Folder;
import com.epam.pipeline.entity.pipeline.Pipeline;
import com.epam.pipeline.entity.pipeline.PipelineWithMetadata;
import com.epam.pipeline.entity.pipeline.run.RunVisibilityPolicy;
import com.epam.pipeline.entity.security.acl.AclClass;
import com.epam.pipeline.test.jdbc.AbstractJdbcTest;
Expand Down Expand Up @@ -183,7 +184,7 @@ public void shouldLoadPipelinesByTagsFilter() {
final EntityFilterVO filter = new EntityFilterVO();
filter.setTags(Collections.singletonMap(KEY_1, Collections.singletonList(VALUE_1)));

final List<Pipeline> actual = pipelineDao.loadAllPipelines(filter);
final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(false, filter);
assertThat(actual).hasSize(2);
}

Expand All @@ -197,7 +198,7 @@ public void shouldLoadPipelinesByTagsFilterWithMultipleMetadataValues() {
final EntityFilterVO filter = new EntityFilterVO();
filter.setTags(Collections.singletonMap(KEY_1, Collections.singletonList(VALUE_1)));

final List<Pipeline> actual = pipelineDao.loadAllPipelines(filter);
final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(false, filter);
assertThat(actual).hasSize(2);
assertThat(actual.stream().map(Pipeline::getId).collect(Collectors.toList()))
.containsOnly(pipeline1.getId(), pipeline2.getId());
Expand All @@ -213,7 +214,7 @@ public void shouldLoadPipelinesByTagsFilterWithMultipleFilterValues() {
final EntityFilterVO filter = new EntityFilterVO();
filter.setTags(Collections.singletonMap(KEY_1, Arrays.asList(VALUE_1, VALUE_2)));

final List<Pipeline> actual = pipelineDao.loadAllPipelines(filter);
final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(false, filter);
assertThat(actual).hasSize(2);
assertThat(actual.stream().map(Pipeline::getId).collect(Collectors.toList()))
.containsOnly(pipeline1.getId(), pipeline3.getId());
Expand All @@ -229,7 +230,7 @@ public void shouldNotLoadPipelinesByTagsFilterIfNotSuchKey() {
final EntityFilterVO filter = new EntityFilterVO();
filter.setTags(Collections.singletonMap(KEY_2, Collections.singletonList(VALUE_1)));

final List<Pipeline> actual = pipelineDao.loadAllPipelines(filter);
final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(false, filter);
assertThat(actual).hasSize(0);
}

Expand All @@ -243,10 +244,21 @@ public void shouldNotLoadPipelinesByTagsFilterIfNotSuchValue() {
final EntityFilterVO filter = new EntityFilterVO();
filter.setTags(Collections.singletonMap(KEY_1, Collections.singletonList(VALUE_2)));

final List<Pipeline> actual = pipelineDao.loadAllPipelines(filter);
final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(false, filter);
assertThat(actual).hasSize(0);
}

@Test
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = Exception.class)
public void shouldLoadPipelinesWithMetadata() {
createPipelineWithMetadata();
createPipelineWithMetadata();
pipelineDao.createPipeline(getPipeline(TEST_NAME));

final List<PipelineWithMetadata> actual = pipelineDao.loadPipelinesWithMetadata(true, null);
assertThat(actual).hasSize(3);
}

private void assertPipelineWithParameters(List<Pipeline> expected, Integer pageNum, Integer pageSize) {
Set<Pipeline> loaded = pipelineDao.loadAllPipelinesWithParents(pageNum, pageSize);
assertEquals(expected.size(), loaded.size());
Expand Down

0 comments on commit 750fefe

Please sign in to comment.