Skip to content

Commit

Permalink
feat(rest): Update REST Attachment endpoints and documentation
Browse files Browse the repository at this point in the history
- update documentation
- refactored attachment service
- added attachment endpoints

Signed-off-by: Thomas Maier <thomas.maier@evosoft.com>
  • Loading branch information
maierthomas committed Aug 14, 2018
1 parent 93cee8b commit 8d36000
Show file tree
Hide file tree
Showing 38 changed files with 776 additions and 395 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright Siemens AG, 2013-2015. Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/

package org.eclipse.sw360.datahandler.db;

import org.eclipse.sw360.datahandler.couchdb.DatabaseConnector;
import org.eclipse.sw360.datahandler.couchdb.DatabaseRepository;
import org.eclipse.sw360.datahandler.permissions.PermissionUtils;
import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
import org.eclipse.sw360.datahandler.thrift.users.User;
import org.ektorp.DocumentOperationResult;
import org.ektorp.ViewQuery;
import org.ektorp.support.View;
import org.ektorp.support.Views;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
* CRUD access for the Attachment class
*
* @author cedric.bodet@tngtech.com
* @author Johannes.Najjar@tngtech.com
* @author daniele.fognini@tngtech.com
*/
@Views({
@View(name = "all", map = "function(doc) { if (doc.type == 'attachment') emit(null, doc._id) }"),
@View(name = "onlyRemotes", map = "function(doc) { if(doc.type == 'attachment' && doc.onlyRemote) { emit(null, doc) } }")
})
public class AttachmentContentRepository extends DatabaseRepository<AttachmentContent> {

public AttachmentContentRepository(DatabaseConnector db) {
super(AttachmentContent.class, db);

initStandardDesignDocument();
}

public List<AttachmentContent> getOnlyRemoteAttachments() {
ViewQuery query = createQuery("onlyRemotes");
query.includeDocs(false);
return queryView(query);
}

public RequestSummary vacuumAttachmentDB(User user, final Set<String> usedIds) {
final RequestSummary requestSummary = new RequestSummary();
if (!PermissionUtils.isAdmin(user))
return requestSummary.setRequestStatus(RequestStatus.FAILURE);

final List<AttachmentContent> allAttachmentContents = getAll();
final Set<AttachmentContent> unusedAttachmentContents = allAttachmentContents.stream()
.filter(input -> !usedIds.contains(input.getId()))
.collect(Collectors.toSet());

requestSummary.setTotalElements(allAttachmentContents.size());
requestSummary.setTotalAffectedElements(unusedAttachmentContents.size());

final List<DocumentOperationResult> documentOperationResults = deleteBulk(unusedAttachmentContents);
if (documentOperationResults.isEmpty()) {
requestSummary.setRequestStatus(RequestStatus.SUCCESS);
}else{
requestSummary.setRequestStatus(RequestStatus.FAILURE);
}
return requestSummary;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
import org.eclipse.sw360.datahandler.thrift.Source;
import org.eclipse.sw360.datahandler.thrift.attachments.Attachment;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentUsage;
import org.eclipse.sw360.datahandler.thrift.attachments.UsageData;
Expand Down Expand Up @@ -48,36 +49,41 @@
*/
public class AttachmentDatabaseHandler {
private final DatabaseConnector db;
private final AttachmentRepository repository;
private final AttachmentContentRepository attachmentContentRepository;
private final AttachmentConnector attachmentConnector;
private final AttachmentUsageRepository attachmentUsageRepository;
private final AttachmentRepository attachmentRepository;
private final AttachmentOwnerRepository attachmentOwnerRepository;


private static final Logger log = Logger.getLogger(AttachmentDatabaseHandler.class);

public AttachmentDatabaseHandler(Supplier<HttpClient> httpClient, String dbName, String attachmentDbName) throws MalformedURLException {
db = new DatabaseConnector(httpClient, attachmentDbName);
attachmentConnector = new AttachmentConnector(httpClient, attachmentDbName, durationOf(30, TimeUnit.SECONDS));
repository = new AttachmentRepository(db);
attachmentContentRepository = new AttachmentContentRepository(db);
attachmentUsageRepository = new AttachmentUsageRepository(new DatabaseConnector(httpClient, dbName));
attachmentRepository = new AttachmentRepository(new DatabaseConnector(httpClient, dbName));
attachmentOwnerRepository = new AttachmentOwnerRepository(new DatabaseConnector(httpClient, dbName));
}

public AttachmentConnector getAttachmentConnector(){
return attachmentConnector;
}

public AttachmentContent add(AttachmentContent attachmentContent){
repository.add(attachmentContent);
attachmentContentRepository.add(attachmentContent);
return attachmentContent;
}
public List<AttachmentContent> makeAttachmentContents(List<AttachmentContent> attachmentContents) throws TException {
final List<DocumentOperationResult> documentOperationResults = repository.executeBulk(attachmentContents);
final List<DocumentOperationResult> documentOperationResults = attachmentContentRepository.executeBulk(attachmentContents);
if (!documentOperationResults.isEmpty())
log.error("Failed Attachment store results " + documentOperationResults);

return attachmentContents.stream().filter(AttachmentContent::isSetId).collect(Collectors.toList());
}
public AttachmentContent getAttachmentContent(String id) throws TException {
AttachmentContent attachment = repository.get(id);
AttachmentContent attachment = attachmentContentRepository.get(id);
assertNotNull(attachment, "Cannot find "+ id + " in database.");
validateAttachment(attachment);

Expand All @@ -87,7 +93,7 @@ public void updateAttachmentContent(AttachmentContent attachment) throws TExcept
attachmentConnector.updateAttachmentContent(attachment);
}
public RequestSummary bulkDelete(List<String> ids) {
final List<DocumentOperationResult> documentOperationResults = repository.deleteIds(ids);
final List<DocumentOperationResult> documentOperationResults = attachmentContentRepository.deleteIds(ids);
return CommonUtils.getRequestSummary(ids, documentOperationResults);
}
public RequestStatus deleteAttachmentContent(String attachmentId) throws TException {
Expand All @@ -96,7 +102,7 @@ public RequestStatus deleteAttachmentContent(String attachmentId) throws TExcept
return RequestStatus.SUCCESS;
}
public RequestSummary vacuumAttachmentDB(User user, Set<String> usedIds) throws TException {
return repository.vacuumAttachmentDB(user, usedIds);
return attachmentContentRepository.vacuumAttachmentDB(user, usedIds);
}
public String getSha1FromAttachmentContentId(String attachmentContentId){
return attachmentConnector.getSha1FromAttachmentContentId(attachmentContentId);
Expand Down Expand Up @@ -228,4 +234,14 @@ public Map<Map<Source, String>, Integer> getAttachmentUsageCount(Map<Source, Set
return ImmutableMap.of(new Source(idToType.get(key.getKey()), key.getKey()), key.getValue());
}, Map.Entry::getValue));
}

public List<Attachment> getAttachmentsByIds(Set<String> ids) {
return attachmentRepository.getAttachmentsByIds(ids);
}
public List<Attachment> getAttachmentsBySha1s(Set<String> sha1s) {
return attachmentRepository.getAttachmentsBySha1s(sha1s);
}
public List<Source> getAttachmentOwnersByIds(Set<String> ids) {
return attachmentOwnerRepository.getOwnersByIds(ids);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Copyright Siemens AG, 2018. Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.sw360.datahandler.db;

import org.eclipse.sw360.datahandler.couchdb.DatabaseConnector;
import org.eclipse.sw360.datahandler.couchdb.DatabaseRepository;
import org.eclipse.sw360.datahandler.thrift.Source;
import org.ektorp.ViewQuery;
import org.ektorp.support.View;
import org.ektorp.support.Views;

import java.util.List;
import java.util.Set;

@Views({
@View(name = "attachmentOwner",
map = "function(doc) { if (doc.type == 'project' || doc.type == 'component' || doc.type == 'release') { " +
"for(var i in doc.attachments) { " +
"var source;" +
"if (doc.type == 'project') {source = {projectId: doc._id}}" +
"if (doc.type == 'component') {source = {componentId: doc._id}}" +
"if (doc.type == 'release') {source = {releaseId: doc._id}}" +
"emit(doc.attachments[i].attachmentContentId, source); } } }")
})

public class AttachmentOwnerRepository extends DatabaseRepository<Source> {

public AttachmentOwnerRepository(DatabaseConnector db) {
super(Source.class, db);
initStandardDesignDocument();
}

public List<Source> getOwnersByIds(Set<String> ids) {
ViewQuery viewQuery = createQuery("attachmentOwner").includeDocs(false).keys(ids);
return queryView(viewQuery);
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright Siemens AG, 2013-2015. Part of the SW360 Portal Project.
* Copyright Siemens AG, 2018. Part of the SW360 Portal Project.
*
* SPDX-License-Identifier: EPL-1.0
*
Expand All @@ -8,69 +8,43 @@
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/

package org.eclipse.sw360.datahandler.db;

import org.eclipse.sw360.datahandler.couchdb.DatabaseConnector;
import org.eclipse.sw360.datahandler.couchdb.DatabaseRepository;
import org.eclipse.sw360.datahandler.permissions.PermissionUtils;
import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
import org.eclipse.sw360.datahandler.thrift.users.User;
import org.ektorp.DocumentOperationResult;
import org.eclipse.sw360.datahandler.thrift.attachments.Attachment;
import org.ektorp.ViewQuery;
import org.ektorp.support.View;
import org.ektorp.support.Views;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
* CRUD access for the Attachment class
*
* @author cedric.bodet@tngtech.com
* @author Johannes.Najjar@tngtech.com
* @author daniele.fognini@tngtech.com
*/
@Views({
@View(name = "all", map = "function(doc) { if (doc.type == 'attachment') emit(null, doc._id) }"),
@View(name = "onlyRemotes", map = "function(doc) { if(doc.type == 'attachment' && doc.onlyRemote) { emit(null, doc) } }")
@View(name = "byid",
map = "function(doc) { if (doc.type == 'project' || doc.type == 'component' || doc.type == 'release') { " +
"for(var i in doc.attachments) { " +
"emit(doc.attachments[i].attachmentContentId, doc.attachments[i]); } } }"),
@View(name = "bysha1",
map = "function(doc) { if (doc.type == 'project' || doc.type == 'component' || doc.type == 'release') { " +
"for(var i in doc.attachments) { " +
"emit(doc.attachments[i].sha1, doc.attachments[i]); } } }")
})
public class AttachmentRepository extends DatabaseRepository<AttachmentContent> {

public AttachmentRepository(DatabaseConnector db) {
super(AttachmentContent.class, db);
public class AttachmentRepository extends DatabaseRepository<Attachment> {

public AttachmentRepository(DatabaseConnector db) {
super(Attachment.class, db);
initStandardDesignDocument();
}

public List<AttachmentContent> getOnlyRemoteAttachments() {
ViewQuery query = createQuery("onlyRemotes");
query.includeDocs(false);
return queryView(query);
public List<Attachment> getAttachmentsByIds(Set<String> ids) {
ViewQuery viewQuery = createQuery("byid").includeDocs(false).keys(ids);
return queryView(viewQuery);
}

public RequestSummary vacuumAttachmentDB(User user, final Set<String> usedIds) {
final RequestSummary requestSummary = new RequestSummary();
if (!PermissionUtils.isAdmin(user))
return requestSummary.setRequestStatus(RequestStatus.FAILURE);

final List<AttachmentContent> allAttachmentContents = getAll();
final Set<AttachmentContent> unusedAttachmentContents = allAttachmentContents.stream()
.filter(input -> !usedIds.contains(input.getId()))
.collect(Collectors.toSet());

requestSummary.setTotalElements(allAttachmentContents.size());
requestSummary.setTotalAffectedElements(unusedAttachmentContents.size());

final List<DocumentOperationResult> documentOperationResults = deleteBulk(unusedAttachmentContents);
if (documentOperationResults.isEmpty()) {
requestSummary.setRequestStatus(RequestStatus.SUCCESS);
}else{
requestSummary.setRequestStatus(RequestStatus.FAILURE);
}
return requestSummary;
public List<Attachment> getAttachmentsBySha1s(Set<String> sha1s) {
ViewQuery viewQuery = createQuery("bysha1").includeDocs(false).keys(sha1s);
return queryView(viewQuery);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,7 @@
import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.Source;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentService;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentUsage;
import org.eclipse.sw360.datahandler.thrift.attachments.UsageData;
import org.eclipse.sw360.datahandler.thrift.attachments.*;
import org.eclipse.sw360.datahandler.thrift.users.User;

import java.net.MalformedURLException;
Expand All @@ -42,10 +39,8 @@
*/
public class AttachmentHandler implements AttachmentService.Iface {

private static final Logger log = Logger.getLogger(AttachmentHandler.class);
private final AttachmentDatabaseHandler handler;


public AttachmentHandler() throws MalformedURLException {
handler = new AttachmentDatabaseHandler(DatabaseSettings.getConfiguredHttpClient(), DatabaseSettings.COUCH_DB_DATABASE, DatabaseSettings.COUCH_DB_ATTACHMENTS);
}
Expand Down Expand Up @@ -167,12 +162,7 @@ public void deleteAttachmentUsagesByUsageDataType(Source usedBy, UsageData usage

@Override
public List<AttachmentUsage> getAttachmentUsages(Source owner, String attachmentContentId, UsageData filter) throws TException {
assertNotNull(owner);
assertTrue(owner.isSet());
assertNotNull(attachmentContentId);
assertNotEmpty(attachmentContentId);

return handler.getAttachmentUsages(owner, attachmentContentId, filter);
return getAttachmentsUsages(owner, Collections.singleton(attachmentContentId), filter);
}

@Override
Expand Down Expand Up @@ -220,4 +210,21 @@ public Map<Map<Source, String>, Integer> getAttachmentUsageCount(Map<Source, Set
assertNotNull(attachments);
return handler.getAttachmentUsageCount(attachments, filter);
}

@Override
public List<Attachment> getAttachmentsByIds(Set<String> ids) throws TException {
assertNotEmpty(ids);
return handler.getAttachmentsByIds(ids);
}
@Override
public List<Attachment> getAttachmentsBySha1s(Set<String> sha1s) throws TException {
assertNotEmpty(sha1s);
return handler.getAttachmentsBySha1s(sha1s);
}
@Override
public List<Source> getAttachmentOwnersByIds(Set<String> ids) throws TException {
assertNotEmpty(ids);
return handler.getAttachmentOwnersByIds(ids);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import org.eclipse.sw360.datahandler.common.Duration;
import org.eclipse.sw360.datahandler.couchdb.AttachmentConnector;
import org.eclipse.sw360.datahandler.couchdb.DatabaseConnector;
import org.eclipse.sw360.datahandler.db.AttachmentRepository;
import org.eclipse.sw360.datahandler.db.AttachmentContentRepository;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentContent;
import org.apache.log4j.Logger;
Expand Down Expand Up @@ -48,9 +48,9 @@ public static void main(String[] args) throws MalformedURLException {

public static int retrieveRemoteAttachments(Supplier<HttpClient> httpClient, String dbAttachments, Duration downloadTimeout) throws MalformedURLException {
AttachmentConnector attachmentConnector = new AttachmentConnector(httpClient, dbAttachments, downloadTimeout);
AttachmentRepository attachmentRepository = new AttachmentRepository(new DatabaseConnector(httpClient, dbAttachments));
AttachmentContentRepository attachmentContentRepository = new AttachmentContentRepository(new DatabaseConnector(httpClient, dbAttachments));

List<AttachmentContent> remoteAttachments = attachmentRepository.getOnlyRemoteAttachments();
List<AttachmentContent> remoteAttachments = attachmentContentRepository.getOnlyRemoteAttachments();
log.info(format("we have %d remote attachments to retrieve", remoteAttachments.size()));

int count = 0;
Expand Down
Loading

0 comments on commit 8d36000

Please sign in to comment.