Skip to content

Commit

Permalink
feat(licenseImport): fix and improve
Browse files Browse the repository at this point in the history 
adds the functionallity to overwrite licenses with import when
- an external id matches uniqly or
- the license id matches

Additionally the import was broken and is now fixed again
  • Loading branch information
@maxhbr
maxhbr committed Oct 30, 2018
1 parent 3eca7fc commit a20a225
Show file tree
Hide file tree
Showing 34 changed files with 884 additions and 458 deletions.
7 changes: 6 additions & 1 deletion backend/src/src-licenses/src/main/java/org/eclipse/sw360/licenses/LicenseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,12 @@ public List<LicenseType> addLicenseTypes(List<LicenseType> licenseTypes, User us

@Override
public List<License> addLicenses(List<License> licenses, User user) throws TException {
return handler.addLicenses(licenses, user);
return handler.addOrOverwriteLicenses(licenses, user, false);
}

@Override
public List<License> addOrOverwriteLicenses(List<License> licenses, User user) throws TException {
return handler.addOrOverwriteLicenses(licenses, user, true);
}

@Override
Expand Down
29 changes: 25 additions & 4 deletions .../src/src-licenses/src/main/java/org/eclipse/sw360/licenses/db/LicenseDatabaseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,20 +568,38 @@ public List<LicenseType> addLicenseTypes(List<LicenseType> licenseTypes, User us
} else return null;
}

public List<License> addLicenses(List<License> licenses, User user) throws SW360Exception {
public List<License> addOrOverwriteLicenses(List<License> licenses, User user, boolean allowOverwriting) throws SW360Exception {
final List<License> knownLicenses = licenseRepository.getAll();
for (License license : licenses) {
if(! makePermission(license, user).isActionAllowed(RequestedAction.CLEARING)){
license.setChecked(false);
}

validateNewLicense(license);
if(! license.isSetId()){
validateNewLicense(license);
} else {
if(allowOverwriting) {
knownLicenses.stream()
.filter(kl -> license.getId().equals(kl.getId()))
.findFirst()
.map(License::getRevision)
.ifPresent(license::setRevision);
} else {
license.unsetRevision();
}
validateExistingLicense(license);
}
prepareLicense(license);
}

final List<DocumentOperationResult> documentOperationResults = licenseRepository.executeBulk(licenses);
if (documentOperationResults.isEmpty()) {
return licenses;
} else return null;
} else {
documentOperationResults.forEach(dor ->
log.error("Adding license=[" + dor.getId() + "] produced an [" + dor.getError() + "] due to: " + dor.getReason()));
return null;
}
}

public List<Obligation> addObligations(List<Obligation> obligations, User user) throws SW360Exception {
Expand Down Expand Up @@ -614,6 +632,9 @@ public List<Todo> addTodos(List<Todo> todos, User user) throws SW360Exception {

public List<License> getLicenses() {
final List<License> licenses = licenseRepository.getAll();
if (licenses == null) {
return Collections.emptyList();
}
final List<Todo> todos = getTodosFromLicenses(licenses);
final List<LicenseType> licenseTypes = getLicenseTypesFromLicenses(licenses);
final List<Risk> risks = getRisksFromLicenses(licenses);
Expand Down Expand Up @@ -858,7 +879,7 @@ public RequestSummary importAllSpdxLicenses(User user) {
}

try {
addLicenses(newLicenses,user);
addOrOverwriteLicenses(newLicenses,user, false);

if (mismatchedLicenses.size() > 0){
requestSummary.setMessage("The following licenses did not match their SPDX equivalent: " + COMMA_JOINER.join(mismatchedLicenses));
Expand Down
4 changes: 3 additions & 1 deletion backend/src/src-licenses/src/main/java/org/eclipse/sw360/licenses/tools/SpdxConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.spdx.rdfparser.license.SpdxListedLicense;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -54,7 +55,8 @@ public static Optional<License> getSpdxLicenseAsSW360License(SpdxListedLicense s
.setShortname(spdxListedLicense.getLicenseId())
.setFullname(spdxListedLicense.getName())
.setText(spdxListedLicense.getLicenseText())
.setExternalLicenseLink("https://spdx.org/licenses/" + spdxListedLicense.getLicenseId()+ ".html");
.setExternalLicenseLink("https://spdx.org/licenses/" + spdxListedLicense.getLicenseId()+ ".html")
.setExternalIds(Collections.singletonMap("SPDX-License-Identifier", spdxListedLicense.getLicenseId()));
return Optional.of(license);
}

Expand Down
Binary file modified frontend/configuration/PrivatePages.lar
View file
Binary file not shown.
7 changes: 6 additions & 1 deletion frontend/sw360-portlet/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,12 @@
<artifactId>exporters</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.eclipse.sw360</groupId>
<artifactId>license-importer</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>org.webjars</groupId>
Expand Down
160 changes: 0 additions & 160 deletions ...portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/ComponentUploadPortlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,49 +13,36 @@

import com.google.common.base.Joiner;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.SetMultimap;
import com.liferay.portal.kernel.portlet.PortletResponseUtil;
import com.liferay.portal.kernel.upload.UploadPortletRequest;
import com.liferay.portal.util.PortalUtil;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.eclipse.sw360.commonIO.ConvertRecord;
import org.eclipse.sw360.datahandler.common.CommonUtils;
import org.eclipse.sw360.datahandler.thrift.ReleaseRelationship;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
import org.eclipse.sw360.datahandler.thrift.ThriftUtils;
import org.eclipse.sw360.datahandler.thrift.attachments.Attachment;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentService;
import org.eclipse.sw360.datahandler.thrift.components.Component;
import org.eclipse.sw360.datahandler.thrift.components.ComponentService;
import org.eclipse.sw360.datahandler.thrift.components.Release;
import org.eclipse.sw360.datahandler.thrift.licenses.*;
import org.eclipse.sw360.datahandler.thrift.users.User;
import org.eclipse.sw360.datahandler.thrift.vendors.VendorService;
import org.eclipse.sw360.exporter.CSVExport;
import org.eclipse.sw360.exporter.ZipTools;
import org.eclipse.sw360.importer.*;
import org.eclipse.sw360.portal.common.PortalConstants;
import org.eclipse.sw360.portal.common.UsedAsLiferayAction;
import org.eclipse.sw360.portal.portlets.Sw360Portlet;
import org.eclipse.sw360.portal.users.UserCacheHolder;
import org.jetbrains.annotations.NotNull;

import javax.portlet.*;
import java.io.*;
import java.util.*;
import java.util.function.Consumer;
import java.util.zip.ZipOutputStream;

import static org.eclipse.sw360.commonIO.ConvertRecord.*;
import static org.eclipse.sw360.commonIO.TypeMappings.*;
import static org.eclipse.sw360.datahandler.common.ImportCSV.readAsCSVRecords;
import static org.eclipse.sw360.exporter.ZipTools.*;
import static org.eclipse.sw360.importer.ComponentImportUtils.*;

/**
Expand Down Expand Up @@ -117,13 +104,6 @@ public void serveResource(ResourceRequest request, ResourceResponse response) th
log.error("Something went wrong with the CSV creation", e);
}
break;
case PortalConstants.DOWNLOAD_LICENSE_BACKUP:
try {
backUpLicenses(request, response);
} catch (IOException | TException e) {
log.error("Something went wrong with the license zip creation", e);
}
break;
}
}

Expand Down Expand Up @@ -276,67 +256,6 @@ public void backUpComponents(ResourceRequest request, ResourceResponse response)
PortletResponseUtil.sendFile(request, response, "ComponentsReleasesVendors.csv", byteArrayInputStream, "text/csv");
}

public void backUpLicenses(ResourceRequest request, ResourceResponse response) throws IOException, TException {
Map<String, InputStream> fileNameToStreams = getFilenameToCSVStreams();

final ByteArrayOutputStream outB = new ByteArrayOutputStream();
final ZipOutputStream zipOutputStream = new ZipOutputStream(outB);

for (Map.Entry<String, InputStream> entry : fileNameToStreams.entrySet()) {
ZipTools.addToZip(zipOutputStream, entry.getKey(), entry.getValue());
}

zipOutputStream.flush();
zipOutputStream.close(); // this closes outB

final ByteArrayInputStream zipFile = new ByteArrayInputStream(outB.toByteArray());
PortletResponseUtil.sendFile(request, response, "LicensesBackup.lics", zipFile, "application/zip");
}

@NotNull
private Map<String, InputStream> getFilenameToCSVStreams() throws TException, IOException {
Map<String, InputStream> fileNameToStreams = new HashMap<>();

final LicenseService.Iface client = thriftClients.makeLicenseClient();

fileNameToStreams.put(RISK_CATEGORY_FILE, getCsvStream(serialize(client.getRiskCategories(), riskCategorySerializer())));

fileNameToStreams.put(ZipTools.RISK_FILE, getCsvStream(serialize(client.getRisks(), riskSerializer())));
fileNameToStreams.put(ZipTools.OBLIGATION_FILE, getCsvStream(serialize(client.getObligations(), obligationSerializer())));

final List<Todo> todos = client.getTodos();
List<PropertyWithValueAndId> customProperties = new ArrayList<>();
SetMultimap<Integer, Integer> todoCustomPropertyMap = HashMultimap.create();
ConvertRecord.fillTodoCustomPropertyInfo(todos, customProperties, todoCustomPropertyMap);
fileNameToStreams.put(ZipTools.TODO_CUSTOM_PROPERTIES_FILE, getCsvStream(serialize(todoCustomPropertyMap, ImmutableList.of("T_ID", "P_ID"))));
fileNameToStreams.put(ZipTools.CUSTOM_PROPERTIES_FILE, getCsvStream(serialize(customProperties, customPropertiesSerializer())));
fileNameToStreams.put(ZipTools.OBLIGATION_TODO_FILE, getCsvStream(serialize(getTodoToObligationMap(todos), ImmutableList.of("O_ID", "T_ID"))));
fileNameToStreams.put(ZipTools.TODO_FILE, getCsvStream(serialize(todos, todoSerializer())));

fileNameToStreams.put(ZipTools.LICENSETYPE_FILE, getCsvStream(serialize(client.getLicenseTypes(), licenseTypeSerializer())));

final List<License> licenses = client.getLicenses();
fileNameToStreams.put(ZipTools.LICENSE_TODO_FILE, getCsvStream(serialize(getLicenseToTodoMap(licenses), ImmutableList.of("Identifier", "ID"))));
fileNameToStreams.put(ZipTools.LICENSE_RISK_FILE, getCsvStream(serialize(getLicenseToRiskMap(licenses), ImmutableList.of("Identifier", "ID"))));
fileNameToStreams.put(ZipTools.LICENSE_FILE, getCsvStream(serialize(licenses, licenseSerializer())));
return fileNameToStreams;
}

@NotNull
private ByteArrayOutputStream writeCsvStream(List<List<String>> listList) throws TException, IOException {
final ByteArrayOutputStream riskCategoryCsvStream = new ByteArrayOutputStream();
Writer out = new BufferedWriter(new OutputStreamWriter(riskCategoryCsvStream));
CSVPrinter csvPrinter = new CSVPrinter(out, CommonUtils.sw360CsvFormat);
csvPrinter.printRecords(listList);
csvPrinter.flush();
csvPrinter.close();
return riskCategoryCsvStream;
}

private ByteArrayInputStream getCsvStream(List<List<String>> listList) throws TException, IOException {
return new ByteArrayInputStream(writeCsvStream(listList).toByteArray());
}

@UsedAsLiferayAction
public void updateComponents(ActionRequest request, ActionResponse response) throws PortletException, IOException, TException {
List<CSVRecord> releaseRecords = getCSVFromRequest(request, "file");
Expand Down Expand Up @@ -388,83 +307,4 @@ public void updateReleaseLinks(ActionRequest request, ActionResponse response) t

renderRequestSummary(request, response, requestSummary);
}

@UsedAsLiferayAction
public void updateLicenses(ActionRequest request, ActionResponse response) throws PortletException, IOException, TException {
final HashMap<String, InputStream> inputMap = new HashMap<>();
User user = UserCacheHolder.getUserFromRequest(request);
try {
fillFilenameInputStreamMap(request, inputMap);
if (ZipTools.isValidLicenseArchive(inputMap)) {

final LicenseService.Iface licenseClient = thriftClients.makeLicenseClient();

log.debug("Parsing risk categories ...");
Map<Integer, RiskCategory> riskCategoryMap = getIdentifierToTypeMapAndWriteMissingToDatabase(licenseClient,
inputMap.get(RISK_CATEGORY_FILE), RiskCategory.class, Integer.class, user);

log.debug("Parsing risks ...");
Map<Integer, Risk> riskMap = getIntegerRiskMap(licenseClient, riskCategoryMap, inputMap.get(RISK_FILE), user);

log.debug("Parsing obligations ...");
Map<Integer, Obligation> obligationMap = getIdentifierToTypeMapAndWriteMissingToDatabase(licenseClient,
inputMap.get(OBLIGATION_FILE), Obligation.class, Integer.class, user);

log.debug("Parsing obligation todos ...");
List<CSVRecord> obligationTodoRecords = readAsCSVRecords(inputMap.get(OBLIGATION_TODO_FILE));
Map<Integer, Set<Integer>> obligationTodoMapping = convertRelationalTableWithIntegerKeys(obligationTodoRecords);

log.debug("Parsing license types ...");
Map<Integer, LicenseType> licenseTypeMap = getIdentifierToTypeMapAndWriteMissingToDatabase(licenseClient,
inputMap.get(LICENSETYPE_FILE), LicenseType.class, Integer.class, user);

log.debug("Parsing todos ...");
Map<Integer, Todo> todoMap = getTodoMapAndWriteMissingToDatabase(licenseClient, obligationMap, obligationTodoMapping, inputMap.get(TODO_FILE), user);

if(inputMap.containsKey(CUSTOM_PROPERTIES_FILE)) {
log.debug("Parsing custom properties ...");
Map<Integer, PropertyWithValue> customPropertiesMap =
getCustomPropertiesWithValuesByIdAndWriteMissingToDatabase(licenseClient, inputMap.get(CUSTOM_PROPERTIES_FILE), user);

log.debug("Parsing todo custom properties relation ...");
List<CSVRecord> todoPropertiesRecord = readAsCSVRecords(inputMap.get(TODO_CUSTOM_PROPERTIES_FILE));
Map<Integer, Set<Integer>> todoPropertiesMap = convertRelationalTableWithIntegerKeys(todoPropertiesRecord);

todoMap = updateTodoMapWithCustomPropertiesAndWriteToDatabase(licenseClient, todoMap, customPropertiesMap, todoPropertiesMap, user);
}

log.debug("Parsing license todos ...");
List<CSVRecord> licenseTodoRecord = readAsCSVRecords(inputMap.get(LICENSE_TODO_FILE));
Map<String, Set<Integer>> licenseTodoMap = convertRelationalTable(licenseTodoRecord);


log.debug("Parsing license risks ...");
List<CSVRecord> licenseRiskRecord = readAsCSVRecords(inputMap.get(LICENSE_RISK_FILE));
Map<String, Set<Integer>> licenseRiskMap = convertRelationalTable(licenseRiskRecord);

log.debug("Parsing licenses ...");
List<CSVRecord> licenseRecord = readAsCSVRecords(inputMap.get(LICENSE_FILE));

final List<License> licensesToAdd = ConvertRecord.fillLicenses(licenseRecord, licenseTypeMap, todoMap, riskMap, licenseTodoMap, licenseRiskMap);
addLicenses(licenseClient, licensesToAdd, log, user);

} else {
throw new SW360Exception("Invalid file format");
}
} finally {
for (InputStream inputStream : inputMap.values()) {
inputStream.close();
}
}
}

private void fillFilenameInputStreamMap(ActionRequest request, HashMap<String, InputStream> fileNameToStream) throws IOException {
InputStream in = null;
try {
in = getInputStreamFromRequest(request, "file");
ZipTools.extractZipToInputStreamMap(in, fileNameToStream);
} finally {
if (in != null) in.close();
}
}
}
31 changes: 0 additions & 31 deletions ...360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/DatabaseSanitation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,10 @@
package org.eclipse.sw360.portal.portlets.admin;

import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.components.ComponentService;
import org.eclipse.sw360.datahandler.thrift.licenses.License;
import org.eclipse.sw360.datahandler.thrift.licenses.LicenseService;
import org.eclipse.sw360.datahandler.thrift.projects.ProjectService;
import org.eclipse.sw360.datahandler.thrift.users.User;
import org.eclipse.sw360.portal.common.PortalConstants;
import org.eclipse.sw360.portal.portlets.Sw360Portlet;
import org.eclipse.sw360.portal.users.UserCacheHolder;
import org.apache.log4j.Logger;
import org.apache.thrift.TException;

Expand Down Expand Up @@ -50,14 +45,6 @@ public void serveResource(ResourceRequest request, ResourceResponse response) th
String action = request.getParameter(PortalConstants.ACTION);
if (PortalConstants.DUPLICATES.equals(action)) {
serveDuplicates(request,response);
}else if(PortalConstants.ACTION_DELETE_ALL_LICENSE_INFORMATION.equals(action)){
deleteAllLicenseInformation(request, response);
}else if(PortalConstants.ACTION_IMPORT_SPDX_LICENSE_INFORMATION.equals(action)){
try {
importLicensesFromSPDX(request, response);
} catch (TException e) {
throw new PortletException(e);
}
}
}

Expand Down Expand Up @@ -90,22 +77,4 @@ private void serveDuplicates(ResourceRequest request, ResourceResponse response)
include("/html/admin/databaseSanitation/duplicatesAjax.jsp", request, response, PortletRequest.RESOURCE_PHASE);
}
}

private void deleteAllLicenseInformation(ResourceRequest request, ResourceResponse response){
User user = UserCacheHolder.getUserFromRequest(request);
LicenseService.Iface licenseClient = thriftClients.makeLicenseClient();
try {
RequestSummary requestSummary = licenseClient.deleteAllLicenseInformation(user);
renderRequestSummary(request, response, requestSummary);
} catch (TException te){
log.error("Got TException when trying to delete all license information." ,te);
}
}

private void importLicensesFromSPDX(ResourceRequest request, ResourceResponse response) throws TException {
User user = UserCacheHolder.getUserFromRequest(request);
LicenseService.Iface licenseClient = thriftClients.makeLicenseClient();
RequestSummary requestSummary = licenseClient.importAllSpdxLicenses(user);
renderRequestSummary(request, response, requestSummary);
}
}
Loading

0 comments on commit a20a225

Please sign in to comment.