feat(search): Improve lucene search logic and handling of results

- disable automatic column order in case of advanced searches
- enables leading wildcard searches

Signed-off-by: Thomas Maier <thomas.maier@evosoft.com>

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/VendorSearchHandler.java

@@ -1,5 +1,5 @@
 /*
- * Copyright Siemens AG, 2013-2015. Part of the SW360 Portal Project.
+ * Copyright Siemens AG, 2013-2018. Part of the SW360 Portal Project.
  *
  * SPDX-License-Identifier: EPL-1.0
  *
@@ -17,7 +17,6 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Set;
 
 import static org.eclipse.sw360.datahandler.couchdb.lucene.LuceneAwareDatabaseConnector.prepareWildcardQuery;
 
@@ -54,7 +53,7 @@ public List<Vendor> search(String searchText) {
         return connector.searchView(Vendor.class, luceneSearchView, prepareWildcardQuery(searchText));
     }
 
-    public Set<String> searchIds(String searchText) {
+    public List<String> searchIds(String searchText) {
         // Query the search view for the provided text
         return connector.searchIds(Vendor.class, luceneSearchView, prepareWildcardQuery(searchText));
     }

backend/src/src-vendors/src/main/java/org/eclipse/sw360/vendors/VendorHandler.java

@@ -1,5 +1,5 @@
 /*
- * Copyright Siemens AG, 2013-2015. Part of the SW360 Portal Project.
+ * Copyright Siemens AG, 2013-2018. Part of the SW360 Portal Project.
  *
  * SPDX-License-Identifier: EPL-1.0
  *
@@ -76,7 +76,7 @@ public List<Vendor> searchVendors(String searchText) throws TException {
     }
 
     @Override
-    public Set<String> searchVendorIds(String searchText) throws TException {
+    public List<String> searchVendorIds(String searchText) throws TException {
         return vendorSearchHandler.searchIds(searchText);
     }
 

build-configuration/resources/couchdb.properties

@@ -1,5 +1,5 @@
 #
-# Copyright Siemens AG, 2013-2016. Part of the SW360 Portal Project.
+# Copyright Siemens AG, 2013-2018. Part of the SW360 Portal Project.
 #
 # All rights reserved. This configuration file is provided to you under the
 # terms and conditions of the Eclipse Distribution License v1.0 which
@@ -18,3 +18,13 @@ couchdb.attachments = sw360attachments
 couchdb.fossologyKeys = sw360fossologykeys
 couchdb.vulnerability_management = sw360vm
 lucenesearch.limit = 150
+
+# Warning: If you enable lucene leading wildcards you have to enable this configuration also in couchdb-lucene.ini
+# leading wildcard search is disabled as default because its a expensive operation.
+# couchdb-lucene.ini (is part of the couchdb-lucene .war package)
+# [lucene]
+# allowLeadingWildcard=true
+# see more: https://wiki.apache.org/lucene-java/LuceneFAQ#What_wildcard_search_support_is_available_from_Lucene.3F
+lucenesearch.leading.wildcard = false
+
+

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/datatables/PaginationParser.java

@@ -10,15 +10,17 @@
  */
 package org.eclipse.sw360.portal.common.datatables;
 
+import org.apache.log4j.Logger;
 import org.eclipse.sw360.portal.common.datatables.data.PaginationParameters;
 
 import javax.servlet.http.HttpServletRequest;
 import java.util.Map;
+import java.util.Optional;
 
-import static org.eclipse.sw360.portal.common.PortalConstants.*;
 import static org.eclipse.sw360.portal.common.datatables.DataTablesUtils.getSimple;
 import static org.eclipse.sw360.portal.common.datatables.DataTablesUtils.getSimpleInt;
 
+
 /**
  * @author thomas.maier@evosoft.com
  */
@@ -34,20 +36,36 @@ public class PaginationParser {
     private static final String DATATABLE_SORT_DIRECTION = "sSortDir_0";
     private static final String DATATABLE_SORT_ASC = "asc";
 
+    private static final Logger log = Logger.getLogger(PaginationParser.class);
+
     protected static PaginationParameters parametersFrom(Map<String, String[]> parameterMap) {
         PaginationParameters paginationParameters = new PaginationParameters();
         paginationParameters.setEcho(getSimple(parameterMap, DATATABLE_ECHO));
         paginationParameters.setColumnNames(getSimple(parameterMap, DATATABLE_COLUMNS));
         paginationParameters.setColumnCount(getSimpleInt(parameterMap, DATATABLE_COLUMN_COUNT));
         paginationParameters.setDisplayLength(getSimpleInt(parameterMap, DATATABLE_DISPLAY_LENGTH));
         paginationParameters.setDisplayStart(getSimpleInt(parameterMap, DATATABLE_DISPLAY_START));
-        paginationParameters.setSortingColumn(getSimpleInt(parameterMap, DATATABLE_SORT_COLUMN));
         paginationParameters.setAscending(isAscendingSortOrder(parameterMap, DATATABLE_SORT_DIRECTION));
+        paginationParameters.setSortingColumn(getColumnSortIndexByParameterMap(parameterMap, DATATABLE_SORT_COLUMN));
         return paginationParameters;
     }
 
-    private static boolean isAscendingSortOrder(Map<String, String[]> parameterMap, String parameterName) {
-        return getSimple(parameterMap, parameterName).equals(DATATABLE_SORT_ASC);
+    private static Optional<Boolean> isAscendingSortOrder(Map<String, String[]> parameterMap, String parameterName) {
+        try {
+            return Optional.ofNullable(getSimple(parameterMap, parameterName).equals(DATATABLE_SORT_ASC));
+        } catch (IllegalArgumentException e) {
+            log.debug("Ascending sort order is not set because of initial load of data");
+            return Optional.empty(); // initial load of data should not destroy the origin sort order (lucene search)
+        }
+    }
+
+    private static Optional<Integer> getColumnSortIndexByParameterMap(Map<String, String[]> parameterMap, String parameterName) {
+        try {
+            return Optional.ofNullable(getSimpleInt(parameterMap, parameterName));
+        } catch (IllegalArgumentException e) {
+            log.debug("Column sort index value is not set because of initial load of data");
+            return Optional.empty(); // initial load of data should not destroy the sort origin order (lucene search)
+        }
     }
 
     public static PaginationParameters parametersFrom(HttpServletRequest request) {

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/datatables/data/PaginationParameters.java

@@ -10,6 +10,8 @@
  */
 package org.eclipse.sw360.portal.common.datatables.data;
 
+import java.util.Optional;
+
 /**
  * @author thomas.maier@evosoft.com
  */
@@ -43,18 +45,18 @@ public class PaginationParameters {
     /**
      * Column index being sorted on
      */
-    private int sortingColumn;
+    private Optional<Integer> sortingColumn;
 
     /**
      * Direction to be sorted - ascending or descending order
      */
-    private boolean ascending;
+    private Optional<Boolean> ascending;
 
 
     public PaginationParameters() {
     }
 
-    public PaginationParameters(String echo, String columnNames, int displayLength, int displayStart, int columnCount, int sortingColumn, boolean ascending) {
+    public PaginationParameters(String echo, String columnNames, int displayLength, int displayStart, int columnCount, Optional<Integer> sortingColumn, Optional<Boolean> ascending) {
         this.echo = echo;
         this.columnNames = columnNames;
         this.displayLength = displayLength;
@@ -104,19 +106,19 @@ public void setColumnCount(int columnCount) {
         this.columnCount = columnCount;
     }
 
-    public int getSortingColumn() {
+    public Optional<Integer> getSortingColumn() {
         return sortingColumn;
     }
 
-    public void setSortingColumn(int sortingColumn) {
+    public void setSortingColumn(Optional<Integer> sortingColumn) {
         this.sortingColumn = sortingColumn;
     }
 
-    public boolean isAscending() {
+    public Optional<Boolean> isAscending() {
         return ascending;
     }
 
-    public void setAscending(boolean ascending) {
+    public void setAscending(Optional<Boolean> ascending) {
         this.ascending = ascending;
     }
 }

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/Sw360Portlet.java

@@ -15,6 +15,7 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.Predicate;
 import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Sets;
 import com.liferay.portal.kernel.json.JSONFactoryUtil;
 import com.liferay.portal.kernel.json.JSONObject;
 import com.liferay.portal.kernel.servlet.SessionErrors;
@@ -39,7 +40,10 @@
 
 import javax.portlet.*;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 
@@ -338,9 +342,9 @@ protected List<Release> serveReleaseListBySearchText(String searchText, User use
             } else {
                 List<Release> searchResult = componentClient.searchReleases(searchText);
                 final VendorService.Iface vendorClient = thriftClients.makeVendorClient();
-                final Set<String> vendorIds = vendorClient.searchVendorIds(searchText);
+                List<String> vendorIds = vendorClient.searchVendorIds(searchText);
                 if (vendorIds != null && vendorIds.size() > 0) {
-                    searchResult.addAll(componentClient.getReleasesFromVendorIds(vendorIds));
+                    searchResult.addAll(componentClient.getReleasesFromVendorIds(Sets.newHashSet(vendorIds)));
                 }
                 return searchResult;
             }

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java

@@ -97,6 +97,7 @@ public class ComponentPortlet extends FossologyAwarePortlet {
     private static final TSerializer JSON_THRIFT_SERIALIZER = new TSerializer(new TSimpleJSONProtocol.Factory());
 
     // Component view datatables, index of columns
+    private static final int COMPONENT_NO_SORT = -1;
     private static final int COMPONENT_DT_ROW_VENDOR = 0;
     private static final int COMPONENT_DT_ROW_NAME = 1;
     private static final int COMPONENT_DT_ROW_MAIN_LICENSES = 2;
@@ -1247,9 +1248,10 @@ private void updateVulnerabilityVerification(ResourceRequest request, ResourceRe
     private void serveComponentList(ResourceRequest request, ResourceResponse response) throws PortletException {
         HttpServletRequest originalServletRequest = PortalUtil.getOriginalServletRequest(PortalUtil.getHttpServletRequest(request));
         PaginationParameters paginationParameters = PaginationParser.parametersFrom(originalServletRequest);
+        handlePaginationSortOrder(request, paginationParameters);
         List<Component> componentList = getFilteredComponentList(request);
-        JSONArray jsonComponents = getComponentData(componentList, paginationParameters);
 
+        JSONArray jsonComponents = getComponentData(componentList, paginationParameters);
         JSONObject jsonResult = createJSONObject();
         jsonResult.put(DATATABLE_RECORDS_TOTAL, componentList.size());
         jsonResult.put(DATATABLE_RECORDS_FILTERED, componentList.size());
@@ -1263,6 +1265,17 @@ private void serveComponentList(ResourceRequest request, ResourceResponse respon
         }
     }
 
+    private void handlePaginationSortOrder(ResourceRequest request, PaginationParameters paginationParameters) {
+        if (!paginationParameters.getSortingColumn().isPresent()) {
+            for (Component._Fields filteredField : componentFilteredFields) {
+                if (!isNullOrEmpty(request.getParameter(filteredField.toString()))) {
+                    paginationParameters.setSortingColumn(Optional.of(COMPONENT_NO_SORT));
+                    break;
+                }
+            }
+        }
+    }
+
     public JSONArray getComponentData(List<Component> componentList, PaginationParameters componentParameters) {
         List<Component> sortedComponents = sortComponentList(componentList, componentParameters);
         int count = getComponentDataCount(componentParameters, componentList.size());
@@ -1305,22 +1318,23 @@ private int getComponentDataCount(PaginationParameters componentParameters, int
     }
 
     private List<Component> sortComponentList(List<Component> componentList, PaginationParameters componentParameters) {
+        boolean isAsc = componentParameters.isAscending().orElse(true);
 
-        switch (componentParameters.getSortingColumn()) {
+        switch (componentParameters.getSortingColumn().orElse(COMPONENT_DT_ROW_NAME)) {
             case COMPONENT_DT_ROW_VENDOR:
-                Collections.sort(componentList, compareByVendor(componentParameters.isAscending()));
+                Collections.sort(componentList, compareByVendor(isAsc));
                 break;
             case COMPONENT_DT_ROW_NAME:
-                Collections.sort(componentList, compareByName(componentParameters.isAscending()));
+                Collections.sort(componentList, compareByName(isAsc));
                 break;
             case COMPONENT_DT_ROW_MAIN_LICENSES:
-                Collections.sort(componentList, compareByMainLicenses(componentParameters.isAscending()));
+                Collections.sort(componentList, compareByMainLicenses(isAsc));
                 break;
             case COMPONENT_DT_ROW_TYPE:
-                Collections.sort(componentList, compareByComponentType(componentParameters.isAscending()));
+                Collections.sort(componentList, compareByComponentType(isAsc));
                 break;
             case COMPONENT_DT_ROW_ACTION:
-                Collections.sort(componentList, compareById(componentParameters.isAscending()));
+                Collections.sort(componentList, compareById(isAsc));
                 break;
             default:
                 break;

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/projects/ProjectPortlet.java

@@ -98,6 +98,7 @@ public class ProjectPortlet extends FossologyAwarePortlet {
     private static final String LICENSE_NAME_WITH_TEXT_FILE = "file";
 
     // Project view datatables, index of columns
+    private static final int PROJECT_NO_SORT = -1;
     private static final int PROJECT_DT_ROW_NAME = 0;
     private static final int PROJECT_DT_ROW_DESCRIPTION = 1;
     private static final int PROJECT_DT_ROW_RESPONSIBLE = 2;
@@ -1147,6 +1148,7 @@ private String getProjectDefaultLicenseInfoHeaderText() {
     private void serveProjectList(ResourceRequest request, ResourceResponse response) throws IOException, PortletException {
         HttpServletRequest originalServletRequest = PortalUtil.getOriginalServletRequest(PortalUtil.getHttpServletRequest(request));
         PaginationParameters paginationParameters = PaginationParser.parametersFrom(originalServletRequest);
+        handlePaginationSortOrder(request, paginationParameters);
         List<Project> projectList = getFilteredProjectList(request);
 
         JSONArray jsonProjects = getProjectData(projectList, paginationParameters);
@@ -1162,6 +1164,17 @@ private void serveProjectList(ResourceRequest request, ResourceResponse response
         }
     }
 
+    private void handlePaginationSortOrder(ResourceRequest request, PaginationParameters paginationParameters) {
+        if (!paginationParameters.getSortingColumn().isPresent()) {
+            for (Project._Fields filteredField : projectFilteredFields) {
+                if (!isNullOrEmpty(request.getParameter(filteredField.toString()))) {
+                    paginationParameters.setSortingColumn(Optional.of(PROJECT_NO_SORT));
+                    break;
+                }
+            }
+        }
+    }
+
     public JSONArray getProjectData(List<Project> projectList, PaginationParameters projectParameters) {
         List<Project> sortedProjects = sortProjectList(projectList, projectParameters);
         int count = getProjectDataCount(projectParameters, projectList.size());
@@ -1196,19 +1209,20 @@ private int getProjectDataCount(PaginationParameters projectParameters, int maxS
     }
 
     private List<Project> sortProjectList(List<Project> projectList, PaginationParameters projectParameters) {
+        boolean isAsc = projectParameters.isAscending().orElse(true);
 
-        switch (projectParameters.getSortingColumn()) {
+        switch (projectParameters.getSortingColumn().orElse(PROJECT_DT_ROW_NAME)) {
             case PROJECT_DT_ROW_NAME:
-                Collections.sort(projectList, compareByName(projectParameters.isAscending()));
+                Collections.sort(projectList, compareByName(isAsc));
                 break;
             case PROJECT_DT_ROW_DESCRIPTION:
-                Collections.sort(projectList, compareByDescription(projectParameters.isAscending()));
+                Collections.sort(projectList, compareByDescription(isAsc));
                 break;
             case PROJECT_DT_ROW_RESPONSIBLE:
-                Collections.sort(projectList, compareByResponsible(projectParameters.isAscending()));
+                Collections.sort(projectList, compareByResponsible(isAsc));
                 break;
             case PROJECT_DT_ROW_STATE:
-                Collections.sort(projectList, compareByState(projectParameters.isAscending()));
+                Collections.sort(projectList, compareByState(isAsc));
                 break;
             case PROJECT_DT_ROW_CLEARING_STATE:
                 break;

frontend/sw360-portlet/src/main/webapp/html/components/view.jsp

@@ -270,7 +270,7 @@
                         {"title": "Component Type", data: "cType"},
                         {"title": "Actions", data: "id", render: {display: renderComponentActions}}
                     ],
-                    "order": [[1, 'asc']],
+                    "aaSorting": [],
                     "lengthMenu": [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
                     "autoWidth": false
                 });

frontend/sw360-portlet/src/main/webapp/html/projects/view.jsp

@@ -326,7 +326,7 @@
                         {title: "<span title=\"Release clearing state\">Clearing Status</span>", data: "clearing"},
                         {title: "Actions", data: "id", render: {display: renderProjectActions}}
                     ],
-                    "order": [[0, 'asc']],
+                    "aaSorting": [],
                     "lengthMenu": [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
                     "autoWidth": false,
                     "fnDrawCallback": function (oSettings) {

frontend/sw360-portlet/src/main/webapp/html/utils/includes/searchReleases.jsp

@@ -101,6 +101,7 @@
                     "bFilter": false,
                     "language": {"processing":     "Processing..."},
                     "processing": true,
+                    "aaSorting": [],
                     "initComplete": toggleReleasesSearchNotification
                 });
         }