Check if Library Id (GAV) is well known

rest-backend/src/main/java/com/sap/psr/vulas/backend/model/Library.java

@@ -23,6 +23,7 @@
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import javax.persistence.Column;
@@ -406,9 +407,6 @@ public void prePersist() {
 		if(this.getModifiedAt()==null) {
 			this.setModifiedAt(Calendar.getInstance());
 		}
-		if(this.getWellknownDigest()==null) {
-			this.verifyDigest();
-		}
 		// If uploaded by old client (<3.0.0), the digest also must be set
 		if(this.sha1!=null && (this.digest==null || this.digestAlgorithm==null)) {
 			this.digest = this.sha1;
@@ -478,17 +476,25 @@ public final String toString(boolean _deep) {
 
 	//changed to public temporarily to recreate wellknownDigest flag for already persisted libs
 	/**
-	 * <p>verifyDigest.</p>
+	 * Verifies that the digest is well known by either one of the existing
+	 * verifiers (e.g. maven central, pypi). If the digests is well known and the
+	 * provided libraryId is NOT among those returned by the verifiers, it sets the
+	 * LibraryId with the (first) value returned by the verifier.
+	 * 
 	 */
 	public void verifyDigest() {
 		if(this.getWellknownDigest()==null || this.getDigestVerificationUrl()==null || this.getDigestTimestamp()==null) {
 			try {
 				final DigestVerifierEnumerator dv = new DigestVerifierEnumerator();
-				final Boolean verified = dv.verify(this);
+				final List<LibraryId> verified = dv.verify(this);
 				if(verified!=null) {
 					this.setDigestVerificationUrl(dv.getVerificationUrl());
-					this.setWellknownDigest(verified);
+					this.setWellknownDigest(verified.size()>0);
 					this.setDigestTimestamp(dv.getReleaseTimestamp());
+
+					if(this.getLibraryId()!=null && verified.size()>0 && !verified.contains(this.getLibraryId())) {
+						this.setLibraryId(verified.get(0));
+					}
 				}
 			} catch (VerificationException e) {
 				log.error(e.getMessage());

rest-backend/src/main/java/com/sap/psr/vulas/backend/repo/LibraryRepositoryImpl.java

@@ -21,7 +21,6 @@
 
 import java.nio.file.Path;
 import java.util.Calendar;
-import java.util.Collection;
 
 import javax.persistence.EntityNotFoundException;
 import javax.persistence.PersistenceException;
@@ -87,17 +86,7 @@ public Library customSave(Library _lib) {
 			_lib.setId(managed_lib.getId());
 			_lib.setCreatedAt(managed_lib.getCreatedAt());
 			_lib.setModifiedAt(Calendar.getInstance());
-
-			//	Re-create wellknownDigest if it is null in our current db (this part should be removed once it is created for all)
-			if(managed_lib.getWellknownDigest()==null || (managed_lib.getDigestTimestamp()==null && managed_lib.getWellknownDigest())) {
-				_lib.verifyDigest();
-			}
-			else {
-				_lib.setWellknownDigest(managed_lib.getWellknownDigest());
-				_lib.setDigestVerificationUrl(managed_lib.getDigestVerificationUrl());
-				_lib.setDigestTimestamp(managed_lib.getDigestTimestamp());
-			}
-
+
 			//keep the existing lib GAV if the newly posted/put doesn't have one
 			if(_lib.getLibraryId()==null){
 				// Recreate libId if null (to be removed once we ensure we added to the existing libs)
@@ -124,6 +113,11 @@ public Library customSave(Library _lib) {
 
 		_lib.setBundledLibraryIds(refUpdater.saveNestedBundledLibraryIds(_lib.getBundledLibraryIds()));
 
+		// we always verify the digest to make sure all fields are filled and check that
+		// libraryId provided by the client is known (replaced with known one otherwise)
+		// Note that this replaces digestVerificationUrl, digestTimestamp and WellknownDigest
+		_lib.verifyDigest();
+
 		_lib = this.saveNestedLibraryId(_lib);
 
 		// Save
@@ -169,6 +163,7 @@ private Library saveNestedLibraryId(@NotNull Library _lib) {
 	/** {@inheritDoc} */
 	public Library saveIncomplete(Library _lib) throws PersistenceException {
 		Library incomplete = new Library(_lib.getDigest());
+		incomplete.setDigestAlgorithm(_lib.getDigestAlgorithm());
 		if (_lib.getLibraryId()!=null){
 			LibraryRepositoryImpl.log.debug("Setting library Id ["+_lib.getLibraryId().toString()+"] of incomplete library [" +_lib.getDigest()+ "]");
 			incomplete.setLibraryId(_lib.getLibraryId());

rest-backend/src/main/java/com/sap/psr/vulas/backend/util/DigestVerifier.java

@@ -19,9 +19,11 @@
  */
 package com.sap.psr.vulas.backend.util;
 
+import java.util.List;
 import java.util.Set;
 
 import com.sap.psr.vulas.backend.model.Library;
+import com.sap.psr.vulas.backend.model.LibraryId;
 import com.sap.psr.vulas.shared.enums.DigestAlgorithm;
 import com.sap.psr.vulas.shared.enums.ProgrammingLanguage;
 
@@ -64,11 +66,11 @@ public interface DigestVerifier {
 
 	/**
 	 * Returns null if the verification did not succeed, e.g., due to connectivity issues.
-	 * Returns true or false depending on whether the digest is known to the respective package repo.
+	 * Returns the list of verified LibraryId if the digest is known to the respective package repo (an empty list otherwise).
 	 *
 	 * @throws com.sap.psr.vulas.backend.util.VerificationException if the verification URL could not be reached, the HTTP response was malformed or similar
 	 * @param _lib a {@link com.sap.psr.vulas.backend.model.Library} object.
 	 * @return a {@link java.lang.Boolean} object.
 	 */
-	public Boolean verify(Library _lib) throws VerificationException;
+	public List<LibraryId> verify(Library _lib) throws VerificationException;
 }

rest-backend/src/main/java/com/sap/psr/vulas/backend/util/DigestVerifierEnumerator.java

@@ -20,13 +20,15 @@
 package com.sap.psr.vulas.backend.util;
 
 import java.util.HashSet;
+import java.util.List;
 import java.util.ServiceLoader;
 import java.util.Set;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.sap.psr.vulas.backend.model.Library;
+import com.sap.psr.vulas.backend.model.LibraryId;
 import com.sap.psr.vulas.shared.enums.DigestAlgorithm;
 import com.sap.psr.vulas.shared.enums.ProgrammingLanguage;
 import com.sap.psr.vulas.shared.util.CollectionUtil;
@@ -78,12 +80,12 @@ public Set<DigestAlgorithm> getSupportedDigestAlgorithms() {
 	 *
 	 * Loops over available implementations of {@link DigestVerifier} in order to verify the digest of a given {@link Library}.
 	 */
-	public Boolean verify(Library _lib) throws VerificationException {
+	public List<LibraryId> verify(Library _lib) throws VerificationException {
 		if(_lib==null || _lib.getDigest()==null)
 			throw new IllegalArgumentException("No library or digest provided: [" + _lib + "]");
 
-		// Will only have a true or false value if either one verifier returns true, or all returned false (thus, no exception happened)
-		Boolean verified = null;
+		// Will only have a list of LibraryIds or an empty list if either one verifier returns (thus, no exception happened)
+		List<LibraryId> verified = null;
 		int exception_count = 0;
 
 		// Perform the loop
@@ -96,7 +98,7 @@ public Boolean verify(Library _lib) throws VerificationException {
 					l.getSupportedDigestAlgorithms().contains(_lib.getDigestAlgorithm())) {
 				try {
 					verified = l.verify(_lib);
-					if(verified!=null && verified) {
+					if(verified!=null && verified.size()>0) {
 						this.url = l.getVerificationUrl();
 						this.timestamp = l.getReleaseTimestamp();
 						break;

rest-backend/src/main/java/com/sap/psr/vulas/backend/util/MavenCentralVerifier.java

@@ -19,8 +19,10 @@
  */
 package com.sap.psr.vulas.backend.util;
 
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.TimeZone;
 
@@ -33,11 +35,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.jayway.jsonpath.JsonPath;
 import com.sap.psr.vulas.backend.model.Library;
 import com.sap.psr.vulas.backend.model.LibraryId;
 import com.sap.psr.vulas.shared.enums.DigestAlgorithm;
 import com.sap.psr.vulas.shared.enums.ProgrammingLanguage;
+import com.sap.psr.vulas.shared.json.JacksonUtil;
+import com.sap.psr.vulas.shared.json.model.mavenCentral.MavenVersionsSearch;
+import com.sap.psr.vulas.shared.json.model.mavenCentral.ResponseDoc;
 
 /**
  * <p>MavenCentralVerifier class.</p>
@@ -83,14 +87,14 @@ public Set<DigestAlgorithm> getSupportedDigestAlgorithms() {
 
 	/** {@inheritDoc} */
 	@Override
-	public Boolean verify(final Library _lib) throws VerificationException {
+	public List<LibraryId> verify(final Library _lib) throws VerificationException {
 		if(_lib==null || _lib.getDigest()==null)
 			throw new IllegalArgumentException("No library or digest provided: [" + _lib + "]");
 
 		this.url = new String("http://search.maven.org/solrsearch/select?q=1:<SHA1>&rows=20&wt=json").replaceAll("<SHA1>", _lib.getDigest());
 
 		String mvnResponse = null;
-		Boolean verified = null;
+		List<LibraryId> verified_lids = null;
 		int sc = -1;
 		try {
 			final CloseableHttpClient httpclient = HttpClients.createDefault();
@@ -103,29 +107,35 @@ public Boolean verify(final Library _lib) throws VerificationException {
 				HttpEntity entity = response.getEntity();
 				if (sc==HttpStatus.SC_OK && entity != null) {
 					mvnResponse = ConnectionUtil.readInputStream(entity.getContent());
-					int num_found = ((Integer)JsonPath.read(mvnResponse, "$.response.numFound")).intValue();
-					verified = num_found > 0;
+					MavenVersionsSearch json_response = (MavenVersionsSearch) JacksonUtil.asObject(mvnResponse, MavenVersionsSearch.class);
+					long num_found = json_response.getResponse().getNumFound();
 
-					if(num_found==1) {
-						final long ms = (Long)JsonPath.read(mvnResponse, "$.response.docs[0].timestamp");
-						this.timestamp = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
-						this.timestamp.setTimeInMillis(ms);						
-
-						// Check whether given and returned libid correspond
-						final LibraryId returned_libid = new LibraryId((String)JsonPath.read(mvnResponse, "$.response.docs[0].g"),(String)JsonPath.read(mvnResponse, "$.response.docs[0].a"),(String)JsonPath.read(mvnResponse, "$.response.docs[0].v"));
-						if(_lib.getLibraryId()!=null && !_lib.getLibraryId().equals(returned_libid))
-							log.warn("Given and returned library identifiers do not match: Given [" + _lib.getLibraryId() + "], returned [" + returned_libid + "]");
-					}
-					else if(num_found>1) {
+					verified_lids = new ArrayList<LibraryId>();
+					if(num_found>1) {
 						log.warn("The lookup of SHA1 digest [" + _lib.getDigest() + "] in Maven Central returned [" + num_found + "] artifacts");
 					}
+					if(num_found>=1) {
+
+						for(ResponseDoc d: json_response.getResponse().getSortedDocs()) {
+							LibraryId current = new LibraryId(d.getG(),d.getA(),d.getV());
+							verified_lids.add(current);
+							// take timestamp of the artifact corresponding to the provided libraryId or the
+							// first one as that's the one we will use in case the provided one is not among
+							// the list of verified 
+							if(this.timestamp==null || current.equals(_lib.getLibraryId())) {
+								final long ms = d.getTimestamp();;
+								this.timestamp = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
+								this.timestamp.setTimeInMillis(ms);						
+							}
+						}	
+					}
 				}
 			} finally {
 				response.close();
 			}
 		} catch (Exception e) {
 			throw new VerificationException(_lib, this.url, e);
 		}
-		return verified;
+		return verified_lids;
 	}
 }

rest-backend/src/main/java/com/sap/psr/vulas/backend/util/PyPiVerifier.java

@@ -21,6 +21,7 @@
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashSet;
@@ -39,6 +40,7 @@
 
 import com.jayway.jsonpath.JsonPath;
 import com.sap.psr.vulas.backend.model.Library;
+import com.sap.psr.vulas.backend.model.LibraryId;
 import com.sap.psr.vulas.shared.enums.DigestAlgorithm;
 import com.sap.psr.vulas.shared.enums.ProgrammingLanguage;
 
@@ -96,7 +98,7 @@ public Set<DigestAlgorithm> getSupportedDigestAlgorithms() {
 
 	/** {@inheritDoc} */
 	@Override
-	public Boolean verify(final Library _lib) throws VerificationException {
+	public List<LibraryId> verify(final Library _lib) throws VerificationException {
 		if(_lib==null || _lib.getDigest()==null)
 			throw new IllegalArgumentException("No library or digest provided: [" + _lib + "]");
 
@@ -106,7 +108,7 @@ public Boolean verify(final Library _lib) throws VerificationException {
 		this.url = new String("https://pypi.python.org/pypi/<name>/<version>/json").replaceAll("<name>", _lib.getLibraryId().getMvnGroup()).replaceAll("<version>", _lib.getLibraryId().getVersion());
 
 		String response_body = null;
-		Boolean verified = false;
+		List<LibraryId> verified_lids = null;
 		int sc = -1;
 		try {
 			final CloseableHttpClient httpclient = HttpClients.createDefault();
@@ -119,20 +121,18 @@ public Boolean verify(final Library _lib) throws VerificationException {
 				HttpEntity entity = response.getEntity();
 				if (sc==HttpStatus.SC_OK && entity != null) {
 					response_body = ConnectionUtil.readInputStream(entity.getContent());
-					verified = this.containsMD5(response_body, _lib.getDigest());
+					verified_lids = new ArrayList<LibraryId>();
+					if(this.containsMD5(response_body, _lib.getDigest()))
+						verified_lids.add(_lib.getLibraryId());
 
-					// Check whether given and returned libid correspond
-					//final LibraryId returned_libid = new LibraryId((String)JsonPath.read(response_body, "$.response.docs[0].g"),(String)JsonPath.read(response_body, "$.response.docs[0].a"),(String)JsonPath.read(response_body, "$.response.docs[0].v"));
-					//if(_lib.getLibraryId()!=null && !_lib.getLibraryId().equals(returned_libid))
-					//	log.warn("Given and returned library identifiers do not match: Given [" + _lib.getLibraryId() + "], returned [" + returned_libid + "]");
 				}
 			} finally {
 				response.close();
 			}
 		} catch (Exception e) {
 			throw new VerificationException(_lib, this.url, e);
 		}
-		return verified;
+		return verified_lids;
 	}
 
 	/**

rest-backend/src/test/java/com/sap/psr/vulas/backend/rest/BugControllerTest.java

@@ -553,17 +553,20 @@ public void testGetWellKnownAffectedLibrary() throws Exception {
     	LibraryId lid2 = new LibraryId("com.foo", "bar", "1.0-copy");
     	libIdRepository.save(lid2);
 
-    	Library l1 = new Library("123FD");
+    	Library l1 = new Library("1E48256A2341047E7D729217ADEEC8217F6E3A1A");
     	l1.setLibraryId(lid1);
     	l1.setWellknownDigest(true);
     	libRepository.customSave(l1);
 
-    	Library l2 = new Library("456EC");
+    	Library l2 = new Library("123FD");
     	l2.setLibraryId(lid2);
     	l2.setWellknownDigest(false);
     	libRepository.customSave(l2);
 
-    	AffectedLibrary afflib1 = new AffectedLibrary(bug, lid1, true, null, null, null);
+		// the libraryId for l1 must have been replaced with the official one (from
+		// Maven Central) during the digest verification, thus we create the affected
+		// library for the latter
+    	AffectedLibrary afflib1 = new AffectedLibrary(bug, new LibraryId("commons-fileupload","commons-fileupload","1.2.2"), true, null, null, null);
     	afflib1.setSource(AffectedVersionSource.MANUAL);
 
 

rest-backend/src/test/java/com/sap/psr/vulas/backend/rest/HubIntegrationControllerTest.java

@@ -206,14 +206,17 @@ public void testGetHubApps() throws Exception {
 
     @Test
     public void testGetHubAppVulnerabilities() throws Exception {
-    	final Library lib_foo = (Library)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/dummy_app/lib.json")), Library.class);
+    	final Library lib_foo = (Library)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/real_examples/lib_commons-fileupload-1.2.2.json")), Library.class);
     	this.libRepository.customSave(lib_foo);
 
     	final Library lib_bar = (Library)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/dummy_app/lib_bar.json")), Library.class);
     	this.libRepository.customSave(lib_bar);
 
-    	final Bug bug = (Bug)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/dummy_app/bug_foo.json")), Bug.class);
-    	this.bugRepository.customSave(bug,true);    	
+    	final Bug bug_foo = (Bug)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/dummy_app/bug_foo.json")), Bug.class);
+    	this.bugRepository.customSave(bug_foo,true);    	
+
+    	final Bug bug = (Bug)JacksonUtil.asObject(FileUtil.readFile(Paths.get("./src/test/resources/real_examples/bug_CVE-2014-0050.json")), Bug.class);
+    	this.bugRepository.customSave(bug,true);    
 
     	// App with dependencies on foo and bar
     	final Application app = new Application(APP_GROUP, APP_ARTIFACT, "0.0." + APP_VERSION);

rest-backend/src/test/java/com/sap/psr/vulas/backend/rest/LibraryControllerTest.java

@@ -52,8 +52,10 @@
 import org.springframework.web.context.WebApplicationContext;
 
 import com.sap.psr.vulas.backend.model.Library;
+import com.sap.psr.vulas.backend.model.LibraryId;
 import com.sap.psr.vulas.backend.repo.LibraryRepository;
 import com.sap.psr.vulas.shared.categories.RequiresNetwork;
+import com.sap.psr.vulas.shared.enums.DigestAlgorithm;
 import com.sap.psr.vulas.shared.json.JacksonUtil;
 import com.sap.psr.vulas.shared.util.FileUtil;
 
@@ -190,4 +192,27 @@ public void testPostCommonsFileUpload() throws Exception {
     	managed_lib = LibraryRepository.FILTER.findOne(libRepository.findByDigest("1E48256A2341047E7D729217ADEEC8217F6E3A1A"));
     	assertTrue(modifiedAt.getTimeInMillis()==managed_lib.getModifiedAt().getTimeInMillis());
     }
+
+    /**
+     * post lib with well known digest, not well known libid
+     * @throws Exception
+     */
+    @Test
+    @Category(RequiresNetwork.class)
+    public void testPostNotWellKnownLibId() throws Exception {
+
+    	Library lib = new Library("1E48256A2341047E7D729217ADEEC8217F6E3A1A");
+    	lib.setDigestAlgorithm(DigestAlgorithm.SHA1);
+    	lib.setLibraryId(new LibraryId("bar","bar","0"));
+    	MockHttpServletRequestBuilder post_builder = post("/libs/")
+    			.content(JacksonUtil.asJsonString(lib).getBytes())
+				.contentType(MediaType.APPLICATION_JSON)
+				.accept(MediaType.APPLICATION_JSON);
+    	mockMvc.perform(post_builder)	
+                .andExpect(status().isCreated())
+                .andExpect(content().contentType(contentType))
+                .andExpect(jsonPath("$.wellknownDigest", is(true)))
+                .andExpect(jsonPath("$.libraryId.artifact", is("commons-fileupload")))
+                .andExpect(jsonPath("$.digest", is("1E48256A2341047E7D729217ADEEC8217F6E3A1A")));
+        }
 }