diff --git a/datasets/wheat-beta/client/sampleQuery.xml b/datasets/wheat-beta/client/sampleQuery.xml index 6a7f181a5..5353438f8 100644 --- a/datasets/wheat-beta/client/sampleQuery.xml +++ b/datasets/wheat-beta/client/sampleQuery.xml @@ -4,85 +4,51 @@ Example 1 Grain colour color OR flavon* OR proanthocyanidin - TraesCS1D02G351600 - TraesCS1D02G276300 - TraesCS2A02G527700 - TraesCS2A02G025700 - TraesCS2B02G558400 - TraesCS2B02G038700 - TraesCS2D02G530600 - TraesCS2D02G493400 - TraesCS3A02G464900 - TraesCS3A02G226600 - TraesCS3B02G006100 - TraesCSU02G046300 - TraesCSU02G043000 - TraesCS3B02G280700 - TraesCS1B02G420400 - TraesCS3B02G510700 - TraesCSU02G159400 - TraesCS3B02G006600 - TraesCS3B02G257900 - TraesCS3B02G277600 - TraesCS3B02G007400 - TraesCS3B02G006500 - TraesCS3A02G007300 - TraesCS3B02G006300 - TraesCS3B02G010500 - TraesCS3D02G224600 - TraesCSU02G140100 - TraesCS3D02G463200 - TraesCS3D02G468500 - TraesCS3D02G468400 - TraesCSU02G040800 - TraesCS4A02G206800 - TraesCS4A02G085900 - TraesCS6A02G369700 - TraesCS6B02G406400 - TraesCS6B02G405900 - TraesCSU02G069200 + TRAESCS3D02G468400 + TRAESCS2B02G038700 + TRAESCS2A02G025700 + TRAESCS2A02G527700 + TRAESCS2D02G530600 + TRAESCS2B02G558400 + TRAESCS3B02G257900 + TRAESCS3A02G226600 + TRAESCS3D02G224600 + TRAESCS2D02G493400 + TRAESCS6A02G369700 + TRAESCS6B02G406400 + TRAESCS6B02G405900 + TRAESCS3B02G280700 + TRAESCS4A02G085900 + TRAESCS3B02G006300 + TRAESCS3B02G007400 + TRAESCS4A02G206800 + TRAESCS3B02G006100 + TRAESCS3A02G464900 Example 2 Pre-harvest sprouting "seed germination" OR "seed dormancy" - TraesCS1D02G351600 - TraesCS1D02G276300 - TraesCS2A02G527700 - TraesCS2A02G025700 - TraesCS2B02G558400 - TraesCS2B02G038700 - TraesCS2D02G530600 - TraesCS2D02G493400 - TraesCS3A02G464900 - TraesCS3A02G226600 - TraesCS3B02G006100 - TraesCSU02G046300 - TraesCSU02G043000 - TraesCS3B02G280700 - TraesCS1B02G420400 - TraesCS3B02G510700 - TraesCSU02G159400 - TraesCS3B02G006600 - TraesCS3B02G257900 - TraesCS3B02G277600 - TraesCS3B02G007400 - TraesCS3B02G006500 - TraesCS3A02G007300 - TraesCS3B02G006300 - TraesCS3B02G010500 - TraesCS3D02G224600 - TraesCSU02G140100 - TraesCS3D02G463200 - TraesCS3D02G468500 - TraesCS3D02G468400 - TraesCSU02G040800 - TraesCS4A02G206800 - TraesCS4A02G085900 - TraesCS6A02G369700 - TraesCS6B02G406400 - TraesCS6B02G405900 - TraesCSU02G069200 + TRAESCS3D02G468400 + TRAESCS2B02G038700 + TRAESCS2A02G025700 + TRAESCS2A02G527700 + TRAESCS2D02G530600 + TRAESCS2B02G558400 + TRAESCS3B02G257900 + TRAESCS3A02G226600 + TRAESCS3D02G224600 + TRAESCS2D02G493400 + TRAESCS6A02G369700 + TRAESCS6B02G406400 + TRAESCS6B02G405900 + TRAESCS3B02G280700 + TRAESCS4A02G085900 + TRAESCS3B02G006300 + TRAESCS3B02G007400 + TRAESCS4A02G206800 + TRAESCS3B02G006100 + TRAESCS3A02G464900 Example 3 @@ -105,46 +71,21 @@ Example 4 Rust response rust OR response OR disease - TraesCS1B02G343700 - TraesCS1B02G343800 - TraesCS1B02G343900 - TraesCS1D02G156800 - TraesCS2A02G157900 - TraesCS2A02G395200 - TraesCS2B02G183200 - TraesCS2B02G184100 - TraesCS2B02G184100 - TraesCS2B02G184100 - TraesCS2B02G184100 - TraesCS2D02G009700 - TraesCS2D02G010700 - TraesCS2D02G165200 - TraesCS3B02G381600 - TraesCS4A02G456300 - TraesCS5B02G241800 - TraesCS5B02G337300 - TraesCS5B02G347900 - TraesCS5D02G559100 - TraesCS5D02G560600 - TraesCS5D02G560900 - TraesCS5D02G560900 - TraesCS5D02G560900 - TraesCS6A02G416300 - TraesCS6A02G417500 - TraesCS6A02G417600 - TraesCS6A02G419100 - TraesCS6B02G462300 - TraesCS6B02G468200 - TraesCS6B02G470800 - TraesCS6B02G471500 - TraesCS6B02G471700 - TraesCS6B02G473000 - TraesCS6B02G473300 - TraesCS6D02G398300 - TraesCS6D02G401700 - TraesCS6D02G405600 - TraesCS6D02G405600 - TraesCS7A02G033100 + TRAESCS5B02G337300 + TRAESCS6D02G405600 + TRAESCS6A02G417600 + TRAESCS5D02G560900 + TRAESCS2A02G395200 + TRAESCS2D02G010700 + TRAESCS6A02G416300 + TRAESCS5B02G241800 + TRAESCS2B02G183200 + TRAESCS5D02G560600 + TRAESCS6D02G401700 + TRAESCS1B02G343800 + TRAESCS6B02G462300 + TRAESCS7A02G033100 + TRAESCS4A02G456300 Example 5 @@ -155,9 +96,13 @@ 5A 580000000 590000000 - + - ADA2A - BPM5 + + 2D + 280000000 + 310000000 + + diff --git a/rres/dockerhub-deploy.sh b/rres/dockerhub-deploy.sh index a1cc6472b..9b7a69920 100644 --- a/rres/dockerhub-deploy.sh +++ b/rres/dockerhub-deploy.sh @@ -36,8 +36,8 @@ EOT echo -e "--- Stopping, cleaning and updating Docker\n" docker stop wheat-ci || true docker rm wheat-ci || true -#docker system prune --all --force -#docker pull knetminer/knetminer +docker system prune --all --force +docker pull knetminer/knetminer echo -e "--- Cleaning Knetminer dataset directory\n" ./cleanup-volume.sh --all "$dataset_dir" diff --git a/server-base/pom.xml b/server-base/pom.xml index 7b016f4e2..996e4d58c 100644 --- a/server-base/pom.xml +++ b/server-base/pom.xml @@ -19,18 +19,6 @@ the server-datasource-api module. - - - - src/main/resources - true - - ga.properties - - - - - org.springframework @@ -100,4 +88,21 @@ javax.annotation-api + + + + + src/main/resources + true + + ga.properties + + + + src/main/resources + false + + + + diff --git a/server-base/src/main/java/rres/knetminer/datasource/server/KnetminerExceptionHandler.java b/server-base/src/main/java/rres/knetminer/datasource/server/KnetminerExceptionHandler.java index 0a6fa84c1..06fca6e17 100644 --- a/server-base/src/main/java/rres/knetminer/datasource/server/KnetminerExceptionHandler.java +++ b/server-base/src/main/java/rres/knetminer/datasource/server/KnetminerExceptionHandler.java @@ -70,6 +70,9 @@ protected final ResponseEntity handleExceptionInternal ( * status code if present. Else, let {@link #handleExceptionInternal(Exception, Object, HttpHeaders, HttpStatus, WebRequest)} * to set a generic {@link HttpStatus#INTERNAL_SERVER_ERROR}. * + * TODO: NotReadyException requires {@link HttpStatus#SERVICE_UNAVAILABLE}, including this exception here requires + * jutils dependency. + * */ @ExceptionHandler @SuppressWarnings ( "unchecked" ) @@ -85,7 +88,7 @@ protected final ResponseEntity handleExceptionInternal ( return (ResponseEntity) (ResponseEntity) this.handleExceptionInternal ( ex, null, new HttpHeaders (), status, request ); } - + /** * Takes the status code from {@link ResponseStatusException#getStatus() code}. */ diff --git a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/OndexLocalDataSource.java b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/OndexLocalDataSource.java index 75fbec5cd..ab538113f 100644 --- a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/OndexLocalDataSource.java +++ b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/OndexLocalDataSource.java @@ -85,8 +85,9 @@ private void init () var ondexServiceProvider = OndexServiceProvider.getInstance (); var dataService = ondexServiceProvider.getDataService (); - // this is quick and can be done in advance, so that we have what we need to be able to start answering the - // URLs + // this pre-loads some properties in advance, so that we have what we need (ie, data source name) to be able + // to start answering the API URLs + // This is also quick enough to be done synchronously. dataService.loadOptions ( configXmlPath ); var dsName = dataService.getDataSourceName (); if ( dsName == null ) throw new IllegalArgumentException ( @@ -202,12 +203,9 @@ private T _keyword(T response, KnetminerRequest requ genes.addAll(geneMap.keySet()); log.info("Number of genes: " + genes.size()); - if (userGenes != null) { - /* use this (Set userGenes) in place of the genes ArrayList genes. */ - // genes= new ArrayList (userGenes); - - /* filter scored results (ArrayList genes) to only retain sorted genes (by KnetScore) - from user gene list (Set userGenes) */ + if (userGenes != null) + { + // Filter by user-provided list Iterator itr = genes.iterator(); while (itr.hasNext()) { ONDEXConcept gene = itr.next(); @@ -215,16 +213,15 @@ from user gene list (Set userGenes) */ itr.remove(); } } - - /* also, add any missing genes from user list (Set userGenes) that weren't already in the scored results - (ArrayList genes) due to no evidences */ + + // And re-add missing user-genes (which possibly, didn't score well) for (ONDEXConcept userGene : userGenes) { if (!genes.contains(userGene)) { genes.add(userGene); - geneMap.put(userGene, 0.0); // Ensure the gene is placed into the HashMap + geneMap.put(userGene, 0.0); } } - log.info("Using user gene list... genes: " + genes.size()); + log.info("Using user gene list, genes: " + genes.size()); } if (response.getClass().equals(QtlResponse.class)) { log.info("QTL response..."); diff --git a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/UIService.java b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/UIService.java index d93185a4f..51003699d 100644 --- a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/UIService.java +++ b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/UIService.java @@ -6,6 +6,7 @@ import java.util.Set; import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.queryparser.classic.ParseException; @@ -84,12 +85,16 @@ public String renderSynonymTable ( String keyword ) throws ParseException // Only start a KEY tag if it will have contents. Otherwise skip it. out.append ( "<" + synonymKey + ">\n" ); + // we store this no of top synonyms per concept type. That is, in the ordered loop below, we scan + // key-associated concepts in order of search score and, we skip the rendering of those concepts + // which of type count has reached this threshold. + final int MAX_SYNONYMS = 25; + Stream> sortedSynonyms = synonyms2Scores.entrySet () .stream () .sorted ( Collections.reverseOrder ( Map.Entry.comparingByValue () ) ); Map entryCountsByType = new HashMap<> (); - final int MAX_SYNONYMS = 25; // we store this no of top synonyms per concept // writes the topX values in table sortedSynonyms.forEach ( entry -> @@ -97,23 +102,24 @@ public String renderSynonymTable ( String keyword ) throws ParseException int synonymId = entry.getKey (); float score = entry.getValue (); - ONDEXConcept eoc = graph.getConcept ( synonymId ); - String type = eoc.getOfType ().getId (); + ONDEXConcept synonymConcept = graph.getConcept ( synonymId ); + String synonymType = synonymConcept.getOfType ().getId (); - if ( ( type.equals ( "Publication" ) || type.equals ( "Thing" ) ) ) return; + if ( StringUtils.trimToNull ( synonymType ) == null ) return; + if ( ( synonymType.equals ( "Publication" ) || synonymType.equals ( "Thing" ) ) ) return; // TODO: before, this count was incremented in the cNames loop below, however, that way either we // get the same because there's one preferred name only, - // or the count computed that way is likely wrong, cause it increases with names + // or the computed count is likely wrong, cause it increases with names // - int synCount = entryCountsByType.compute ( type, - (thisType, thisCount) -> thisType == null ? 1 : ++thisCount + int typeCount = entryCountsByType.compute ( synonymType, + (thisType, thisCount) -> thisCount == null ? 1 : ++thisCount ); - if ( synCount > MAX_SYNONYMS ) return; - + // See above for details + if ( typeCount > MAX_SYNONYMS ) return; - Set cNames = eoc.getConceptNames (); + Set cNames = synonymConcept.getConceptNames (); cNames.stream () .filter ( ConceptName::isPreferred ) @@ -128,7 +134,7 @@ public String renderSynonymTable ( String keyword ) throws ParseException // suggestions if ( name.contains ( "\"" ) ) name = name.replaceAll ( "\"", "" ); - out.append ( name + "\t" + type + "\t" + Float.toString ( score ) + "\t" + synonymId + "\n" ); + out.append ( name + "\t" + synonymType + "\t" + Float.toString ( score ) + "\t" + synonymId + "\n" ); }); }); // forEach synonym @@ -136,6 +142,7 @@ public String renderSynonymTable ( String keyword ) throws ParseException } // for synonymKeys return out.toString (); - } // + + } // renderSynonymTable() } diff --git a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/utils/KGUtils.java b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/utils/KGUtils.java index c912c092f..cf5299c15 100644 --- a/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/utils/KGUtils.java +++ b/server-datasource-ondexlocal/src/main/java/rres/knetminer/datasource/ondexlocal/service/utils/KGUtils.java @@ -63,8 +63,8 @@ public static Set filterGenesByAccessionKeywords ( DataService dat Set normAccs = accessions.stream () .map ( acc -> - acc.replaceAll ( "^[\"()]+", "" ) - .replaceAll ( "[\"()]+$", "" ) + acc.replaceAll ( "^[\"()]+", "" ) // remove initial \" \( or \) + .replaceAll ( "[\"()]+$", "" ) // remove the same characters as ending chars .toUpperCase () ).collect ( Collectors.toSet () );