Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PRODTASKS-698 Prevent unmarshalling null values. Revert timing change… #694

Merged
merged 1 commit into from
Oct 9, 2019
Merged

PRODTASKS-698 Prevent unmarshalling null values. Revert timing change… #694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions common/src/main/java/org/commonjava/maven/ext/common/util/JSONUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,24 +29,24 @@
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.JsonNodeType;
import kong.unirest.JacksonObjectMapper;
import org.commonjava.maven.atlas.ident.ref.ProjectVersionRef;
import org.commonjava.maven.atlas.ident.ref.SimpleProjectVersionRef;
import org.commonjava.maven.ext.common.json.ExtendedLookupReport;
import org.commonjava.maven.ext.common.json.PME;
import org.jboss.da.listings.model.rest.RestProductInput;
import org.jboss.da.reports.model.response.LookupReport;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;

public class JSONUtils
{
private static final String GROUP_ID = "groupId";
private static final String ARTIFACT_ID = "artifactId";
private static final String VERSION = "version";
private static final String BEST_MATCH = "bestMatchVersion";

private static final ObjectMapper MAPPER = new ObjectMapper();

Expand Down Expand Up @@ -130,9 +130,16 @@ public ExtendedLookupReport deserialize( JsonParser p, DeserializationContext ct
final String version = node.get(VERSION).asText();

ExtendedLookupReport result = new ExtendedLookupReport();
// TODO: Is there any point in demarshalling this?
result.setAvailableVersions( node.get("availableVersions").traverse( p.getCodec() ).readValueAs( stringList ) );

result.setBlacklisted( node.get("blacklisted").asBoolean() );
result.setBestMatchVersion( node.get("bestMatchVersion").asText() );
if ( node.has( BEST_MATCH ) &&
! node.get(BEST_MATCH).getNodeType().equals( JsonNodeType.NULL ) )
{
// Calling asText on a NullNode returns a String ' "null" '.
result.setBestMatchVersion( node.get(BEST_MATCH).asText() );
}
result.setProjectVersionRef( new SimpleProjectVersionRef( groupId, artifactId, version) );

return result;
Expand Down
6 changes: 6 additions & 0 deletions core/src/main/java/org/commonjava/maven/ext/core/impl/VersionCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,12 @@ else if ( incrementalSuffix != null )

/**
* Find matching version strings in the remote repo.
*
* @param state Current VersionState configuration
* @param groupId to look for
* @param artifactId to look for
* @return the set of potential candidates
* @throws ManipulationException if an error occurs.
*/
protected Set<String> getVersionCandidates(VersioningState state, String groupId, String artifactId)
throws ManipulationException
Expand Down
96 changes: 50 additions & 46 deletions io/src/main/java/org/commonjava/maven/ext/io/rest/DefaultTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,22 +90,13 @@ public class DefaultTranslator

static
{
Logger unirestLogger = LoggerFactory.getLogger( DefaultTranslator.class );

// According to https://kong.github.io/unirest-java/#configuration the default connection timeout is 10000
// and the default socketTimeout is 60000.
// We have increased the first to 30 seconds and the second to 10 minutes.
Unirest.config()
.socketTimeout( 600000 )
.connectTimeout( 30000 )
.setObjectMapper( new InternalObjectMapper( new com.fasterxml.jackson.databind.ObjectMapper() ) )
.instrumentWith( requestSummary -> {
long startNanos = System.nanoTime();
return ( responseSummary, exception ) ->
printFinishTime( unirestLogger, startNanos,
( responseSummary != null && responseSummary.getStatus() / 100 == 2 ) );
}
);
.setObjectMapper( new InternalObjectMapper( new com.fasterxml.jackson.databind.ObjectMapper() ) );
}

// Allow test to override this.
Expand Down Expand Up @@ -299,57 +290,69 @@ public Map<ProjectVersionRef, String> translateVersions( List<ProjectVersionRef>
logger.info( "Calling REST client... (with {} GAVs)", projects.size() );
final Queue<Task> queue = new ArrayDeque<>();
final Map<ProjectVersionRef, String> result = new HashMap<>();
final long start = System.nanoTime();
boolean finishedSuccessfully = false;

partition( projects, queue );

while ( !queue.isEmpty() )
try
{
Task task = queue.remove();
task.executeTranslate();
if ( task.isSuccess() )
{
result.putAll( task.getResult() );
}
else
{
if ( task.canSplit() && isRecoverable( task.getStatus() ) )
{
if ( task.getStatus() == HttpStatus.SC_SERVICE_UNAVAILABLE )
{
logger.info( "The DA server is unavailable. Waiting {} before splitting the tasks and retrying",
retryDuration );

waitBeforeRetry( retryDuration );
}

List<Task> tasks = task.split();
partition( projects, queue );

logger.warn( "Failed to translate versions for task @{} due to {}, splitting and retrying. Chunk size was: {} and new chunk size {} in {} segments.",
task.hashCode(), task.getStatus(), task.getChunkSize(), tasks.get( 0 ).getChunkSize(),
tasks.size() );
queue.addAll( tasks );
while ( !queue.isEmpty() )
{
Task task = queue.remove();
task.executeTranslate();
if ( task.isSuccess() )
{
result.putAll( task.getResult() );
}
else
{
if ( task.getStatus() < 0 )
{
logger.debug( "Caught exception calling server with message {}", task.getErrorMessage() );
}
else
if ( task.canSplit() && isRecoverable( task.getStatus() ) )
{
logger.debug( "Did not get status {} but received {}", SC_OK, task.getStatus() );
}
if ( task.getStatus() == HttpStatus.SC_SERVICE_UNAVAILABLE )
{
logger.info( "The DA server is unavailable. Waiting {} before splitting the tasks and retrying",
retryDuration );

if ( task.getStatus() > 0 )
{
throw new RestException( "Received response status " + task.getStatus() + " with message: " + task.getErrorMessage() );
waitBeforeRetry( retryDuration );
}

List<Task> tasks = task.split();

logger.warn( "Failed to translate versions for task @{} due to {}, splitting and retrying. Chunk size was: {} and new chunk size {} in {} segments.",
task.hashCode(), task.getStatus(), task.getChunkSize(), tasks.get( 0 ).getChunkSize(),
tasks.size() );
queue.addAll( tasks );
}
else
{
throw new RestException( "Received response status " + task.getStatus() + " with message " + task.getErrorMessage() );
if ( task.getStatus() < 0 )
{
logger.debug( "Caught exception calling server with message {}", task.getErrorMessage() );
}
else
{
logger.debug( "Did not get status {} but received {}", SC_OK, task.getStatus() );
}

if ( task.getStatus() > 0 )
{
throw new RestException( "Received response status " + task.getStatus() + " with message: "
+ task.getErrorMessage() );
}
else
{
throw new RestException( "Received response status " + task.getStatus() + " with message " + task.getErrorMessage() );
}
}
}
}
finishedSuccessfully = true;
}
finally
{
printFinishTime( logger, start, finishedSuccessfully);
}

return result;
Expand Down Expand Up @@ -434,6 +437,7 @@ void executeTranslate()
.asObject( lookupType )
.ifSuccess( successResponse -> result = successResponse.getBody()
.stream()
.filter( f -> isNotBlank( f.getBestMatchVersion() ) )
.collect( Collectors.toMap( e -> ((ExtendedLookupReport)e).getProjectVersionRef(),
LookupReport::getBestMatchVersion ) ) )
.ifFailure( failedResponse -> {
Expand Down
24 changes: 24 additions & 0 deletions io/src/test/java/org/commonjava/maven/ext/io/rest/VersionTranslatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import java.util.Scanner;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;

Expand Down Expand Up @@ -117,6 +118,29 @@ public void testTranslateVersions()
assertThat( actualResult, is( expectedResult ) );
}


@Test
public void testTranslateVersionsWithNulls()
{
this.versionTranslator = new DefaultTranslator( mockServer.getUrl(), 0, Translator.CHUNK_SPLIT_COUNT, "NullBestMatchVersion",
"" );
List<ProjectVersionRef> gavs = Arrays.asList(
new SimpleProjectVersionRef( "com.example", "example", "1.0" ),
new SimpleProjectVersionRef( "com.example", "example-dep", "2.0" ),
new SimpleProjectVersionRef( "org.commonjava", "example", "1.0" ),
new SimpleProjectVersionRef( "org.commonjava", "example", "1.1" ));

Map<ProjectVersionRef, String> actualResult = versionTranslator.translateVersions( gavs );

System.out.println ("### actual " + actualResult);

// All values with null bestMatchVersion should have been filtered out.
Map<ProjectVersionRef, String> expectedResult = new HashMap<ProjectVersionRef, String>()
{{
}};
assertEquals( expectedResult, actualResult );
}

@Test
public void testTranslateVersionsFailNoResponse()
{
Expand Down
6 changes: 5 additions & 1 deletion io/src/test/java/org/commonjava/maven/ext/io/rest/handler/AddSuffixJettyHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ public void handle( String target, Request baseRequest, HttpServletRequest reque
LookupGAVsRequest lookupGAVsRequest = objectMapper.readValue( jb.toString(), LookupGAVsRequest.class );
requestBody = lookupGAVsRequest.getGavs();

boolean returnNullBestMatch = "NullBestMatchVersion".equals( lookupGAVsRequest.getRepositoryGroup() );
boolean useCustomMixedSuffix = requestBody.stream().anyMatch( r -> r.getArtifactId().equals( "rest-version-manip-mixed-suffix-orig-rh" ) );
boolean usePartialCustomMixedSuffix = requestBody.stream().anyMatch( r -> r.getArtifactId().equals( "rest-version-manip-mixed-suffix-orig-rh-norhalign" ) );

Expand Down Expand Up @@ -200,7 +201,10 @@ else if ( useCustomMixedSuffix && gav.getArtifactId().equals( "commons-httpclien

availableVersions.add( bestMatchVersion );

lr.setBestMatchVersion( bestMatchVersion );
if ( !returnNullBestMatch )
{
lr.setBestMatchVersion( bestMatchVersion );
}
lr.setBlacklisted( false );
lr.setAvailableVersions( availableVersions );

Expand Down