From 87b4314e7c5207316ed33fdf5d1d456322acdad3 Mon Sep 17 00:00:00 2001 From: Michael Bar-Sinai Date: Mon, 25 Sep 2017 18:58:42 +0300 Subject: [PATCH 01/89] Make a Dataset have multiple locks. Re-arrange the workflow steps execution system from recursive to iterative, in order to support single transaction per step execution. --- .../dataverse/DOIDataCiteRegisterService.java | 12 +- .../iq/dataverse/DataCiteRESTfullClient.java | 5 +- .../edu/harvard/iq/dataverse/Dataset.java | 76 ++++++-- .../edu/harvard/iq/dataverse/DatasetLock.java | 10 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 25 +-- .../iq/dataverse/DatasetServiceBean.java | 51 ++---- .../harvard/iq/dataverse/DatasetVersion.java | 3 +- .../iq/dataverse/EditDatafilesPage.java | 19 +- .../harvard/iq/dataverse/api/Datasets.java | 40 +--- .../harvard/iq/dataverse/api/Workflows.java | 13 ++ .../api/datadeposit/StatementManagerImpl.java | 12 +- .../dataverse/api/datadeposit/SwordUtil.java | 12 +- .../filesystem/FileRecordJobListener.java | 9 +- .../impl/AbstractPublishDatasetCommand.java | 6 +- .../engine/command/impl/AddLockCommand.java | 3 +- .../FinalizeDatasetPublicationCommand.java | 10 +- .../command/impl/PublishDatasetCommand.java | 27 +-- .../command/impl/RemoveLockCommand.java | 7 +- .../impl/ReturnDatasetToAuthorCommand.java | 5 +- .../dataverse/ingest/IngestMessageBean.java | 11 +- .../workflow/PendingWorkflowInvocation.java | 4 +- .../dataverse/workflow/WorkflowContext.java | 16 +- .../workflow/WorkflowServiceBean.java | 173 +++++++++++++----- .../HttpSendReceiveClientStep.java | 3 +- src/main/resources/META-INF/persistence.xml | 3 +- .../edu/harvard/iq/dataverse/DatasetTest.java | 53 ++++++ .../iq/dataverse/DatasetVersionTest.java | 4 +- .../ReturnDatasetToAuthorCommandTest.java | 2 +- 28 files changed, 378 insertions(+), 236 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 4224b565159..17a6b1759eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -15,11 +15,9 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.PreDestroy; import javax.ejb.Stateless; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import javax.persistence.Query; import javax.persistence.TypedQuery; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -57,7 +55,7 @@ public String createIdentifier(String identifier, HashMap metada metadataTemplate.setPublisherYear(metadata.get("datacite.publicationyear")); String xmlMetadata = metadataTemplate.generateXML(); - logger.fine("XML to send to DataCite: " + xmlMetadata); + logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); String status = metadata.get("_status").trim(); String target = metadata.get("_target"); @@ -92,8 +90,14 @@ public String createIdentifier(String identifier, HashMap metada try (DataCiteRESTfullClient client = openClient()) { retString = client.postMetadata(xmlMetadata); client.postUrl(identifier.substring(identifier.indexOf(":") + 1), target); + } catch (UnsupportedEncodingException ex) { - Logger.getLogger(DOIDataCiteRegisterService.class.getName()).log(Level.SEVERE, null, ex); + logger.log(Level.SEVERE, null, ex); + + } catch ( RuntimeException rte ) { + logger.log(Level.SEVERE, "Error creating DOI at DataCite: {0}", rte.getMessage()); + logger.log(Level.SEVERE, "Exception", rte); + } } } else if (status.equals("unavailable")) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java index 93607a56541..a329f663fb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCiteRESTfullClient.java @@ -169,11 +169,11 @@ public boolean testDOIExists(String doi) { * @param metadata * @return */ - public String postMetadata(String metadata) throws UnsupportedEncodingException { + public String postMetadata(String metadata) { HttpPost httpPost = new HttpPost(this.url + "/metadata"); httpPost.setHeader("Content-Type", "application/xml;charset=UTF-8"); - httpPost.setEntity(new StringEntity(metadata, "utf-8")); try { + httpPost.setEntity(new StringEntity(metadata, "utf-8")); HttpResponse response = httpClient.execute(httpPost,context); String data = EntityUtils.toString(response.getEntity(), encoding); @@ -183,6 +183,7 @@ public String postMetadata(String metadata) throws UnsupportedEncodingException throw new RuntimeException(errMsg); } return data; + } catch (IOException ioe) { logger.log(Level.SEVERE, "IOException when post metadata"); throw new RuntimeException("IOException when post metadata", ioe); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 144285299ad..5925b6cc48f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -9,9 +9,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.logging.Logger; +import java.util.Set; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -73,7 +74,6 @@ sequence. Used when the Dataverse is (optionally) configured to use @Index(columnList = "thumbnailfile_id")}, uniqueConstraints = @UniqueConstraint(columnNames = {"authority,protocol,identifier,doiseparator"})) public class Dataset extends DvObjectContainer { - private static final Logger logger = Logger.getLogger(Dataset.class.getCanonicalName()); public static final String TARGET_URL = "/citation?persistentId="; private static final long serialVersionUID = 1L; @@ -100,8 +100,8 @@ public class Dataset extends DvObjectContainer { @OrderBy("versionNumber DESC, minorVersionNumber DESC") private List versions = new ArrayList<>(); - @OneToOne(mappedBy = "dataset", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}, orphanRemoval = true) - private DatasetLock datasetLock; + @OneToMany(mappedBy = "dataset", cascade = CascadeType.ALL, orphanRemoval = true) + private Set datasetLocks; @OneToOne(cascade = {CascadeType.MERGE, CascadeType.PERSIST}) @JoinColumn(name = "thumbnailfile_id") @@ -154,7 +154,63 @@ public Dataset() { datasetVersion.setMinorVersionNumber((long) 0); versions.add(datasetVersion); } + + /** + * Tests whether {@code this} dataset is locked for a given reason. + * @param reason the reason we test for. + * @return {@code true} iff the data set is locked for {@code reason}. + */ + public boolean isLockedFor( DatasetLock.Reason reason ) { + for ( DatasetLock l : getLocks() ) { + if ( l.getReason() == reason ) { + return true; + } + } + return false; + } + + /** + * Retrieves the dataset lock for the passed reason. + * @param reason + * @return the dataset lock, or {@code null}. + */ + public DatasetLock getLockFor( DatasetLock.Reason reason ) { + for ( DatasetLock l : getLocks() ) { + if ( l.getReason() == reason ) { + return l; + } + } + return null; + } + + public Set getLocks() { + // lazy set creation + if ( datasetLocks == null ) { + setLocks( new HashSet<>() ); + } + return datasetLocks; + } + /** + * JPA use only! + * @param datasetLocks + */ + void setLocks(Set datasetLocks) { + this.datasetLocks = datasetLocks; + } + + public void addLock(DatasetLock datasetLock) { + getLocks().add(datasetLock); + } + + public void removeLock( DatasetLock aDatasetLock ) { + getLocks().remove( aDatasetLock ); + } + + public boolean isLocked() { + return !getLocks().isEmpty(); + } + public String getProtocol() { return protocol; } @@ -240,18 +296,6 @@ public void setFiles(List files) { this.files = files; } - public DatasetLock getDatasetLock() { - return datasetLock; - } - - public void setDatasetLock(DatasetLock datasetLock) { - this.datasetLock = datasetLock; - } - - public boolean isLocked() { - return (getDatasetLock()!=null); - } - public boolean isDeaccessioned() { // return true, if all published versions were deaccessioned boolean hasDeaccessionedVersions = false; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetLock.java b/src/main/java/edu/harvard/iq/dataverse/DatasetLock.java index 71d0456fa67..0353039df69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetLock.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetLock.java @@ -20,6 +20,7 @@ package edu.harvard.iq.dataverse; +import static edu.harvard.iq.dataverse.DatasetLock.Reason.Workflow; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import java.util.Date; import java.io.Serializable; @@ -33,7 +34,6 @@ import javax.persistence.Index; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; -import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; @@ -52,7 +52,7 @@ @Table(indexes = {@Index(columnList="user_id"), @Index(columnList="dataset_id")}) @NamedQueries( @NamedQuery(name="DatasetLock.getLocksByDatasetId", - query="SELECT l FROM DatasetLock l WHERE l.dataset.id=:datasetId") + query="SELECT lock FROM DatasetLock lock WHERE lock.dataset.id=:datasetId") ) public class DatasetLock implements Serializable { @@ -76,13 +76,13 @@ public enum Reason { @Temporal(value = TemporalType.TIMESTAMP) private Date startTime; - @OneToOne + @ManyToOne @JoinColumn(nullable=false) private Dataset dataset; @ManyToOne @JoinColumn(nullable=false) - private AuthenticatedUser user; + private AuthenticatedUser user; @Enumerated(EnumType.STRING) @Column(nullable=false) @@ -116,7 +116,7 @@ public DatasetLock(Reason aReason, AuthenticatedUser aUser, String infoMessage) startTime = new Date(); user = aUser; info = infoMessage; - + } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 8f6301f0b70..53cd8ec20f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -78,10 +78,7 @@ import java.util.logging.Level; import edu.harvard.iq.dataverse.datasetutility.TwoRavensHelper; import edu.harvard.iq.dataverse.datasetutility.WorldMapPermissionHelper; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.impl.AddLockCommand; import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult; -import edu.harvard.iq.dataverse.engine.command.impl.RemoveLockCommand; import edu.harvard.iq.dataverse.engine.command.impl.RestrictFileCommand; import edu.harvard.iq.dataverse.engine.command.impl.ReturnDatasetToAuthorCommand; import edu.harvard.iq.dataverse.engine.command.impl.SubmitDatasetForReviewCommand; @@ -1445,9 +1442,10 @@ private String init(boolean initFull) { // when we sync up with the rsync-upload branch, there will be a merge // conflict here; once resolved, there will also be code here for // rsync upload in progress, and maybe other kinds of locks. - if (dataset.getDatasetLock().getReason().equals(DatasetLock.Reason.Workflow)) { + if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) { JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.publish.workflow.inprogress")); - } else if (dataset.getDatasetLock().getReason().equals(DatasetLock.Reason.InReview)) { + } + if (dataset.isLockedFor(DatasetLock.Reason.InReview)) { JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.inreview.infoMessage")); } } @@ -1798,7 +1796,7 @@ private String releaseDataset(boolean minor) { // has been published. If a publishing workflow is configured, this may have sent the // dataset into a workflow limbo, potentially waiting for a third party system to complete // the process. So it may be premature to show the "success" message at this point. - if (dataset.isLocked() && dataset.getDatasetLock().getReason().equals(DatasetLock.Reason.Workflow)) { + if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) { JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.publish.workflow.inprogress")); } else { JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.publishSuccess")); @@ -2596,7 +2594,7 @@ public void refreshLock() { public boolean isLockedInProgress() { if (dataset != null) { - logger.fine("checking lock status of dataset " + dataset.getId()); + logger.log(Level.FINE, "checking lock status of dataset {0}", dataset.getId()); if (dataset.isLocked()) { return true; } @@ -2605,19 +2603,14 @@ public boolean isLockedInProgress() { } public boolean isDatasetLockedInWorkflow() { - if (dataset != null) { - if (dataset.isLocked()) { - if (dataset.getDatasetLock().getReason().equals(DatasetLock.Reason.Workflow)) { - return true; - } - } - } - return false; + return (dataset != null) + ? dataset.isLockedFor(DatasetLock.Reason.Workflow) + : false; } public boolean isStillLocked() { if (dataset != null && dataset.getId() != null) { - logger.fine("checking lock status of dataset " + dataset.getId()); + logger.log(Level.FINE, "checking lock status of dataset {0}", dataset.getId()); if (datasetService.checkDatasetLock(dataset.getId())) { return true; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index a020a17fde4..c5abe0b5cc1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; @@ -24,6 +19,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -39,11 +35,8 @@ import javax.inject.Inject; import javax.inject.Named; import javax.persistence.EntityManager; -import javax.persistence.NamedStoredProcedureQuery; -import javax.persistence.ParameterMode; import javax.persistence.PersistenceContext; import javax.persistence.Query; -import javax.persistence.StoredProcedureParameter; import javax.persistence.StoredProcedureQuery; import javax.persistence.TypedQuery; import javax.xml.stream.XMLOutputFactory; @@ -511,8 +504,9 @@ public boolean checkDatasetLock(Long datasetId) { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void addDatasetLock(Dataset dataset, DatasetLock lock) { - dataset.setDatasetLock(lock); - em.persist(lock); + lock.setDataset(dataset); + dataset.addLock(lock); + em.merge(dataset); } public void addDatasetLock(Long datasetId, DatasetLock.Reason reason, Long userId, String info) { @@ -536,28 +530,21 @@ public void addDatasetLock(Long datasetId, DatasetLock.Reason reason, Long userI } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void removeDatasetLock(Long datasetId) { + public void removeDatasetLock(Long datasetId, DatasetLock.Reason aReason) { Dataset dataset = em.find(Dataset.class, datasetId); - //em.refresh(dataset); (?) - DatasetLock lock = dataset.getDatasetLock(); - if (lock != null) { - AuthenticatedUser user = lock.getUser(); - dataset.setDatasetLock(null); - user.getDatasetLocks().remove(lock); - /* - * TODO - ? - * throw an exception if for whatever reason we can't remove the lock? - try { - */ - em.remove(lock); - /* - } catch (TransactionRequiredException te) { - ... - } catch (IllegalArgumentException iae) { - ... - } - */ - } + Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.INFO, "Removing locks from dataset " + dataset.getId()); + Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.INFO, dataset.getLocks().toString() ); + new HashSet<>(dataset.getLocks()).stream() + .filter( l -> l.getReason() == aReason ) + .forEach( lock -> { + Logger.getLogger(DatasetServiceBean.class.getName()).log(Level.INFO, "Removing lock " + lock); + dataset.removeLock(lock); + + AuthenticatedUser user = lock.getUser(); + user.getDatasetLocks().remove(lock); + + em.remove(lock); + }); } /* @@ -594,7 +581,7 @@ public String getTitleFromLatestVersion(Long datasetId, boolean includeDraft){ + ";").getSingleResult(); } catch (Exception ex) { - logger.info("exception trying to get title from latest version: " + ex); + logger.log(Level.INFO, "exception trying to get title from latest version: {0}", ex); return ""; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 9e97e8d475a..030a10244a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -221,8 +221,7 @@ public void setDatasetFields(List datasetFields) { */ public boolean isInReview() { if (versionState != null && versionState.equals(VersionState.DRAFT)) { - DatasetLock l = getDataset().getDatasetLock(); - return (l != null) && l.getReason()==DatasetLock.Reason.InReview; + return getDataset().isLockedFor(DatasetLock.Reason.InReview); } else { return false; } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 8b1aaea7f27..35eca0f1d98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2083,19 +2083,16 @@ private boolean isFileAlreadyUploaded(DataFile dataFile) { public boolean isLocked() { if (dataset != null) { - logger.fine("checking lock status of dataset " + dataset.getId()); + logger.log(Level.FINE, "checking lock status of dataset {0}", dataset.getId()); if (dataset.isLocked()) { // refresh the dataset and version, if the current working // version of the dataset is locked: } Dataset lookedupDataset = datasetService.find(dataset.getId()); - DatasetLock datasetLock = null; - if (lookedupDataset != null) { - datasetLock = lookedupDataset.getDatasetLock(); - if (datasetLock != null) { - logger.fine("locked!"); - return true; - } + + if ( (lookedupDataset!=null) && lookedupDataset.isLocked() ) { + logger.fine("locked!"); + return true; } } return false; @@ -2126,12 +2123,12 @@ public void setFileMetadataSelected(FileMetadata fm){ public void setFileMetadataSelected(FileMetadata fm, String guestbook) { fileMetadataSelected = fm; - logger.fine("set the file for the advanced options popup (" + fileMetadataSelected.getLabel() + ")"); + logger.log(Level.FINE, "set the file for the advanced options popup ({0})", fileMetadataSelected.getLabel()); } public FileMetadata getFileMetadataSelected() { if (fileMetadataSelected != null) { - logger.fine("returning file metadata for the advanced options popup (" + fileMetadataSelected.getLabel() + ")"); + logger.log(Level.FINE, "returning file metadata for the advanced options popup ({0})", fileMetadataSelected.getLabel()); } else { logger.fine("file metadata for the advanced options popup is null."); } @@ -2225,7 +2222,7 @@ public void saveAsDesignatedThumbnail() { } public void deleteDatasetLogoAndUseThisDataFileAsThumbnailInstead() { - logger.fine("For dataset id " + dataset.getId() + " the current thumbnail is from a dataset logo rather than a dataset file, blowing away the logo and using this FileMetadata id instead: " + fileMetadataSelectedForThumbnailPopup); + logger.log(Level.FINE, "For dataset id {0} the current thumbnail is from a dataset logo rather than a dataset file, blowing away the logo and using this FileMetadata id instead: {1}", new Object[]{dataset.getId(), fileMetadataSelectedForThumbnailPopup}); /** * @todo Rather than deleting and merging right away, try to respect how * this page seems to stage actions and giving the user a chance to diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 6b854f9f30b..eb6abb6576b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -405,26 +405,6 @@ public Response publishDataseUsingGetDeprecated( @PathParam("id") String id, @Qu return publishDataset(id, type); } - // TODO SBG: Delete me - @EJB - WorkflowServiceBean workflows; - - @PUT - @Path("{id}/actions/wf/{wfid}") - public Response DELETEME(@PathParam("id") String id, @PathParam("wfid") String wfid) { - try { - Workflow wf = workflows.getWorkflow(Long.parseLong(wfid)).get(); - Dataset ds = findDatasetOrDie(id); - WorkflowContext ctxt = new WorkflowContext(createDataverseRequest(findUserOrDie()), ds, 0, 0, WorkflowContext.TriggerType.PostPublishDataset, "DataCite"); - workflows.start(wf, ctxt); - return ok("Started workflow " + wf.getName() + " on dataset " + ds.getId() ); - - } catch (WrappedResponse ex) { - return ex.getResponse(); - } - } - // TODO SBG: /Delete me - @POST @Path("{id}/actions/:publish") public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type) { @@ -655,7 +635,7 @@ public Response getRsync(@PathParam("identifier") String id) { @POST @Path("{identifier}/dataCaptureModule/checksumValidation") public Response receiveChecksumValidationResults(@PathParam("identifier") String id, JsonObject jsonFromDcm) { - logger.fine("jsonFromDcm: " + jsonFromDcm); + logger.log(Level.FINE, "jsonFromDcm: {0}", jsonFromDcm); AuthenticatedUser authenticatedUser = null; try { authenticatedUser = findAuthenticatedUserOrDie(); @@ -712,13 +692,7 @@ public Response submitForReview(@PathParam("id") String idSupplied) { Dataset updatedDataset = execCommand(new SubmitDatasetForReviewCommand(createDataverseRequest(findUserOrDie()), findDatasetOrDie(idSupplied))); JsonObjectBuilder result = Json.createObjectBuilder(); - boolean inReview = false; - try{ - inReview = updatedDataset.getDatasetLock().getReason().equals(DatasetLock.Reason.InReview); - } catch (Exception e){ - System.out.print("submit exception: " + e.getMessage()); - // if there's no lock then it can't be in review by definition - } + boolean inReview = updatedDataset.isLockedFor(DatasetLock.Reason.InReview); result.add("inReview", inReview); result.add("message", "Dataset id " + updatedDataset.getId() + " has been submitted for review."); @@ -747,12 +721,7 @@ public Response returnToAuthor(@PathParam("id") String idSupplied, String jsonBo } AuthenticatedUser authenticatedUser = findAuthenticatedUserOrDie(); Dataset updatedDataset = execCommand(new ReturnDatasetToAuthorCommand(createDataverseRequest(authenticatedUser), dataset, reasonForReturn )); - boolean inReview = false; - try{ - inReview = updatedDataset.getDatasetLock().getReason().equals(DatasetLock.Reason.InReview); - } catch (Exception e){ - // if there's no lock then it can't be in review by definition - } + boolean inReview = updatedDataset.isLockedFor(DatasetLock.Reason.InReview); JsonObjectBuilder result = Json.createObjectBuilder(); result.add("inReview", inReview); @@ -767,9 +736,8 @@ public Response returnToAuthor(@PathParam("id") String idSupplied, String jsonBo * Add a File to an existing Dataset * * @param idSupplied - * @param datasetId * @param jsonData - * @param testFileInputStream + * @param fileInputStream * @param contentDispositionHeader * @param formDataBodyPart * @return diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java index 4269a0215bf..77961369c7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Workflows.java @@ -1,14 +1,17 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; +import edu.harvard.iq.dataverse.engine.command.impl.AddLockCommand; import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation; import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean; import java.util.Arrays; import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; +import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -51,6 +54,16 @@ public Response resumeWorkflow( @PathParam("invocationId") String invocationId, return Response.accepted("/api/datasets/" + pending.getDataset().getId() ).build(); } + @Path("lock/{dsId}") + @GET + public Response lockDataset( @PathParam("dsId") String dsId ) { + return response( req -> { + DatasetLock dl = new DatasetLock(DatasetLock.Reason.Workflow, findAuthenticatedUserOrDie()); + execCommand( new AddLockCommand( req, findDatasetOrDie(dsId), dl) ) ; + return ok("locked dataset " + dsId); + }); + } + private boolean isAllowed(IpAddress addr) { if ( System.currentTimeMillis()-lastWhitelistUpdate > 60*1000 ) { updateWhitelist(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/StatementManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/StatementManagerImpl.java index 5089204f854..f6c9bcca18c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/StatementManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/StatementManagerImpl.java @@ -14,7 +14,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; +import static java.util.stream.Collectors.joining; import javax.ejb.EJB; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -91,14 +93,16 @@ public Statement getStatement(String editUri, Map map, AuthCrede states.put("latestVersionState", dataset.getLatestVersion().getVersionState().toString()); Boolean isMinorUpdate = dataset.getLatestVersion().isMinorUpdate(); states.put("isMinorUpdate", isMinorUpdate.toString()); - DatasetLock lock = dataset.getDatasetLock(); - if (lock != null) { + + if ( dataset.isLocked() ) { states.put("locked", "true"); - states.put("lockedDetail", lock.getInfo()); - states.put("lockedStartTime", lock.getStartTime().toString()); + states.put("lockedDetail", dataset.getLocks().stream().map( l-> l.getInfo() ).collect( joining(",")) ); + Optional earliestLock = dataset.getLocks().stream().min((l1, l2) -> (int)Math.signum(l1.getStartTime().getTime()-l2.getStartTime().getTime()) ); + states.put("lockedStartTime", earliestLock.get().getStartTime().toString()); } else { states.put("locked", "false"); } + statement.setStates(states); List fileMetadatas = dataset.getLatestVersion().getFileMetadatas(); for (FileMetadata fileMetadata : fileMetadatas) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordUtil.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordUtil.java index a35acfb200e..a5efd54559d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordUtil.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.api.datadeposit; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; +import static java.util.stream.Collectors.joining; import org.swordapp.server.SwordError; import org.swordapp.server.UriRegistry; @@ -12,7 +12,7 @@ public class SwordUtil { static String DCTERMS = "http://purl.org/dc/terms/"; - /** + /* * @todo get rid of this method */ public static SwordError throwSpecialSwordErrorWithoutStackTrace(String SwordUriRegistryError, String error) { @@ -28,7 +28,7 @@ public static SwordError throwSpecialSwordErrorWithoutStackTrace(String SwordUri return swordError; } - /** + /* * @todo get rid of this method */ public static SwordError throwRegularSwordErrorWithoutStackTrace(String error) { @@ -42,9 +42,9 @@ public static SwordError throwRegularSwordErrorWithoutStackTrace(String error) { } public static void datasetLockCheck(Dataset dataset) throws SwordError { - DatasetLock datasetLock = dataset.getDatasetLock(); - if (datasetLock != null) { - String message = "Please try again later. Unable to perform operation due to dataset lock: " + datasetLock.getInfo(); + if ( dataset.isLocked() ) { + String message = "Please try again later. Unable to perform operation due to dataset lock: " + + dataset.getLocks().stream().map(l->l.getReason().name() + ": " + l.getInfo()).collect( joining(",") ); throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, message); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java index fa941b9a99b..76e57dd94e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java +++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java @@ -226,9 +226,12 @@ public void afterJob() throws Exception { } // remove dataset lock - if (dataset != null && dataset.getId() != null) { - datasetServiceBean.removeDatasetLock(dataset.getId()); - } + // Disabled now, see L.A.'s comment at beforeJob() +// if (dataset != null && dataset.getId() != null) { +// datasetServiceBean.removeDatasetLock(dataset.getId(), DatasetLock.Reason.Ingest); +// } + + getJobLogger().log(Level.INFO, "Removing dataset lock."); // job step info diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractPublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractPublishDatasetCommand.java index 38708a8efac..9f04f64e0b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractPublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractPublishDatasetCommand.java @@ -21,11 +21,7 @@ public AbstractPublishDatasetCommand(Dataset datasetIn, DataverseRequest aReques } protected WorkflowContext buildContext( String doiProvider, WorkflowContext.TriggerType triggerType) { - return new WorkflowContext(getRequest(), theDataset, - theDataset.getLatestVersion().getVersionNumber(), - theDataset.getLatestVersion().getMinorVersionNumber(), - triggerType, - doiProvider); + return new WorkflowContext(getRequest(), theDataset, doiProvider, triggerType); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddLockCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddLockCommand.java index 1f9ee1e96c2..3001d1532e1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddLockCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AddLockCommand.java @@ -28,8 +28,9 @@ public AddLockCommand(DataverseRequest aRequest, Dataset aDataset, DatasetLock a @Override public DatasetLock execute(CommandContext ctxt) throws CommandException { - lock.setDataset(dataset); + ctxt.datasets().addDatasetLock(dataset, lock); + return lock; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index a031e047902..a904057a067 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -21,7 +21,6 @@ import edu.harvard.iq.dataverse.export.ExportException; import edu.harvard.iq.dataverse.export.ExportService; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; @@ -29,7 +28,6 @@ import java.sql.Timestamp; import java.util.Date; import java.util.Optional; -import java.util.ResourceBundle; import java.util.logging.Level; import java.util.logging.Logger; @@ -111,10 +109,12 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.index().indexDataset(theDataset, doNormalSolrDocCleanUp); ctxt.solrIndex().indexPermissionsForOneDvObject(theDataset); - ctxt.engine().submit(new RemoveLockCommand(getRequest(), theDataset)); + ctxt.engine().submit(new RemoveLockCommand(getRequest(), theDataset, DatasetLock.Reason.Workflow)); + final Optional postPubWorkflow = ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset); - ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset) - .ifPresent(wf -> ctxt.workflows().start(wf, buildContext(doiProvider, TriggerType.PostPublishDataset))); + if ( postPubWorkflow.isPresent() ) { + ctxt.workflows().start(postPubWorkflow.get(), buildContext(doiProvider, TriggerType.PostPublishDataset)); + } return ctxt.em().merge(theDataset); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index 22732ea34f7..2d67fb5cff7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -1,14 +1,8 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetLock; -import edu.harvard.iq.dataverse.DatasetVersionUser; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; @@ -17,14 +11,11 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; -import edu.harvard.iq.dataverse.util.BundleUtil; -import java.io.IOException; -import java.sql.Timestamp; -import java.util.Date; import java.util.Optional; +import static java.util.stream.Collectors.joining; /** - * Kick-off a dataset publication process. The process may complete immediatly, + * Kick-off a dataset publication process. The process may complete immediately, * but may also result in a workflow being started and pending on some external * response. Either way, the process will be completed by an instance of * {@link FinalizeDatasetPublicationCommand}. @@ -64,7 +55,8 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException theDataset.getEditVersion().setVersionNumber(new Long(theDataset.getVersionNumber())); theDataset.getEditVersion().setMinorVersionNumber(new Long(theDataset.getMinorVersionNumber() + 1)); - } else /* major, non-first release */ { + } else { + // major, non-first release theDataset.getEditVersion().setVersionNumber(new Long(theDataset.getVersionNumber() + 1)); theDataset.getEditVersion().setMinorVersionNumber(new Long(0)); } @@ -72,12 +64,12 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException theDataset = ctxt.em().merge(theDataset); //Move remove lock to after merge... SEK 9/1/17 (why? -- L.A.) - ctxt.engine().submit( new RemoveLockCommand(getRequest(), theDataset)); + ctxt.engine().submit( new RemoveLockCommand(getRequest(), theDataset, DatasetLock.Reason.InReview) ); + ctxt.engine().submit( new RemoveLockCommand(getRequest(), theDataset, DatasetLock.Reason.Ingest) ); Optional prePubWf = ctxt.workflows().getDefaultWorkflow(TriggerType.PrePublishDataset); if ( prePubWf.isPresent() ) { // We start a workflow - ctxt.engine().submit( new AddLockCommand(getRequest(), theDataset, new DatasetLock(DatasetLock.Reason.Workflow, getRequest().getAuthenticatedUser()))); ctxt.workflows().start(prePubWf.get(), buildContext(doiProvider, TriggerType.PrePublishDataset) ); return new PublishDatasetResult(theDataset, false); @@ -100,9 +92,10 @@ private void verifyCommandArguments() throws IllegalCommandException { throw new IllegalCommandException("This dataset may not be published because its host dataverse (" + theDataset.getOwner().getAlias() + ") has not been published.", this); } - if (theDataset.isLocked() && !theDataset.getDatasetLock().getReason().equals(DatasetLock.Reason.InReview)) { - - throw new IllegalCommandException("This dataset is locked. Reason: " + theDataset.getDatasetLock().getReason().toString() + ". Please try publishing later.", this); + if ( theDataset.isLocked() ) { + throw new IllegalCommandException("This dataset is locked. Reason: " + + theDataset.getLocks().stream().map(l -> l.getReason().name()).collect( joining(",") ) + + ". Please try publishing later.", this); } if (theDataset.getLatestVersion().isReleased()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RemoveLockCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RemoveLockCommand.java index 669e00ea9ba..9ed38fc6493 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RemoveLockCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RemoveLockCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -16,15 +17,17 @@ public class RemoveLockCommand extends AbstractVoidCommand { private final Dataset dataset; + private final DatasetLock.Reason reason; - public RemoveLockCommand(DataverseRequest aRequest, Dataset aDataset) { + public RemoveLockCommand(DataverseRequest aRequest, Dataset aDataset, DatasetLock.Reason aReason) { super(aRequest, aDataset); dataset = aDataset; + reason = aReason; } @Override protected void executeImpl(CommandContext ctxt) throws CommandException { - ctxt.datasets().removeDatasetLock(dataset.getId()); + ctxt.datasets().removeDatasetLock(dataset.getId(), reason); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java index 3ee601bde30..ba8b0434761 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.DatasetVersionUser; import edu.harvard.iq.dataverse.UserNotification; import edu.harvard.iq.dataverse.authorization.Permission; @@ -44,7 +45,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { throw new IllegalCommandException("You must enter a reason for returning a dataset to the author(s).", this); } */ - ctxt.engine().submit( new RemoveLockCommand(getRequest(), theDataset)); + ctxt.engine().submit( new RemoveLockCommand(getRequest(), theDataset, DatasetLock.Reason.InReview)); Dataset updatedDataset = save(ctxt); return updatedDataset; @@ -56,7 +57,7 @@ public Dataset save(CommandContext ctxt) throws CommandException { theDataset.getEditVersion().setLastUpdateTime(updateTime); // We set "in review" to false because now the ball is back in the author's court. theDataset.setModificationTime(updateTime); - theDataset.setDatasetLock(null); + theDataset.addLock(null); Dataset savedDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java index 46fa7370d3f..48048fd0d74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java @@ -20,30 +20,23 @@ package edu.harvard.iq.dataverse.ingest; -import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.ingest.IngestServiceBean; +import edu.harvard.iq.dataverse.DatasetLock; -import java.io.File; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.logging.Logger; import javax.ejb.ActivationConfigProperty; import javax.ejb.EJB; import javax.ejb.MessageDriven; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import javax.faces.application.FacesMessage; import javax.jms.JMSException; import javax.jms.Message; import javax.jms.MessageListener; import javax.jms.ObjectMessage; -import javax.naming.Context; -import javax.naming.InitialContext; /** * @@ -135,7 +128,7 @@ public void onMessage(Message message) { if (datafile != null) { Dataset dataset = datafile.getOwner(); if (dataset != null && dataset.getId() != null) { - datasetService.removeDatasetLock(dataset.getId()); + datasetService.removeDatasetLock(dataset.getId(), DatasetLock.Reason.Ingest); } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index c335436f5b7..b2f4171a190 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -20,7 +20,7 @@ /** * A workflow whose current step waits for an external system to complete a - * (probably lengthy) process. Meanwhile, it sits in the database, pending. + * (probably lengthy) process. Meanwhile, it sits in the database, pending away. * * @author michael */ @@ -38,6 +38,7 @@ public class PendingWorkflowInvocation implements Serializable { @OneToOne Dataset dataset; + long nextVersionNumber; long nextMinorVersionNumber; @@ -165,5 +166,4 @@ public int getTypeOrdinal() { public void setTypeOrdinal(int typeOrdinal) { this.typeOrdinal = typeOrdinal; } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java index 09129a6d796..0cca2bd64a9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java @@ -6,8 +6,8 @@ import java.util.UUID; /** - * The context in which the workflow is performed. Contains information steps might - * need, such as the dataset being worked on an version data. + * The context in which a workflow is performed. Contains information steps might + * need, such as the dataset being worked on and version data. * * Design-wise, this class allows us to add parameters to {@link WorkflowStep} without * changing its method signatures, which would break break client code. @@ -29,7 +29,16 @@ public enum TriggerType { private String invocationId = UUID.randomUUID().toString(); - public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, long nextMinorVersionNumber, TriggerType type, String doiProvider) { + public WorkflowContext( DataverseRequest aRequest, Dataset aDataset, String doiProvider, TriggerType aTriggerType ) { + this( aRequest, aDataset, + aDataset.getLatestVersion().getVersionNumber(), + aDataset.getLatestVersion().getMinorVersionNumber(), + aTriggerType, + doiProvider); + } + + public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, + long nextMinorVersionNumber, TriggerType type, String doiProvider) { this.request = request; this.dataset = dataset; this.nextVersionNumber = nextVersionNumber; @@ -74,5 +83,4 @@ public TriggerType getType() { return type; } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 3791e9f3851..d55200ee038 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -1,8 +1,11 @@ package edu.harvard.iq.dataverse.workflow; +import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.AddLockCommand; import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand; import edu.harvard.iq.dataverse.engine.command.impl.RemoveLockCommand; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -17,15 +20,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.Asynchronous; import javax.ejb.EJB; import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; +import javax.persistence.Query; /** * Service bean for managing and executing {@link Workflow}s @@ -38,8 +42,11 @@ public class WorkflowServiceBean { private static final Logger logger = Logger.getLogger(WorkflowServiceBean.class.getName()); private static final String WORKFLOW_ID_KEY = "WorkflowServiceBean.WorkflowId:"; - @PersistenceContext + @PersistenceContext(unitName = "VDCNet-ejbPU") EntityManager em; + + @EJB + DatasetServiceBean datasets; @EJB SettingsServiceBean settings; @@ -76,9 +83,13 @@ public WorkflowServiceBean() { * * @param wf the workflow to execute. * @param ctxt the context in which the workflow is executed. + * @throws CommandException If the dataset could not be locked. */ - public void start(Workflow wf, WorkflowContext ctxt) { - forward(wf, ctxt, 0); + @Asynchronous + public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { + ctxt = refresh(ctxt); + lockDataset(ctxt); + forward(wf, ctxt); } /** @@ -92,37 +103,22 @@ public void start(Workflow wf, WorkflowContext ctxt) { * #doResume(edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation, * java.lang.String) */ + @Asynchronous public void resume(PendingWorkflowInvocation pending, String body) { em.remove(em.merge(pending)); doResume(pending, body); } + @Asynchronous - private void forward(Workflow wf, WorkflowContext ctxt, int idx) { - WorkflowStepData wsd = wf.getSteps().get(idx); - WorkflowStep step = createStep(wsd); - WorkflowStepResult res = step.run(ctxt); - - if (res == WorkflowStepResult.OK) { - if (idx == wf.getSteps().size() - 1) { - workflowCompleted(wf, ctxt); - } else { - forward(wf, ctxt, ++idx); - } - - } else if (res instanceof Failure) { - logger.log(Level.WARNING, "Workflow {0} failed: {1}", new Object[]{ctxt.getInvocationId(), ((Failure) res).getReason()}); - rollback(wf, ctxt, (Failure) res, idx - 1); - - } else if (res instanceof Pending) { - pauseAndAwait(wf, ctxt, (Pending) res, idx); - } + private void forward(Workflow wf, WorkflowContext ctxt) { + executeSteps(wf, ctxt, 0); } - @Asynchronous private void doResume(PendingWorkflowInvocation pending, String body) { Workflow wf = pending.getWorkflow(); List stepsLeft = wf.getSteps().subList(pending.getPendingStepIdx(), wf.getSteps().size()); + WorkflowStep pendingStep = createStep(stepsLeft.get(0)); final WorkflowContext ctxt = pending.reCreateContext(roleAssignees); @@ -132,52 +128,129 @@ private void doResume(PendingWorkflowInvocation pending, String body) { } else if (res instanceof Pending) { pauseAndAwait(wf, ctxt, (Pending) res, pending.getPendingStepIdx()); } else { - forward(wf, ctxt, pending.getPendingStepIdx() + 1); + executeSteps(wf, ctxt, pending.getPendingStepIdx() + 1); } } @Asynchronous - private void rollback(Workflow wf, WorkflowContext ctxt, Failure failure, int idx) { - WorkflowStepData wsd = wf.getSteps().get(idx); - logger.log(Level.INFO, "{0} rollback of step {1}", new Object[]{ctxt.getInvocationId(), idx}); - try { - createStep(wsd).rollback(ctxt, failure); - } finally { - if (idx > 0) { - rollback(wf, ctxt, failure, --idx); - } else { - unlockDataset(ctxt); + private void rollback(Workflow wf, WorkflowContext ctxt, Failure failure, int lastCompletedStepIdx) { + ctxt = refresh(ctxt); + final List steps = wf.getSteps(); + + for ( int stepIdx = lastCompletedStepIdx; stepIdx >= 0; --stepIdx ) { + WorkflowStepData wsd = steps.get(stepIdx); + WorkflowStep step = createStep(wsd); + + try { + logger.log(Level.INFO, "Workflow {0} step {1}: Rollback", new Object[]{ctxt.getInvocationId(), stepIdx}); + rollbackStep(step, ctxt, failure); + + } catch (Exception e) { + logger.log(Level.WARNING, "Workflow " + ctxt.getInvocationId() + + " step " + stepIdx + ": Rollback error: " + e.getMessage(), e); } + + } + + logger.log( Level.INFO, "Removing workflow lock"); + try { + engine.submit( new RemoveLockCommand(ctxt.getRequest(), ctxt.getDataset(), DatasetLock.Reason.Workflow) ); + + // Corner case - delete locks generated within this same transaction. + Query deleteQuery = em.createQuery("DELETE from DatasetLock l WHERE l.dataset.id=:id AND l.reason=:reason"); + deleteQuery.setParameter("id", ctxt.getDataset().getId() ); + deleteQuery.setParameter("reason", DatasetLock.Reason.Workflow ); + deleteQuery.executeUpdate(); + + } catch (CommandException ex) { + logger.log(Level.SEVERE, "Error restoring dataset locks state after rollback: " + ex.getMessage(), ex); } } /** - * Unlocks the dataset after the workflow is over. - * @param ctxt + * Execute the passed workflow, starting from {@code initialStepIdx}. + * @param wf The workflow to run. + * @param ctxt Execution context to run the workflow in. + * @param initialStepIdx 0-based index of the first step to run. */ - @Asynchronous - private void unlockDataset( WorkflowContext ctxt ) { - try { - engine.submit( new RemoveLockCommand(ctxt.getRequest(), ctxt.getDataset()) ); - } catch (CommandException ex) { - logger.log(Level.SEVERE, "Cannot unlock dataset after rollback: " + ex.getMessage(), ex); + private void executeSteps(Workflow wf, WorkflowContext ctxt, int initialStepIdx ) { + final List steps = wf.getSteps(); + + for ( int stepIdx = initialStepIdx; stepIdx < steps.size(); stepIdx++ ) { + WorkflowStepData wsd = steps.get(stepIdx); + WorkflowStep step = createStep(wsd); + WorkflowStepResult res = runStep(step, ctxt); + + try { + if (res == WorkflowStepResult.OK) { + logger.log(Level.INFO, "Workflow {0} step {1}: OK", new Object[]{ctxt.getInvocationId(), stepIdx}); + + } else if (res instanceof Failure) { + logger.log(Level.WARNING, "Workflow {0} failed: {1}", new Object[]{ctxt.getInvocationId(), ((Failure) res).getReason()}); + rollback(wf, ctxt, (Failure) res, stepIdx-1 ); + return; + + } else if (res instanceof Pending) { + pauseAndAwait(wf, ctxt, (Pending) res, stepIdx); + return; + } + + } catch ( Exception e ) { + logger.log(Level.WARNING, "Workflow {0} step {1}: Uncought exception:", new Object[]{ctxt.getInvocationId(), e.getMessage()}); + logger.log(Level.WARNING, "Trace:", e); + rollback(wf, ctxt, (Failure) res, stepIdx-1 ); + return; + } } + + workflowCompleted(wf, ctxt); + + } + + ////////////////////////////////////////////////////////////// + // Internal methods to run each step in its own transaction. + // + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + WorkflowStepResult runStep( WorkflowStep step, WorkflowContext ctxt ) { + return step.run(ctxt); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + WorkflowStepResult resumeStep( WorkflowStep step, WorkflowContext ctxt, Map localData, String externalData ) { + return step.resume(ctxt, localData, externalData); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + void rollbackStep( WorkflowStep step, WorkflowContext ctxt, Failure reason ) { + step.rollback(ctxt, reason); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + void lockDataset( WorkflowContext ctxt ) throws CommandException { + final DatasetLock datasetLock = new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser()); +// engine.submit(new AddLockCommand(ctxt.getRequest(), ctxt.getDataset(), datasetLock)); + datasetLock.setDataset(ctxt.getDataset()); + em.persist(datasetLock); + em.flush(); } + // + // + ////////////////////////////////////////////////////////////// + private void pauseAndAwait(Workflow wf, WorkflowContext ctxt, Pending pendingRes, int idx) { PendingWorkflowInvocation pending = new PendingWorkflowInvocation(wf, ctxt, pendingRes); pending.setPendingStepIdx(idx); em.persist(pending); } - @Asynchronous private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { logger.log(Level.INFO, "Workflow {0} completed.", ctxt.getInvocationId()); if ( ctxt.getType() == TriggerType.PrePublishDataset ) { try { engine.submit( new FinalizeDatasetPublicationCommand(ctxt.getDataset(), ctxt.getDoiProvider(), ctxt.getRequest()) ); - unlockDataset(ctxt); - + } catch (CommandException ex) { logger.log(Level.SEVERE, "Exception finalizing workflow " + ctxt.getInvocationId() +": " + ex.getMessage(), ex); rollback(wf, ctxt, new Failure("Exception while finalizing the publication: " + ex.getMessage()), wf.steps.size()-1); @@ -273,5 +346,11 @@ private WorkflowStep createStep(WorkflowStepData wsd) { } return provider.getStep(wsd.getStepType(), wsd.getStepParameters()); } + + private WorkflowContext refresh( WorkflowContext ctxt ) { + return new WorkflowContext( ctxt.getRequest(), + datasets.find( ctxt.getDataset().getId() ), ctxt.getNextVersionNumber(), + ctxt.getNextMinorVersionNumber(), ctxt.getType(), ctxt.getDoiProvider() ); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/HttpSendReceiveClientStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/HttpSendReceiveClientStep.java index 8d882de5303..3bbd294ee72 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/HttpSendReceiveClientStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/HttpSendReceiveClientStep.java @@ -54,7 +54,7 @@ public WorkflowStepResult run(WorkflowContext context) { } } catch (Exception ex) { - logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex ); + logger.log(Level.SEVERE, "Error communicating with remote server: " + ex.getMessage(), ex); return new Failure("Error executing request: " + ex.getLocalizedMessage(), "Cannot communicate with remote server."); } } @@ -66,6 +66,7 @@ public WorkflowStepResult resume(WorkflowContext context, Map in if ( pat.matcher(response).matches() ) { return OK; } else { + logger.log(Level.WARNING, "Remote system returned a bad reposonse: {0}", externalData); return new Failure("Response from remote server did not match expected one (response:" + response + ")"); } } diff --git a/src/main/resources/META-INF/persistence.xml b/src/main/resources/META-INF/persistence.xml index 9303aa98ea4..8b4e33858ac 100644 --- a/src/main/resources/META-INF/persistence.xml +++ b/src/main/resources/META-INF/persistence.xml @@ -15,8 +15,9 @@ - + + - + #{bundle.download} From 7a1bc4373cccb8004d2160045974662378fd834f Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Thu, 19 Oct 2017 18:17:12 -0400 Subject: [PATCH 27/89] put existing response text into files to allow validation --- .../oauth2/impl/OrcidOAuth2APTest.java | 51 ++++++++++++++-- .../xml/oauth2/orcid/v12_no_email.xml | 28 +++++++++ .../xml/oauth2/orcid/v12_no_email_has_aff.xml | 59 +++++++++++++++++++ .../xml/oauth2/orcid/v12_response.xml | 32 ++++++++++ 4 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 src/test/resources/xml/oauth2/orcid/v12_no_email.xml create mode 100644 src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml create mode 100644 src/test/resources/xml/oauth2/orcid/v12_response.xml diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 80a94f16a75..622e7a7afdd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -5,14 +5,18 @@ import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import org.junit.Test; +import org.junit.Before; /** * * @author michael */ public class OrcidOAuth2APTest extends OrcidOAuth2AP { - + private final String response_file="src/test/resources/xml/oauth2/orcid/v12_response.xml"; + private static String RESPONSE = null; + /* private static final String RESPONSE = "\n" + "\n" @@ -46,7 +50,10 @@ public class OrcidOAuth2APTest extends OrcidOAuth2AP { + " \n" + " \n" + ""; - + */ + private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml"; + private static String NO_EMAIL_HAS_AFFILIATION = null; + /* private final String NO_EMAIL_HAS_AFFILIATION = "\n" + "\n" + " 1.2\n" @@ -106,14 +113,47 @@ public class OrcidOAuth2APTest extends OrcidOAuth2AP { + " \n" + " \n" + ""; + */ + private final String no_email_file="src/test/resources/xml/oauth2/orcid/v12_no_email.xml"; + private static String NO_EMAIL = null; public OrcidOAuth2APTest() { super("", "", ""); } + + + @Before + public void setUp() + { + RESPONSE = loadResponseXML( response_file ); + NO_EMAIL_HAS_AFFILIATION = loadResponseXML( no_email_has_affiliation_file ); + NO_EMAIL = loadResponseXML( no_email_file ); + } + /** + * load XML responses from filesystem (resources). + * Why? To allow validating against the XSD prior to 1.2 -> 2.0 upgrade + */ + private static String loadResponseXML(String fname) + { + String txt = null; + try + { + java.io.File inp = new java.io.File( fname ); + //java.io.InputStream inp = new java.io.FileInputStream( new java.io.File( fname ) ); + txt = org.apache.commons.io.FileUtils.readFileToString( inp ); + } + catch( java.io.IOException ie ) + { + // no-op; assert that the needed strings are not null in tests + } + return txt; + } + @Test public void testParseUserResponse() { OrcidOAuth2AP sut = new OrcidOAuth2AP("clientId", "clientSecret", "userEndpoint"); + assertNotNull( RESPONSE ); System.out.println("withEmailResponse = " + RESPONSE); final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(RESPONSE); @@ -131,6 +171,7 @@ public void testParseUserResponse() { @Test public void testParseUserResponseNoEmailHasAffiliation() { OrcidOAuth2AP sut = new OrcidOAuth2AP("clientId", "clientSecret", "userEndpoint"); + assertNotNull( NO_EMAIL_HAS_AFFILIATION ); final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(NO_EMAIL_HAS_AFFILIATION); assertEquals("0000-0002-3283-0661", actual.userIdInProvider); @@ -148,9 +189,9 @@ public void testParseUserResponseNoEmailHasAffiliation() { @Test public void testParseUserResponse_noEmails() { OrcidOAuth2AP sut = new OrcidOAuth2AP("clientId", "clientSecret", "userEndpoint"); - String noEmailResponse = RESPONSE.replaceAll(".*", ""); - System.out.println("noEmailResponse = " + noEmailResponse); - final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(noEmailResponse); + assertNotNull( NO_EMAIL ); + System.out.println("noEmailResponse = " + NO_EMAIL ); + final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(NO_EMAIL); assertEquals("0000-0002-3283-0661", actual.userIdInProvider); assertEquals("Pete.Dataversky", actual.username); diff --git a/src/test/resources/xml/oauth2/orcid/v12_no_email.xml b/src/test/resources/xml/oauth2/orcid/v12_no_email.xml new file mode 100644 index 00000000000..0a82ccda355 --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v12_no_email.xml @@ -0,0 +1,28 @@ + + + 1.2 + + + http://sandbox.orcid.org/0000-0002-3283-0661 + 0000-0002-3283-0661 + sandbox.orcid.org + + + en + + + Member-referred + 2016-10-12T21:59:25.760Z + 2016-10-16T21:11:33.032Z + true + false + false + + + + Pete K. + Dataversky + + + + diff --git a/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml b/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml new file mode 100644 index 00000000000..a852aa3acab --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml @@ -0,0 +1,59 @@ + + + 1.2 + + + http://orcid.org/0000-0002-3283-0661 + 0000-0002-3283-0661 + orcid.org + + + en + + + Direct + 2015-11-30T23:14:04.260Z + 2016-12-13T19:45:57.958Z + true + true + true + + + + Pete K. + Dataversky + + + + + + employment + BCMP + + Harvard Medical School +
+ Boston + MA + US +
+ + 1811 + RINGGOLD + + + + + http://orcid.org/0000-0002-3283-0661 + 0000-0002-3283-0661 + orcid.org + + Pete K. Dataversky + 2015-11-30T23:18:22.764Z + + 2015-11-30T23:18:22.764Z + 2016-05-07T02:26:10.970Z + + + + + diff --git a/src/test/resources/xml/oauth2/orcid/v12_response.xml b/src/test/resources/xml/oauth2/orcid/v12_response.xml new file mode 100644 index 00000000000..e94f8921fcb --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v12_response.xml @@ -0,0 +1,32 @@ + + + 1.2 + + + http://sandbox.orcid.org/0000-0002-3283-0661 + 0000-0002-3283-0661 + sandbox.orcid.org + + + en + + + Member-referred + 2016-10-12T21:59:25.760Z + 2016-10-16T21:11:33.032Z + true + false + false + + + + Pete K. + Dataversky + + + pete2@mailinator.com + pete@mailinator.com + " + + + From 64f2e7289ad379cf9309ccd96ffd098bcf243dce Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Thu, 19 Oct 2017 18:48:41 -0400 Subject: [PATCH 28/89] fix typo so response text validates vs orcid-messages-1.2.xsd --- src/test/resources/xml/oauth2/orcid/v12_response.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/xml/oauth2/orcid/v12_response.xml b/src/test/resources/xml/oauth2/orcid/v12_response.xml index e94f8921fcb..c030a6062f7 100644 --- a/src/test/resources/xml/oauth2/orcid/v12_response.xml +++ b/src/test/resources/xml/oauth2/orcid/v12_response.xml @@ -26,7 +26,7 @@ pete2@mailinator.com pete@mailinator.com - " + From b39ec51b3121b7afe82169de000df6dc5c76b97a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Oct 2017 09:24:25 -0400 Subject: [PATCH 29/89] temporarily allow API token lookup in API test #3153 --- .../java/edu/harvard/iq/dataverse/api/UsersIT.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 8ff30a0b3c4..be66791e599 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -4,6 +4,7 @@ import static com.jayway.restassured.RestAssured.given; import com.jayway.restassured.http.ContentType; import com.jayway.restassured.response.Response; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.util.UUID; import javax.json.Json; import javax.json.JsonObjectBuilder; @@ -20,6 +21,11 @@ public class UsersIT { @BeforeClass public static void setUp() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + Response removeAllowApiTokenLookupViaApi = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); + removeAllowApiTokenLookupViaApi.then().assertThat() + .statusCode(200); + } @Test @@ -41,9 +47,17 @@ public void convertNonBcryptUserFromBuiltinToShib() { convertToSha1.then().assertThat() .statusCode(OK.getStatusCode()); + Response setAllowApiTokenLookupViaApi = UtilIT.setSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi, "true"); + setAllowApiTokenLookupViaApi.then().assertThat() + .statusCode(OK.getStatusCode()); + password = "sha-1Pass"; Response getApiTokenUsingUsername = getApiTokenUsingUsername(usernameOfNonBcryptUserToConvert, password); assertEquals(200, getApiTokenUsingUsername.getStatusCode()); + + Response removeAllowApiTokenLookupViaApi = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); + removeAllowApiTokenLookupViaApi.then().assertThat() + .statusCode(200); String data = emailOfNonBcryptUserToConvert + ":" + password + ":" + newEmailAddressToUse; System.out.println("data: " + data); From efb38cf86638993d65ebb1b4b2ae6cedaa62c057 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Oct 2017 09:25:35 -0400 Subject: [PATCH 30/89] cleanup, rename method to have sane names #3153 --- .../iq/dataverse/api/BuiltinUsersIT.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java index 99a7b395379..a5fa332a46e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java @@ -39,8 +39,8 @@ public class BuiltinUsersIT { public static void setUp() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); - Response removeIdentifierGenerationStyle = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); - removeIdentifierGenerationStyle.then().assertThat() + Response removeAllowApiTokenLookupViaApi = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); + removeAllowApiTokenLookupViaApi.then().assertThat() .statusCode(200); } @@ -72,18 +72,18 @@ public void testCreateTimeAndApiLastUse() { .statusCode(OK.getStatusCode()); } - + @Test public void testLastApiUse() { Response createApiUser = UtilIT.createRandomUser(); String apiUsername = UtilIT.getUsernameFromResponse(createApiUser); String secondApiToken = UtilIT.getApiTokenFromResponse(createApiUser); - + Response createDataverse = UtilIT.createRandomDataverse(secondApiToken); String alias = UtilIT.getAliasFromResponse(createDataverse); Response createDatasetViaApi = UtilIT.createRandomDatasetViaNativeApi(alias, secondApiToken); Response getApiUserAsJson = UtilIT.getAuthenticatedUser(apiUsername, secondApiToken); - + getApiUserAsJson.prettyPrint(); getApiUserAsJson.then().assertThat() // Checking that it's 2017 or whatever. Not y3k compliant! @@ -142,8 +142,8 @@ public void testLeadingWhitespaceInUsername() { createUserResponse.prettyPrint(); assertEquals(400, createUserResponse.statusCode()); } - - @Test + + @Test public void testBadCharacterInUsername() { String randomUsername = getRandomUsername() + "/"; String email = randomUsername + "@mailinator.com"; @@ -151,7 +151,7 @@ public void testBadCharacterInUsername() { createUserResponse.prettyPrint(); assertEquals(400, createUserResponse.statusCode()); } - + @Test public void testAccentInUsername() { String randomUsername = getRandomUsername(); @@ -204,8 +204,8 @@ public void testLogin() { assertEquals(createdToken, retrievedTokenUsingEmail); } - Response removeIdentifierGenerationStyle = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); - removeIdentifierGenerationStyle.then().assertThat() + Response removeAllowApiTokenLookupViaApi = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); + removeAllowApiTokenLookupViaApi.then().assertThat() .statusCode(200); } From a2d227fae42766056f40574925d8750f71f9a31f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Oct 2017 09:31:13 -0400 Subject: [PATCH 31/89] centralize getApiTokenUsingUsername into one method #3153 --- .../harvard/iq/dataverse/api/BuiltinUsersIT.java | 13 +++---------- .../java/edu/harvard/iq/dataverse/api/UsersIT.java | 9 +-------- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 7 +++++++ 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java index a5fa332a46e..8698c5fc18d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/BuiltinUsersIT.java @@ -177,7 +177,7 @@ public void testLogin() { String createdToken = createdUser.getString("data.apiToken"); logger.info(createdToken); - Response getApiTokenShouldFail = getApiTokenUsingUsername(usernameToCreate, usernameToCreate); + Response getApiTokenShouldFail = UtilIT.getApiTokenUsingUsername(usernameToCreate, usernameToCreate); getApiTokenShouldFail.then().assertThat() .body("message", equalTo("This API endpoint has been disabled.")) .statusCode(FORBIDDEN.getStatusCode()); @@ -186,13 +186,13 @@ public void testLogin() { setAllowApiTokenLookupViaApi.then().assertThat() .statusCode(OK.getStatusCode()); - Response getApiTokenUsingUsername = getApiTokenUsingUsername(usernameToCreate, usernameToCreate); + Response getApiTokenUsingUsername = UtilIT.getApiTokenUsingUsername(usernameToCreate, usernameToCreate); getApiTokenUsingUsername.prettyPrint(); assertEquals(200, getApiTokenUsingUsername.getStatusCode()); String retrievedTokenUsingUsername = JsonPath.from(getApiTokenUsingUsername.asString()).getString("data.message"); assertEquals(createdToken, retrievedTokenUsingUsername); - Response failExpected = getApiTokenUsingUsername("junk", "junk"); + Response failExpected = UtilIT.getApiTokenUsingUsername("junk", "junk"); failExpected.prettyPrint(); assertEquals(400, failExpected.getStatusCode()); @@ -358,13 +358,6 @@ private Response createUser(String username, String firstName, String lastName, return response; } - private Response getApiTokenUsingUsername(String username, String password) { - Response response = given() - .contentType(ContentType.JSON) - .get("/api/builtin-users/" + username + "/api-token?username=" + username + "&password=" + password); - return response; - } - private Response getApiTokenUsingEmail(String email, String password) { Response response = given() .contentType(ContentType.JSON) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index be66791e599..c7dcba198ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -52,7 +52,7 @@ public void convertNonBcryptUserFromBuiltinToShib() { .statusCode(OK.getStatusCode()); password = "sha-1Pass"; - Response getApiTokenUsingUsername = getApiTokenUsingUsername(usernameOfNonBcryptUserToConvert, password); + Response getApiTokenUsingUsername = UtilIT.getApiTokenUsingUsername(usernameOfNonBcryptUserToConvert, password); assertEquals(200, getApiTokenUsingUsername.getStatusCode()); Response removeAllowApiTokenLookupViaApi = UtilIT.deleteSetting(SettingsServiceBean.Key.AllowApiTokenLookupViaApi); @@ -84,13 +84,6 @@ public void convertNonBcryptUserFromBuiltinToShib() { } - private Response getApiTokenUsingUsername(String username, String password) { - Response response = given() - .contentType(ContentType.JSON) - .get("/api/builtin-users/" + username + "/api-token?username=" + username + "&password=" + password); - return response; - } - private Response convertUserFromBcryptToSha1(long idOfBcryptUserToConvert, String password) { JsonObjectBuilder data = Json.createObjectBuilder(); data.add("builtinUserId", idOfBcryptUserToConvert); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 8251755043a..2f0d17476b1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1131,6 +1131,13 @@ static Response dataCaptureModuleChecksumValidation(String datasetPersistentId, return requestSpecification.post("/api/datasets/" + persistentIdInPath + "/dataCaptureModule/checksumValidation" + optionalQueryParam); } + static Response getApiTokenUsingUsername(String username, String password) { + Response response = given() + .contentType(ContentType.JSON) + .get("/api/builtin-users/" + username + "/api-token?username=" + username + "&password=" + password); + return response; + } + @Test public void testGetFileIdFromSwordStatementWithNoFiles() { String swordStatementWithNoFiles = "\n" From 3c8adc2f5dbb6d7136ccb4bb243b76589dbb41a0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Oct 2017 09:48:30 -0400 Subject: [PATCH 32/89] remove upload methods after testing #4189 testPrivateUrl was failing: https://build.hmdc.harvard.edu:8443/job/phoenix.dataverse.org-apitest-develop/edu.harvard.iq$dataverse/145/testReport/junit/edu.harvard.iq.dataverse.api/DatasetsIT/testPrivateUrl/ --- .../harvard/iq/dataverse/api/DatasetsIT.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 18973b3d66c..9a6957adcb7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -41,6 +41,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.Matchers.startsWith; +import org.junit.AfterClass; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -69,6 +70,26 @@ public static void setUpClass() { .statusCode(200); } + @AfterClass + public static void afterClass() { + + Response removeIdentifierGenerationStyle = UtilIT.deleteSetting(SettingsServiceBean.Key.IdentifierGenerationStyle); + removeIdentifierGenerationStyle.then().assertThat() + .statusCode(200); + + Response removeExcludeEmail = UtilIT.deleteSetting(SettingsServiceBean.Key.ExcludeEmailFromExport); + removeExcludeEmail.then().assertThat() + .statusCode(200); + + Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl); + removeDcmUrl.then().assertThat() + .statusCode(200); + + Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); + removeUploadMethods.then().assertThat() + .statusCode(200); + } + @Test public void testCreateDataset() { @@ -1172,6 +1193,10 @@ public void testDcmChecksumValidationMessages() throws IOException, InterruptedE .body("data.notifications[0].type", equalTo("CHECKSUMFAIL")) .statusCode(OK.getStatusCode()); + Response removeUploadMethods = UtilIT.deleteSetting(SettingsServiceBean.Key.UploadMethods); + removeUploadMethods.then().assertThat() + .statusCode(200); + String uploadFolder = identifier; /** From 0ed400ae89bf2e5d20da817adfed30081704537c Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:28:23 -0400 Subject: [PATCH 33/89] temporarilly checking v1.2 vs v2.0 API responses --- src/test/resources/xml/oauth2/orcid/v12_3.xml | 61 ++++++++++++++ src/test/resources/xml/oauth2/orcid/v20_3.xml | 83 +++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 src/test/resources/xml/oauth2/orcid/v12_3.xml create mode 100644 src/test/resources/xml/oauth2/orcid/v20_3.xml diff --git a/src/test/resources/xml/oauth2/orcid/v12_3.xml b/src/test/resources/xml/oauth2/orcid/v12_3.xml new file mode 100644 index 00000000000..cb901fa3267 --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v12_3.xml @@ -0,0 +1,61 @@ + + + 1.2 + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + + en + + + Direct + 2017-08-08T16:49:27.162Z + 2017-10-20T16:22:41.922Z + true + true + true + + + + Bob + Doc + + + bdoc@mailinator.com + + + + + + employment + + 2017 + 10 + 31 + + + Miskatonic University +
+ Arkham + US +
+ + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob Doc + 2017-10-20T16:20:46.204Z + + 2017-10-20T16:20:46.204Z + 2017-10-20T16:20:46.204Z + + + + + diff --git a/src/test/resources/xml/oauth2/orcid/v20_3.xml b/src/test/resources/xml/oauth2/orcid/v20_3.xml new file mode 100644 index 00000000000..3ea15c62b77 --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v20_3.xml @@ -0,0 +1,83 @@ + + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + + en + + + Direct + 2017-08-08T16:49:27.162Z + 2017-10-20T16:22:41.922Z + true + true + true + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.162Z + 2017-08-08T16:49:27.421Z + Bob + Doc + + + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.420Z + 2017-08-08T16:50:43.585Z + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob Doc + + bdoc@mailinator.com + + + + + + + + 2017-10-20T16:20:46.204Z + + + 2017-10-20T16:20:46.204Z + + 2017-10-20T16:20:46.204Z + 2017-10-20T16:20:46.204Z + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob Doc + + + 2017 + 10 + 31 + + + Miskatonic University + + Arkham + US + + + + + + + + + From 63f97dd2d300a234d1f4023a4d3d49be29acc5e7 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:38:58 -0400 Subject: [PATCH 34/89] remove commented-out strings (in files now) --- .../oauth2/impl/OrcidOAuth2APTest.java | 97 ------------------- .../xml/oauth2/orcid/v20_response.xml | 62 ++++++++++++ 2 files changed, 62 insertions(+), 97 deletions(-) create mode 100644 src/test/resources/xml/oauth2/orcid/v20_response.xml diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 622e7a7afdd..2fd3b795f59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -16,105 +16,8 @@ public class OrcidOAuth2APTest extends OrcidOAuth2AP { private final String response_file="src/test/resources/xml/oauth2/orcid/v12_response.xml"; private static String RESPONSE = null; - /* - private static final String RESPONSE - = "\n" - + "\n" - + " 1.2\n" - + " \n" - + " \n" - + " http://sandbox.orcid.org/0000-0002-3283-0661\n" - + " 0000-0002-3283-0661\n" - + " sandbox.orcid.org\n" - + " \n" - + " \n" - + " en\n" - + " \n" - + " \n" - + " Member-referred\n" - + " 2016-10-12T21:59:25.760Z\n" - + " 2016-10-16T21:11:33.032Z\n" - + " true\n" - + " false\n" - + " false\n" - + " \n" - + " \n" - + " \n" - + " Pete K.\n" - + " Dataversky\n" - + " \n" - + " " - + " pete2@mailinator.com" - + " pete@mailinator.com" - + " " - + " \n" - + " \n" - + ""; - */ private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml"; private static String NO_EMAIL_HAS_AFFILIATION = null; - /* - private final String NO_EMAIL_HAS_AFFILIATION = "\n" - + "\n" - + " 1.2\n" - + " \n" - + " \n" - + " http://orcid.org/0000-0002-3283-0661\n" - + " 0000-0002-3283-0661\n" - + " orcid.org\n" - + " \n" - + " \n" - + " en\n" - + " \n" - + " \n" - + " Direct\n" - + " 2015-11-30T23:14:04.260Z\n" - + " 2016-12-13T19:45:57.958Z\n" - + " true\n" - + " true\n" - + " true\n" - + " \n" - + " \n" - + " \n" - + " Pete K.\n" - + " Dataversky\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + " employment\n" - + " BCMP\n" - + " \n" - + " Harvard Medical School\n" - + "
\n" - + " Boston\n" - + " MA\n" - + " US\n" - + "
\n" - + " \n" - + " 1811\n" - + " RINGGOLD\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " http://orcid.org/0000-0002-3283-0661\n" - + " 0000-0002-3283-0661\n" - + " orcid.org\n" - + " \n" - + " Pete K. Dataversky\n" - + " 2015-11-30T23:18:22.764Z\n" - + " \n" - + " 2015-11-30T23:18:22.764Z\n" - + " 2016-05-07T02:26:10.970Z\n" - + " \n" - + " \n" - + " \n" - + " \n" - + ""; - */ - private final String no_email_file="src/test/resources/xml/oauth2/orcid/v12_no_email.xml"; private static String NO_EMAIL = null; public OrcidOAuth2APTest() { diff --git a/src/test/resources/xml/oauth2/orcid/v20_response.xml b/src/test/resources/xml/oauth2/orcid/v20_response.xml new file mode 100644 index 00000000000..0e8741aeabf --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v20_response.xml @@ -0,0 +1,62 @@ + + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + + en + + + Direct + 2017-08-08T16:49:27.162Z + 2017-10-20T16:22:41.922Z + true + true + true + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.162Z + 2017-08-08T16:49:27.421Z + Bob T. + Doc + + + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.420Z + 2017-08-08T16:50:43.585Z + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob T. Doc + + bdoc@mailinator.com + + + 2017-08-08T16:49:27.420Z + 2017-08-08T16:50:43.585Z + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob T. Doc + + bdoc2@mailinator.com + + + + + + + From f62bf06cd66b657df8f25e10f20c60e8f56d6965 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:39:21 -0400 Subject: [PATCH 35/89] remove commented-out strings (in files now) --- src/test/resources/xml/oauth2/orcid/v12_response.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/resources/xml/oauth2/orcid/v12_response.xml b/src/test/resources/xml/oauth2/orcid/v12_response.xml index c030a6062f7..0971b5f10d7 100644 --- a/src/test/resources/xml/oauth2/orcid/v12_response.xml +++ b/src/test/resources/xml/oauth2/orcid/v12_response.xml @@ -3,8 +3,8 @@ 1.2 - http://sandbox.orcid.org/0000-0002-3283-0661 - 0000-0002-3283-0661 + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 sandbox.orcid.org @@ -20,12 +20,12 @@ - Pete K. - Dataversky + Bob T. + Doc - pete2@mailinator.com - pete@mailinator.com + bdoc2@mailinator.com + bdoc@mailinator.com From 2d450b9760f0a868c88bd990e68e407b11ed74d3 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:45:20 -0400 Subject: [PATCH 36/89] update first test --- .../providers/oauth2/impl/OrcidOAuth2APTest.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 2fd3b795f59..4d4ec80a0c5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -60,14 +60,15 @@ public void testParseUserResponse() { System.out.println("withEmailResponse = " + RESPONSE); final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(RESPONSE); - assertEquals("0000-0002-3283-0661", actual.userIdInProvider); - assertEquals("pete", actual.username); - assertEquals("Pete K.", actual.displayInfo.getFirstName()); - assertEquals("Dataversky", actual.displayInfo.getLastName()); - assertEquals("pete@mailinator.com", actual.displayInfo.getEmailAddress()); + assertEquals("0000-0003-2591-1698", actual.userIdInProvider); + //assertEquals("bob", actual.username); + assertEquals("bdoc", actual.username); + assertEquals("Bob T.", actual.displayInfo.getFirstName()); + assertEquals("Doc", actual.displayInfo.getLastName()); + assertEquals("bdoc@mailinator.com", actual.displayInfo.getEmailAddress()); assertEquals("", actual.displayInfo.getAffiliation()); assertEquals("", actual.displayInfo.getPosition()); - assertEquals(Arrays.asList("pete@mailinator.com", "pete2@mailinator.com"), actual.emails); + assertEquals(Arrays.asList("bdoc@mailinator.com", "bdoc2@mailinator.com"), actual.emails); } From c877b500ffd3fcf9d705cd14539be5b14060ea84 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:50:52 -0400 Subject: [PATCH 37/89] update second test --- .../oauth2/impl/OrcidOAuth2APTest.java | 10 ++++---- .../xml/oauth2/orcid/v12_no_email_has_aff.xml | 23 ++++++++----------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 4d4ec80a0c5..167a44a3e2f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -78,15 +78,15 @@ public void testParseUserResponseNoEmailHasAffiliation() { assertNotNull( NO_EMAIL_HAS_AFFILIATION ); final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(NO_EMAIL_HAS_AFFILIATION); - assertEquals("0000-0002-3283-0661", actual.userIdInProvider); - assertEquals("Pete K.", actual.displayInfo.getFirstName()); - assertEquals("Dataversky", actual.displayInfo.getLastName()); + assertEquals("0000-0003-2591-1698", actual.userIdInProvider); + assertEquals("Bob T.", actual.displayInfo.getFirstName()); + assertEquals("Doc", actual.displayInfo.getLastName()); assertEquals("", actual.displayInfo.getEmailAddress()); - assertEquals("Harvard Medical School", actual.displayInfo.getAffiliation()); + assertEquals("Miskatonic University", actual.displayInfo.getAffiliation()); assertEquals("", actual.displayInfo.getPosition()); List emptyList = new ArrayList<>(); assertEquals(emptyList, actual.emails); - assertEquals("Pete.Dataversky", actual.username); + assertEquals("Bob.Doc", actual.username); } diff --git a/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml b/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml index a852aa3acab..e6924b97235 100644 --- a/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml +++ b/src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml @@ -3,8 +3,8 @@ 1.2 - http://orcid.org/0000-0002-3283-0661 - 0000-0002-3283-0661 + http://orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 orcid.org @@ -20,34 +20,29 @@ - Pete K. - Dataversky + Bob T. + Doc employment - BCMP - Harvard Medical School + Miskatonic University
- Boston + Arkham MA US
- - 1811 - RINGGOLD - - http://orcid.org/0000-0002-3283-0661 - 0000-0002-3283-0661 + http://orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 orcid.org - Pete K. Dataversky + Bob T. Doc 2015-11-30T23:18:22.764Z 2015-11-30T23:18:22.764Z From 8249e38bb74d45dc8d00b30b468a7c952f152127 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:53:43 -0400 Subject: [PATCH 38/89] update third test (ready for 1.2->2.0 switch) --- .../providers/oauth2/impl/OrcidOAuth2APTest.java | 8 ++++---- src/test/resources/xml/oauth2/orcid/v12_no_email.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 167a44a3e2f..0966474b17b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -97,10 +97,10 @@ public void testParseUserResponse_noEmails() { System.out.println("noEmailResponse = " + NO_EMAIL ); final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(NO_EMAIL); - assertEquals("0000-0002-3283-0661", actual.userIdInProvider); - assertEquals("Pete.Dataversky", actual.username); - assertEquals("Pete K.", actual.displayInfo.getFirstName()); - assertEquals("Dataversky", actual.displayInfo.getLastName()); + assertEquals("0000-0003-2591-1698", actual.userIdInProvider); + assertEquals("Bob.Doc", actual.username); + assertEquals("Bob T.", actual.displayInfo.getFirstName()); + assertEquals("Doc", actual.displayInfo.getLastName()); assertEquals("", actual.displayInfo.getEmailAddress()); assertEquals("", actual.displayInfo.getAffiliation()); assertEquals("", actual.displayInfo.getPosition()); diff --git a/src/test/resources/xml/oauth2/orcid/v12_no_email.xml b/src/test/resources/xml/oauth2/orcid/v12_no_email.xml index 0a82ccda355..2b4f42901ea 100644 --- a/src/test/resources/xml/oauth2/orcid/v12_no_email.xml +++ b/src/test/resources/xml/oauth2/orcid/v12_no_email.xml @@ -3,8 +3,8 @@ 1.2 - http://sandbox.orcid.org/0000-0002-3283-0661 - 0000-0002-3283-0661 + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 sandbox.orcid.org @@ -20,8 +20,8 @@ - Pete K. - Dataversky + Bob T. + Doc From 37a6d1b56f8a18d4e78c130f7edf32754bc1a3f1 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 12:59:14 -0400 Subject: [PATCH 39/89] data for no email test --- .../xml/oauth2/orcid/v20_no_email.xml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/test/resources/xml/oauth2/orcid/v20_no_email.xml diff --git a/src/test/resources/xml/oauth2/orcid/v20_no_email.xml b/src/test/resources/xml/oauth2/orcid/v20_no_email.xml new file mode 100644 index 00000000000..76137f3bffd --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v20_no_email.xml @@ -0,0 +1,34 @@ + + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + + en + + + Direct + 2017-08-08T16:49:27.162Z + 2017-10-20T16:22:41.922Z + true + true + true + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.162Z + 2017-08-08T16:49:27.421Z + Bob + Doc + + + + + + + + + From 6cc56c49a26db5c912359f6e3e0e1037398d5e20 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 13:00:52 -0400 Subject: [PATCH 40/89] last v2 test file (first was previously added with incorrect commit message) --- .../xml/oauth2/orcid/v20_no_email_has_aff.xml | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml diff --git a/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml b/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml new file mode 100644 index 00000000000..eb22b201f80 --- /dev/null +++ b/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml @@ -0,0 +1,68 @@ + + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + + en + + + Direct + 2017-08-08T16:49:27.162Z + 2017-10-20T16:22:41.922Z + true + true + true + + + 2017-08-08T16:50:43.585Z + + 2017-08-08T16:49:27.162Z + 2017-08-08T16:49:27.421Z + Bob + Doc + + + + + + + + + + 2017-10-20T16:20:46.204Z + + + 2017-10-20T16:20:46.204Z + + 2017-10-20T16:20:46.204Z + 2017-10-20T16:20:46.204Z + + + http://sandbox.orcid.org/0000-0003-2591-1698 + 0000-0003-2591-1698 + sandbox.orcid.org + + Bob Doc + + + 2017 + 10 + 31 + + + Miskatonic University + + Arkham + US + + + + + + + + + From 34164a93797e07826296e7714df913f61b1a4236 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 14:24:54 -0400 Subject: [PATCH 41/89] update test data (forgot middle initial for user) --- src/test/resources/xml/oauth2/orcid/v20_no_email.xml | 2 +- src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/xml/oauth2/orcid/v20_no_email.xml b/src/test/resources/xml/oauth2/orcid/v20_no_email.xml index 76137f3bffd..4df340ca5ea 100644 --- a/src/test/resources/xml/oauth2/orcid/v20_no_email.xml +++ b/src/test/resources/xml/oauth2/orcid/v20_no_email.xml @@ -21,7 +21,7 @@ 2017-08-08T16:49:27.162Z 2017-08-08T16:49:27.421Z - Bob + Bob T. Doc diff --git a/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml b/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml index eb22b201f80..fc09dfcf8f7 100644 --- a/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml +++ b/src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml @@ -21,7 +21,7 @@ 2017-08-08T16:49:27.162Z 2017-08-08T16:49:27.421Z - Bob + Bob T. Doc From f50580f45db772aa441ea279f63e285f0a4a3513 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 14:25:46 -0400 Subject: [PATCH 42/89] start switch --- .../providers/oauth2/impl/OrcidOAuth2AP.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java index c56fe77bcf0..b69b8f5f570 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java @@ -68,18 +68,22 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { try ( StringReader reader = new StringReader(responseBody)) { DocumentBuilder db = dbFact.newDocumentBuilder(); Document doc = db.parse( new InputSource(reader) ); - List orcidIdNodeList = getNodes(doc, "orcid-message", "orcid-profile","orcid-identifier","path"); + //List orcidIdNodeList = getNodes(doc, "orcid-message", "orcid-profile","orcid-identifier","path"); + List orcidIdNodeList = getNodes(doc, "record:record", "common:orcid-identifier","common:path"); if ( orcidIdNodeList.size() != 1 ) { throw new OAuth2Exception(0, responseBody, "Cannot find ORCiD id in response."); } String orcidId = orcidIdNodeList.get(0).getTextContent().trim(); - String firstName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "given-names" ) + //String firstName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "given-names" ) + String firstName = getNodes(doc, "record:record", "person:person", "person:name", "personal-details:given-names" ) .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - String familyName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "family-name" ) + //String familyName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "family-name" ) + String familyName = getNodes(doc, "record:record", "person:person", "person:name", "personal-details:family-name") .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - String affiliation = getNodes(doc, "orcid-message", "orcid-profile", "orcid-activities", "affiliations", "affiliation", "organization", "name" ) + //String affiliation = getNodes(doc, "orcid-message", "orcid-profile", "orcid-activities", "affiliations", "affiliation", "organization", "name" ) + String affiliation = getNodes(doc, "record:record", "activities:activities-summary", "activities:employments", "employment:employment-summary", "employment:organization", "common:name") .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); List emails = new ArrayList<>(); From 29507f600d51838243ff201da648b87d6ffe1f02 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 14:52:26 -0400 Subject: [PATCH 43/89] initial switch of test to 2.0 --- .../providers/oauth2/impl/OrcidOAuth2APTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index 0966474b17b..589c844f913 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -14,11 +14,14 @@ * @author michael */ public class OrcidOAuth2APTest extends OrcidOAuth2AP { - private final String response_file="src/test/resources/xml/oauth2/orcid/v12_response.xml"; + //private final String response_file="src/test/resources/xml/oauth2/orcid/v12_response.xml"; + private final String response_file="src/test/resources/xml/oauth2/orcid/v20_response.xml"; private static String RESPONSE = null; - private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml"; + //private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml"; + private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml"; private static String NO_EMAIL_HAS_AFFILIATION = null; - private final String no_email_file="src/test/resources/xml/oauth2/orcid/v12_no_email.xml"; + //private final String no_email_file="src/test/resources/xml/oauth2/orcid/v12_no_email.xml"; + private final String no_email_file="src/test/resources/xml/oauth2/orcid/v20_no_email.xml"; private static String NO_EMAIL = null; public OrcidOAuth2APTest() { super("", "", ""); @@ -62,13 +65,14 @@ public void testParseUserResponse() { assertEquals("0000-0003-2591-1698", actual.userIdInProvider); //assertEquals("bob", actual.username); - assertEquals("bdoc", actual.username); + //assertEquals("bdoc", actual.username); + assertEquals("Bob.Doc", actual.username); assertEquals("Bob T.", actual.displayInfo.getFirstName()); assertEquals("Doc", actual.displayInfo.getLastName()); - assertEquals("bdoc@mailinator.com", actual.displayInfo.getEmailAddress()); + assertEquals("bdoc@mailinator.com", actual.displayInfo.getEmailAddress());//FIXME - failing assertEquals("", actual.displayInfo.getAffiliation()); assertEquals("", actual.displayInfo.getPosition()); - assertEquals(Arrays.asList("bdoc@mailinator.com", "bdoc2@mailinator.com"), actual.emails); + assertEquals(Arrays.asList("bdoc@mailinator.com", "bdoc2@mailinator.com"), actual.emails); //FIXME - failing } From e6c56537854f6d5e33ad97d7ee3124a44f80960a Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 15:11:52 -0400 Subject: [PATCH 44/89] primary email only --- .../providers/oauth2/impl/OrcidOAuth2AP.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java index b69b8f5f570..0a226c35c74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java @@ -28,6 +28,11 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; +import javax.xml.xpath.XPathFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; /** * OAuth2 identity provider for ORCiD. Note that ORCiD has two systems: sandbox @@ -87,6 +92,7 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); List emails = new ArrayList<>(); + /* getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio","contact-details","email").forEach( n ->{ String email = n.getTextContent().trim(); Node primaryAtt = n.getAttributes().getNamedItem("primary"); @@ -100,6 +106,12 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { } }); String primaryEmail = (emails.size()>1) ? emails.get(0) : ""; + */ + String primaryEmail = getPrimaryEmail(doc); + if ( "" != primaryEmail ) + { + emails.add( primaryEmail ); + } // make the username up String username; @@ -149,6 +161,32 @@ private List getNodes( Node node, List path ) { .orElse( Collections.emptyList() ); } + } + // xmlstarlet sel -t -c "/record:record/person:person/email:emails/email:email[@primary='true']/email:email" + /** + * retrieve email from ORCID 2.0 response document, or empty string if no primary email is present + */ + private String getPrimaryEmail(Document doc) + { + String xp_pattern = "/record/person/emails/email[@primary='true']/email/text()"; + String primary_email = ""; + XPathFactory xpf = XPathFactory.newInstance(); + XPath xp = xpf.newXPath(); + try + { + XPathExpression srch = xp.compile( xp_pattern ); + NodeList emails = (NodeList) srch.evaluate(doc, XPathConstants.NODESET); + if ( 1 == emails.getLength() ) + { + primary_email = emails.item(0).getTextContent(); + } + // if there are no (or somehow more than 1) primary email(s), then we've already at failure value + } + catch( javax.xml.xpath.XPathExpressionException xpe ) + { + //no-op; already at failure value (and this shouldn't happen with hard-coded xpath expression anyway) + } + return primary_email; } @Override From 1f2220e82f5d06d26e7da7bfc0cfac39638e5abd Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 15:28:01 -0400 Subject: [PATCH 45/89] retrieve multiple emails back again --- .../providers/oauth2/impl/OrcidOAuth2AP.java | 46 ++++++++++++++----- .../oauth2/impl/OrcidOAuth2APTest.java | 8 ++-- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java index 0a226c35c74..5c64115852d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java @@ -91,7 +91,8 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { String affiliation = getNodes(doc, "record:record", "activities:activities-summary", "activities:employments", "employment:employment-summary", "employment:organization", "common:name") .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - List emails = new ArrayList<>(); + //List emails = new ArrayList<>(); + List emails = getAllEmails(doc); /* getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio","contact-details","email").forEach( n ->{ String email = n.getTextContent().trim(); @@ -108,10 +109,12 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { String primaryEmail = (emails.size()>1) ? emails.get(0) : ""; */ String primaryEmail = getPrimaryEmail(doc); + /* if ( "" != primaryEmail ) { emails.add( primaryEmail ); } + */ // make the username up String username; @@ -168,25 +171,46 @@ private List getNodes( Node node, List path ) { */ private String getPrimaryEmail(Document doc) { - String xp_pattern = "/record/person/emails/email[@primary='true']/email/text()"; + String p = "/record/person/emails/email[@primary='true']/email/text()"; + NodeList emails = xpath_matches( doc, p ); String primary_email = ""; + if ( 1 == emails.getLength() ) + { + primary_email = emails.item(0).getTextContent(); + } + // if there are no (or somehow more than 1) primary email(s), then we've already at failure value + return primary_email; + } + private List getAllEmails(Document doc) + { + String p = "/record/person/emails/email/email/text()"; + NodeList emails = xpath_matches( doc, p ); + List rs = new ArrayList(); + for(int i=0;i Date: Fri, 20 Oct 2017 15:33:37 -0400 Subject: [PATCH 46/89] clean up --- .../providers/oauth2/impl/OrcidOAuth2AP.java | 35 ++++--------------- .../oauth2/impl/OrcidOAuth2APTest.java | 7 +--- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java index 5c64115852d..f50db1e759c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java @@ -38,6 +38,7 @@ * OAuth2 identity provider for ORCiD. Note that ORCiD has two systems: sandbox * and production. Hence having the user endpoint as a parameter. * @author michael + * But don't blame michael for pameyer's later changes */ public class OrcidOAuth2AP extends AbstractOAuth2AuthenticationProvider { @@ -73,48 +74,23 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { try ( StringReader reader = new StringReader(responseBody)) { DocumentBuilder db = dbFact.newDocumentBuilder(); Document doc = db.parse( new InputSource(reader) ); - //List orcidIdNodeList = getNodes(doc, "orcid-message", "orcid-profile","orcid-identifier","path"); List orcidIdNodeList = getNodes(doc, "record:record", "common:orcid-identifier","common:path"); if ( orcidIdNodeList.size() != 1 ) { throw new OAuth2Exception(0, responseBody, "Cannot find ORCiD id in response."); } String orcidId = orcidIdNodeList.get(0).getTextContent().trim(); - //String firstName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "given-names" ) String firstName = getNodes(doc, "record:record", "person:person", "person:name", "personal-details:given-names" ) .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - //String familyName = getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio", "personal-details", "family-name" ) String familyName = getNodes(doc, "record:record", "person:person", "person:name", "personal-details:family-name") .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - //String affiliation = getNodes(doc, "orcid-message", "orcid-profile", "orcid-activities", "affiliations", "affiliation", "organization", "name" ) String affiliation = getNodes(doc, "record:record", "activities:activities-summary", "activities:employments", "employment:employment-summary", "employment:organization", "common:name") .stream().findFirst().map( Node::getTextContent ) .map( String::trim ).orElse(""); - //List emails = new ArrayList<>(); - List emails = getAllEmails(doc); - /* - getNodes(doc, "orcid-message", "orcid-profile", "orcid-bio","contact-details","email").forEach( n ->{ - String email = n.getTextContent().trim(); - Node primaryAtt = n.getAttributes().getNamedItem("primary"); - boolean isPrimary = (primaryAtt!=null) && - (primaryAtt.getTextContent()!=null) && - (primaryAtt.getTextContent().trim().toLowerCase().equals("true")); - if ( isPrimary ) { - emails.add(0, email); - } else { - emails.add(email); - } - }); - String primaryEmail = (emails.size()>1) ? emails.get(0) : ""; - */ + //switch to XPath based methods here for (debatable) clarity, and structural changes with email on 1.2 orcid message -> 2.0 orcid response. String primaryEmail = getPrimaryEmail(doc); - /* - if ( "" != primaryEmail ) - { - emails.add( primaryEmail ); - } - */ + List emails = getAllEmails(doc); // make the username up String username; @@ -165,12 +141,12 @@ private List getNodes( Node node, List path ) { } } - // xmlstarlet sel -t -c "/record:record/person:person/email:emails/email:email[@primary='true']/email:email" /** * retrieve email from ORCID 2.0 response document, or empty string if no primary email is present */ private String getPrimaryEmail(Document doc) { + // `xmlstarlet sel -t -c "/record:record/person:person/email:emails/email:email[@primary='true']/email:email"`, if you're curious String p = "/record/person/emails/email[@primary='true']/email/text()"; NodeList emails = xpath_matches( doc, p ); String primary_email = ""; @@ -181,6 +157,9 @@ private String getPrimaryEmail(Document doc) // if there are no (or somehow more than 1) primary email(s), then we've already at failure value return primary_email; } + /** + * retrieve all emails (including primary) from ORCID 2.0 response document + */ private List getAllEmails(Document doc) { String p = "/record/person/emails/email/email/text()"; diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java index f87a2eeda21..e134e14065b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2APTest.java @@ -12,15 +12,13 @@ /** * * @author michael + * later changes by pameyer */ public class OrcidOAuth2APTest extends OrcidOAuth2AP { - //private final String response_file="src/test/resources/xml/oauth2/orcid/v12_response.xml"; private final String response_file="src/test/resources/xml/oauth2/orcid/v20_response.xml"; private static String RESPONSE = null; - //private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v12_no_email_has_aff.xml"; private final String no_email_has_affiliation_file="src/test/resources/xml/oauth2/orcid/v20_no_email_has_aff.xml"; private static String NO_EMAIL_HAS_AFFILIATION = null; - //private final String no_email_file="src/test/resources/xml/oauth2/orcid/v12_no_email.xml"; private final String no_email_file="src/test/resources/xml/oauth2/orcid/v20_no_email.xml"; private static String NO_EMAIL = null; public OrcidOAuth2APTest() { @@ -46,7 +44,6 @@ private static String loadResponseXML(String fname) try { java.io.File inp = new java.io.File( fname ); - //java.io.InputStream inp = new java.io.FileInputStream( new java.io.File( fname ) ); txt = org.apache.commons.io.FileUtils.readFileToString( inp ); } catch( java.io.IOException ie ) @@ -64,9 +61,7 @@ public void testParseUserResponse() { final AbstractOAuth2AuthenticationProvider.ParsedUserResponse actual = sut.parseUserResponse(RESPONSE); assertEquals("0000-0003-2591-1698", actual.userIdInProvider); - //assertEquals("bob", actual.username); assertEquals("bdoc", actual.username); - //assertEquals("Bob.Doc", actual.username); assertEquals("Bob T.", actual.displayInfo.getFirstName()); assertEquals("Doc", actual.displayInfo.getLastName()); assertEquals("bdoc@mailinator.com", actual.displayInfo.getEmailAddress()); From 4194ad004403fd3e41e97d01e830ae4204b72289 Mon Sep 17 00:00:00 2001 From: Pete Meyer Date: Fri, 20 Oct 2017 15:59:21 -0400 Subject: [PATCH 47/89] initial (aka - not yet tested) configuration files for ORCID 1.2 -> 2.0 --- .../installation/files/root/auth-providers/orcid-sandbox.json | 2 +- .../_static/installation/files/root/auth-providers/orcid.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid-sandbox.json b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid-sandbox.json index 61bf7b79c82..79363f4d42f 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid-sandbox.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid-sandbox.json @@ -3,6 +3,6 @@ "factoryAlias":"oauth2", "title":"ORCID", "subtitle":"", - "factoryData":"type: orcid | userEndpoint: https://api.sandbox.orcid.org/v1.2/{ORCID}/orcid-profile | clientId: FIXME | clientSecret: FIXME", + "factoryData":"type: orcid | userEndpoint: https://api.sandbox.orcid.org/v2.0/{ORCID}/record | clientId: FIXME | clientSecret: FIXME", "enabled":true } diff --git a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid.json b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid.json index 0615bacb056..249c4226f25 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/orcid.json @@ -3,6 +3,6 @@ "factoryAlias":"oauth2", "title":"ORCID", "subtitle":"", - "factoryData":"type: orcid | userEndpoint: https://api.orcid.org/v1.2/{ORCID}/orcid-profile | clientId: FIXME | clientSecret: FIXME", + "factoryData":"type: orcid | userEndpoint: https://api.orcid.org/v2.0/{ORCID}/record | clientId: FIXME | clientSecret: FIXME", "enabled":true } From 5c3c1f16412e961fc14ec5bded8502dba1a7144a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 20 Oct 2017 16:19:57 -0400 Subject: [PATCH 48/89] #4214 replace references to alias= in dataverse page urls --- .../edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../iq/dataverse/DataverseHeaderFragment.java | 4 +-- .../harvard/iq/dataverse/DataversePage.java | 26 +++++++++++-------- .../edu/harvard/iq/dataverse/LoginPage.java | 4 +-- .../harvard/iq/dataverse/MailServiceBean.java | 2 +- .../java/edu/harvard/iq/dataverse/Shib.java | 7 ++--- .../providers/builtin/DataverseUserPage.java | 6 ++--- .../passwordreset/PasswordResetPage.java | 3 ++- .../dataverse/search/SearchServiceBean.java | 2 +- .../iq/dataverse/util/SystemConfig.java | 2 ++ src/main/webapp/dataset.xhtml | 8 +++--- src/main/webapp/dataverse.xhtml | 6 ++--- src/main/webapp/dataverse_header.xhtml | 10 +++---- src/main/webapp/dataverseuser.xhtml | 16 ++++++------ src/main/webapp/harvestclients.xhtml | 2 +- src/main/webapp/search-include-fragment.xhtml | 6 ++--- 16 files changed, 57 insertions(+), 49 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 030553916d3..fe4748ce8c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1970,7 +1970,7 @@ public String deleteDataset() { logger.severe(ex.getMessage()); } JsfHelper.addSuccessMessage(JH.localize("dataset.message.deleteSuccess")); - return "/dataverse.xhtml?alias=" + dataset.getOwner().getAlias() + "&faces-redirect=true"; + return SystemConfig.HOMEPAGE + dataset.getOwner().getAlias() + "&faces-redirect=true"; } public String editFileMetadata(){ diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java b/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java index d1a2e7ab9dd..27799e2f01c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseHeaderFragment.java @@ -223,11 +223,11 @@ public String logout() { redirectPage = URLDecoder.decode(redirectPage, "UTF-8"); } catch (UnsupportedEncodingException ex) { Logger.getLogger(LoginPage.class.getName()).log(Level.SEVERE, null, ex); - redirectPage = "dataverse.xhtml&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } if (StringUtils.isEmpty(redirectPage)) { - redirectPage = "dataverse.xhtml&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } logger.log(Level.INFO, "Sending user to = " + redirectPage); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index c10bc5724a0..023c3336e9e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -639,7 +639,7 @@ public String save() { JsfHelper.addSuccessMessage(message); editMode = null; - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } catch (CommandException ex) { @@ -732,7 +732,7 @@ public String saveLinkedDataverse() { String msg = "Only authenticated users can link a dataverse."; logger.severe(msg); JsfHelper.addErrorMessage(msg); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } linkingDataverse = dataverseService.find(linkingDataverseId); @@ -745,7 +745,7 @@ public String saveLinkedDataverse() { String msg = "Unable to link " + dataverse.getDisplayName() + " to " + linkingDataverse.getDisplayName() + ". An internal error occurred."; logger.log(Level.SEVERE, "{0} {1}", new Object[]{msg, ex}); JsfHelper.addErrorMessage(msg); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } SavedSearch savedSearchOfChildren = createSavedSearchForChildren(savedSearchCreator); @@ -758,20 +758,20 @@ public String saveLinkedDataverse() { DataverseRequest dataverseRequest = new DataverseRequest(savedSearchCreator, SavedSearchServiceBean.getHttpServletRequest()); savedSearchService.makeLinksForSingleSavedSearch(dataverseRequest, savedSearchOfChildren, debug); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success", getSuccessMessageArguments())); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } catch (SearchException | CommandException ex) { // error: solr is down, etc. can't link children right now JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.linked.internalerror", getSuccessMessageArguments())); String msg = dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + " but contents will not appear until an internal error has been fixed."; logger.log(Level.SEVERE, "{0} {1}", new Object[]{msg, ex}); //JsfHelper.addErrorMessage(msg); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } } else { // defer: please wait for the next timer/cron job //JsfHelper.addSuccessMessage(dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + ". Please wait for its contents to appear."); JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success.wait", getSuccessMessageArguments())); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } } @@ -819,7 +819,7 @@ public String saveSavedSearch() { String msg = "Only authenticated users can save a search."; logger.severe(msg); JsfHelper.addErrorMessage(msg); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } SavedSearch savedSearch = new SavedSearch(searchIncludeFragment.getQuery(), linkingDataverse, savedSearchCreator); @@ -843,12 +843,12 @@ public String saveSavedSearch() { arguments.add(linkString); String successMessageString = BundleUtil.getStringFromBundle("dataverse.saved.search.success", arguments); JsfHelper.addSuccessMessage(successMessageString); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } catch (CommandException ex) { String msg = "There was a problem linking this search to yours: " + ex; logger.severe(msg); JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.saved.search.failure") + " " + ex); - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } } @@ -876,7 +876,7 @@ public String releaseDataverse() { } else { JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.publish.not.authorized")); } - return "/dataverse.xhtml?alias=" + dataverse.getAlias() + "&faces-redirect=true"; + return returnRedirect(); } @@ -889,7 +889,7 @@ public String deleteDataverse() { logger.log(Level.SEVERE, "Unexpected Exception calling delete dataverse command", ex); JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.delete.failure")); } - return "/dataverse.xhtml?alias=" + dataverse.getOwner().getAlias() + "&faces-redirect=true"; + return SystemConfig.HOMEPAGE + dataverse.getOwner().getAlias() + "&faces-redirect=true"; } public String getMetadataBlockPreview(MetadataBlock mdb, int numberOfItems) { @@ -1016,5 +1016,9 @@ public void validateAlias(FacesContext context, UIComponent toValidate, Object v } } } + + private String returnRedirect(){ + return SystemConfig.HOMEPAGE + dataverse.getAlias() + "&faces-redirect=true"; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java index 1f5328f9b35..c33f7d08a66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java @@ -173,14 +173,14 @@ public String login() { session.setUser(r); if ("dataverse.xhtml".equals(redirectPage)) { - redirectPage = redirectPage + "&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } try { redirectPage = URLDecoder.decode(redirectPage, "UTF-8"); } catch (UnsupportedEncodingException ex) { Logger.getLogger(LoginPage.class.getName()).log(Level.SEVERE, null, ex); - redirectPage = "dataverse.xhtml&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } logger.log(Level.FINE, "Sending user to = {0}", redirectPage); diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 09093899cc4..4f4f0ce611f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -239,7 +239,7 @@ private String getDatasetDraftLink(Dataset dataset){ } private String getDataverseLink(Dataverse dataverse){ - return systemConfig.getDataverseSiteUrl() + "/dataverse/" + dataverse.getAlias(); + return systemConfig.getDataverseSiteUrl() + SystemConfig.HOMEPAGE + dataverse.getAlias(); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 9945d10c916..4a6acdf2d32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -13,6 +13,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; +import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; @@ -441,16 +442,16 @@ public String getPrettyFacesHomePageString(boolean includeFacetDashRedirect) { String rootDvAlias = getRootDataverseAlias(); if (includeFacetDashRedirect) { if (rootDvAlias != null) { - return plainHomepageString + "?alias=" + rootDvAlias + "&faces-redirect=true"; + return SystemConfig.HOMEPAGE + rootDvAlias + "&faces-redirect=true"; } else { - return plainHomepageString + "?faces-redirect=true"; + return SystemConfig.HOMEPAGE + "?faces-redirect=true"; } } else if (rootDvAlias != null) { /** * @todo Is there a constant for "/dataverse/" anywhere? I guess * we'll just hard-code it here. */ - return "/dataverse/" + rootDvAlias; + return SystemConfig.HOMEPAGE + rootDvAlias; } else { return plainHomepageString; } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 37c3a75cddd..ec1d0cdda59 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -340,14 +340,14 @@ public String save() { // go back to where user came from if ("dataverse.xhtml".equals(redirectPage)) { - redirectPage = redirectPage + "&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } try { redirectPage = URLDecoder.decode(redirectPage, "UTF-8"); } catch (UnsupportedEncodingException ex) { logger.log(Level.SEVERE, "Server does not support 'UTF-8' encoding.", ex); - redirectPage = "dataverse.xhtml&alias=" + dataverseService.findRootDataverse().getAlias(); + redirectPage = SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias(); } logger.log(Level.FINE, "Sending user to = {0}", redirectPage); @@ -391,7 +391,7 @@ public String save() { public String cancel() { if (editMode == EditMode.CREATE) { - return "/dataverse.xhtml?alias=" + dataverseService.findRootDataverse().getAlias() + "&faces-redirect=true"; + return SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias() + "&faces-redirect=true"; } editMode = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java index 228c6741a80..709b62c6d06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/passwordreset/PasswordResetPage.java @@ -13,6 +13,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; @@ -140,7 +141,7 @@ public String resetPassword() { String builtinAuthProviderId = BuiltinAuthenticationProvider.PROVIDER_ID; AuthenticatedUser au = authSvc.lookupUser(builtinAuthProviderId, user.getUserName()); session.setUser(au); - return "/dataverse.xhtml?alias=" + dataverseService.findRootDataverse().getAlias() + "faces-redirect=true"; + return SystemConfig.HOMEPAGE + dataverseService.findRootDataverse().getAlias() + "faces-redirect=true"; } else { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, response.getMessageSummary(), response.getMessageDetail())); return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 660b5cc80e7..dbf685214c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -455,7 +455,7 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, Dataverse dat */ if (type.equals("dataverses")) { solrSearchResult.setName(name); - solrSearchResult.setHtmlUrl(baseUrl + "/dataverse/" + identifier); + solrSearchResult.setHtmlUrl(baseUrl + SystemConfig.HOMEPAGE + identifier); // Do not set the ImageUrl, let the search include fragment fill in // the thumbnail, similarly to how the dataset and datafile cards // are handled. diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index ee26d1e19c5..6e2d6f2fbec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -42,6 +42,8 @@ public class SystemConfig { @EJB AuthenticationServiceBean authenticationService; + + public static final String HOMEPAGE = "/dataverse/"; /** * A JVM option for the advertised fully qualified domain name (hostname) of diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4a71e3b274f..4a4327afdfc 100755 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -628,7 +628,7 @@ - +