8191 require toa or req access

doc/release-notes/8191-require-toa-or-request-access.md

@@ -0,0 +1,17 @@
+## Release Highlights
+
+### Terms of Access or Request Access Required for Restricted Files
+
+Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled.
+
+In the "Notes for Dataverse Installation Administrators" section, we have provided a query to help proactively identify datasets that need to be updated.
+
+## Notes for Dataverse Installation Administrators
+
+### Identifying Datasets Requiring Terms of Access or Request Access Changes
+
+In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access or Request Access enabled:
+
+https://github.com/IQSS/dataverse/raw/develop/scripts/issues/8191/
+
+This will allow you to reach out to those dataset owners as appropriate.

scripts/issues/8191/datasets_without_toa_or_request_access

@@ -0,0 +1,28 @@
+-- this query will identify datasets where at least one file does not have either terms of access or request access enabled, and will include owner information for those datasets
+
+select au.email, 
+concat(au.firstname, ' ', au.lastname),
+concat('$SERVERNAME/dataset.xhtml?persistentId=doi:' , dvo.authority , '/' ,  dvo.identifier)
+from roleassignment ra, dataverserole dvr,
+authenticateduser au, dvobject dvo
+where
+au.useridentifier = rtrim(substring(ra.assigneeidentifier, 2, 100))
+and dvo.id = ra.definitionpoint_id
+and
+ra.role_id = dvr.id and
+dvr.alias in (
+	'fullContributor',
+'dsContributor',
+'contributor',
+'admin',
+'curator'
+) and
+ra.definitionpoint_id in (
+	select dvo.id  from datasetversion v
+join termsofuseandaccess ua on ua.id = v.termsofuseandaccess_id
+join filemetadata fm on v.id = fm.datasetversion_id
+join datafile f on f.id = fm.datafile_id
+join dvobject dvo on v.dataset_id = dvo.id
+where ua.fileaccessrequest = false and ua.termsofaccess isnull
+and f.restricted = true
+)

scripts/search/tests/data/dataset-finch1.jsonld

@@ -20,7 +20,7 @@
   "http://schema.org/version": "DRAFT",
   "http://schema.org/name": "Darwin's Finches",
     "https://dataverse.org/schema/core#fileTermsOfAccess": {
-        "https://dataverse.org/schema/core#fileRequestAccess": false
+        "https://dataverse.org/schema/core#fileRequestAccess": true
     },
     "http://schema.org/includedInDataCatalog": "Root"
 }

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -294,6 +294,7 @@ public void setVersions(List<DatasetVersion> versions) {
     }
 
     private DatasetVersion createNewDatasetVersion(Template template, FileMetadata fmVarMet) {
+
         DatasetVersion dsv = new DatasetVersion();
         dsv.setVersionState(DatasetVersion.VersionState.DRAFT);
         dsv.setFileMetadatas(new ArrayList<>());
@@ -313,11 +314,11 @@ private DatasetVersion createNewDatasetVersion(Template template, FileMetadata f
             if (latestVersion.getDatasetFields() != null && !latestVersion.getDatasetFields().isEmpty()) {
                 dsv.setDatasetFields(dsv.copyDatasetFields(latestVersion.getDatasetFields()));
             }
-
-            if (latestVersion.getTermsOfUseAndAccess()!= null){
-                dsv.setTermsOfUseAndAccess(latestVersion.getTermsOfUseAndAccess().copyTermsOfUseAndAccess());
-            }
-
+            /*
+            adding file metadatas here and updating terms
+            because the terms need to know about the files
+            in a pre-save validation SEK 12/6/2021
+            */
             for (FileMetadata fm : latestVersion.getFileMetadatas()) {
                 FileMetadata newFm = new FileMetadata();
                 // TODO: 
@@ -348,6 +349,18 @@ private DatasetVersion createNewDatasetVersion(Template template, FileMetadata f
 
                 dsv.getFileMetadatas().add(newFm);
             }
+
+            if (latestVersion.getTermsOfUseAndAccess()!= null){
+                TermsOfUseAndAccess terms = latestVersion.getTermsOfUseAndAccess().copyTermsOfUseAndAccess();
+                terms.setDatasetVersion(dsv);
+                dsv.setTermsOfUseAndAccess(terms);
+            } else {
+                TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
+                terms.setDatasetVersion(dsv);
+                terms.setLicense(null);
+                terms.setFileAccessRequest(true);
+                dsv.setTermsOfUseAndAccess(terms);
+            }
         }
 
         // I'm adding the version to the list so it will be persisted when

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -143,6 +143,7 @@
 import edu.harvard.iq.dataverse.search.SearchServiceBean;
 import edu.harvard.iq.dataverse.search.SearchUtil;
 import edu.harvard.iq.dataverse.search.SolrClientService;
+import edu.harvard.iq.dataverse.util.FileMetadataUtil;
 import java.util.Comparator;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
@@ -394,6 +395,43 @@ public void setRsyncScript(String rsyncScript) {
     public String getRsyncScriptFilename() {
         return rsyncScriptFilename;
     }
+
+    private Boolean hasValidTermsOfAccess = null;
+
+    public Boolean isHasValidTermsOfAccess() {
+        //cache in page to limit processing
+        if (hasValidTermsOfAccess != null){
+            return hasValidTermsOfAccess;
+        } else {
+            if (!isHasRestrictedFiles()){
+               hasValidTermsOfAccess = true;
+               return hasValidTermsOfAccess;
+            } else {
+                hasValidTermsOfAccess = TermsOfUseAndAccessValidator.isTOUAValid(workingVersion.getTermsOfUseAndAccess(), null);
+                return hasValidTermsOfAccess;
+            }
+        }    
+    }
+
+    private Boolean hasRestrictedFiles = null;
+
+    public Boolean isHasRestrictedFiles(){
+        //cache in page to limit processing
+        if (hasRestrictedFiles != null){
+            return hasRestrictedFiles;
+        } else {
+            hasRestrictedFiles = workingVersion.isHasRestrictedFile();
+            return hasRestrictedFiles;
+        }
+    }
+
+    public boolean getHasValidTermsOfAccess(){
+        return isHasValidTermsOfAccess(); //HasValidTermsOfAccess
+    }
+
+    public void setHasValidTermsOfAccess(boolean value){
+        //dummy for ui
+    }
 
     private String thumbnailString = null;
 
@@ -2074,7 +2112,17 @@ private String init(boolean initFull) {
         previewTools = externalToolService.findFileToolsByType(ExternalTool.Type.PREVIEW);
         datasetExploreTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.EXPLORE);
         rowsPerPage = 10;
-
+        if (dataset.getId() != null && canUpdateDataset()) {
+            hasRestrictedFiles = workingVersion.isHasRestrictedFile();
+            hasValidTermsOfAccess = isHasValidTermsOfAccess();
+            if (!hasValidTermsOfAccess) {
+                String message = BundleUtil.getStringFromBundle("dataset.message.editMetadata.invalid.TOUA.message");               
+                if(workingVersion.isDraft()){
+                    message = message + " " + BundleUtil.getStringFromBundle("dataset.message.publish.invalid.TOUA.message");
+                }
+                JsfHelper.addWarningMessage(message);
+            }            
+        }
         return null;
     }
 
@@ -2438,12 +2486,7 @@ public void edit(EditMode editMode) {
         }
         workingVersion = dataset.getEditVersion();
         clone = workingVersion.cloneDatasetVersion();
-        if (editMode == EditMode.INFO) {
-            // ?
-        } else if (editMode == EditMode.FILE) {
-            // JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editFiles"));
-            // FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, "Upload + Edit Dataset Files", " - You can drag and drop your files from your desktop, directly into the upload widget."));
-        } else if (editMode.equals(EditMode.METADATA)) {
+        if (editMode.equals(EditMode.METADATA)) {
             datasetVersionUI = datasetVersionUI.initDatasetVersionUI(workingVersion, true);
             updateDatasetFieldInputLevels();
             JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editMetadata.label"), BundleUtil.getStringFromBundle("dataset.message.editMetadata.message"));
@@ -3238,15 +3281,17 @@ public List<Dataverse> completeHostDataverseMenuList(String query) {
 
     public String restrictFiles(boolean restricted) throws CommandException {
         List filesToRestrict = new ArrayList();
-
         if (fileMetadataForAction != null) {
             filesToRestrict.add(fileMetadataForAction);
         } else {
             filesToRestrict = this.getSelectedFiles();
         }
-
         restrictFiles(filesToRestrict, restricted);
-        return save();
+        if (editMode == EditMode.CREATE) {
+            return "";
+        } else {
+            return save();
+        }
     }
 
     private void restrictFiles(List<FileMetadata> filesToRestrict, boolean restricted) throws CommandException {
@@ -3330,7 +3375,16 @@ public String deleteFiles() throws CommandException{
         }
 
         deleteFiles(filesToDelete);
-        String retVal = save();
+        String retVal;
+
+        if (editMode == EditMode.CREATE) {
+            workingVersion.setFileMetadatas(new ArrayList<>());
+            retVal = "";
+        } else {
+            retVal = save();
+        }
+
+
         //And delete them only after the dataset is updated
         for(Embargo emb: orphanedEmbargoes) {
             embargoService.deleteById(emb.getId(), ((AuthenticatedUser)session.getUser()).getUserIdentifier());
@@ -3365,32 +3419,12 @@ private void deleteFiles(List<FileMetadata> filesToDelete) {
                 // So below we are deleting the metadata from the version; we are
                 // NOT adding the file to the filesToBeDeleted list that will be
                 // passed to the UpdateDatasetCommand. -- L.A. Aug 2017
-                Iterator<FileMetadata> fmit = dataset.getEditVersion().getFileMetadatas().iterator();
-                while (fmit.hasNext()) {
-                    FileMetadata fmd = fmit.next();
-                    if (markedForDelete.getDataFile().getStorageIdentifier().equals(fmd.getDataFile().getStorageIdentifier())) {
-                        // And if this is an image file that happens to be assigned
-                        // as the dataset thumbnail, let's null the assignment here:
-
-                        if (fmd.getDataFile().equals(dataset.getThumbnailFile())) {
-                            dataset.setThumbnailFile(null);
-                        }
-                        /* It should not be possible to get here if this file
-                           is not in fact released! - so the code block below
-                           is not needed.
-                        //if not published then delete identifier
-                        if (!fmd.getDataFile().isReleased()){
-                            try{
-                                commandEngine.submit(new DeleteDataFileCommand(fmd.getDataFile(), dvRequestService.getDataverseRequest()));
-                            } catch (CommandException e){
-                                 //this command is here to delete the identifier of unreleased files
-                                 //if it fails then a reserved identifier may still be present on the remote provider
-                            }
-                        } */
-                        fmit.remove();
-                        break;
-                    }
-                }
+
+                FileMetadataUtil.removeFileMetadataFromList(workingVersion.getFileMetadatas(), markedForDelete);
+
+                FileMetadataUtil.removeDataFileFromList(newFiles, markedForDelete.getDataFile());
+                FileUtil.deleteTempFile(markedForDelete.getDataFile(), dataset, ingestService);
+
             }
         }
 
@@ -3580,6 +3614,7 @@ public String save() {
 
         if (editMode != null) {
             if (editMode.equals(EditMode.CREATE)) {
+
                 // We allow users to upload files on Create:
                 int nNewFiles = newFiles.size();
                 logger.fine("NEW FILES: "+nNewFiles);
@@ -4509,12 +4544,23 @@ public void setCategoriesByName(List<String>  dummy){
     }
 
     public void refreshTagsPopUp(){
+        if(!isHasValidTermsOfAccess()){
+                this.editMode = EditMode.LICENSE;
+                PrimeFaces.current().executeScript("PF('blockDatasetForm').show()");
+                PrimeFaces.current().executeScript("PF('accessPopup').show()");
+                JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.editTerms.label"), BundleUtil.getStringFromBundle("dataset.message.editTerms.message"));
+                this.readOnly = false;
+                return; 
+        }
+
         if (workingVersion.isReleased()) {
             refreshSelectedFiles(selectedFiles);
         }
         updateFileCounts();
-        refreshCategoriesByName();
+        refreshCategoriesByName();        
+
         refreshTabFileTagsByName();
+        PrimeFaces.current().executeScript("PF('fileTagsPopup').show()");
     }
 
     private List<String> tabFileTagsByName;

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -5,6 +5,7 @@
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.dataset.DatasetUtil;
 import edu.harvard.iq.dataverse.license.License;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -565,6 +566,13 @@ public boolean isHasNonPackageFile(){
         // The presence of any non-package file means that HTTP Upload was used (no mixing allowed) so we just check the first file.
         return !this.fileMetadatas.get(0).getDataFile().getContentType().equals(DataFileServiceBean.MIME_TYPE_PACKAGE_FILE);
     }
+
+    public boolean isHasRestrictedFile(){
+        if (this.fileMetadatas == null || this.fileMetadatas.isEmpty()){
+            return false;
+        }
+        return this.fileMetadatas.stream().anyMatch(fm -> (fm.isRestricted()));
+    }
 
     public void updateDefaultValuesFromTemplate(Template template) {
         if (!template.getDatasetFields().isEmpty()) {
@@ -590,10 +598,12 @@ public DatasetVersion cloneDatasetVersion(){
                 dsv.setDatasetFields(dsv.copyDatasetFields(this.getDatasetFields()));
             }
 
-            if (this.getTermsOfUseAndAccess()!= null){
-                dsv.setTermsOfUseAndAccess(this.getTermsOfUseAndAccess().copyTermsOfUseAndAccess());
-            }
-
+            /*
+            adding file metadatas here and updating terms
+            because the terms need to know about the files
+            in a pre-save validation SEK 12/6/2021
+            */
+
             for (FileMetadata fm : this.getFileMetadatas()) {
                 FileMetadata newFm = new FileMetadata();
                 // TODO: 
@@ -613,6 +623,17 @@ public DatasetVersion cloneDatasetVersion(){
 
                 dsv.getFileMetadatas().add(newFm);
             }
+
+            if (this.getTermsOfUseAndAccess()!= null){
+                TermsOfUseAndAccess terms = this.getTermsOfUseAndAccess().copyTermsOfUseAndAccess();
+                terms.setDatasetVersion(dsv);
+                dsv.setTermsOfUseAndAccess(terms);
+            } else {
+                TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
+                terms.setDatasetVersion(dsv);
+               // terms.setLicense(TermsOfUseAndAccess.License.CC0);
+                dsv.setTermsOfUseAndAccess(terms);
+            }
 
         dsv.setDataset(this.getDataset());
         return dsv;
@@ -1655,7 +1676,22 @@ public Set<ConstraintViolation> validate() {
                 }
             }
         }
+
+
+        TermsOfUseAndAccess toua = this.termsOfUseAndAccess;
+        //Only need to test Terms of Use and Access if there are restricted files  
+        if (toua != null && this.isHasRestrictedFile()) {
+            Set<ConstraintViolation<TermsOfUseAndAccess>> constraintViolations = validator.validate(toua);
+            if (constraintViolations.size() > 0) {
+                ConstraintViolation<TermsOfUseAndAccess> violation = constraintViolations.iterator().next();
+                String message = BundleUtil.getStringFromBundle("toua.invalid");
+                logger.info(message);
+                this.termsOfUseAndAccess.setValidationMessage(message);
+                returnSet.add(violation);
+            }
+        }
 
+
         return returnSet;
     }