Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support different checksum algorithms in direct upload #7602

Merged
merged 2 commits into from
Feb 12, 2021
+79 −41
Merged

support different checksum algorithms in direct upload #7602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8b4fb3
support different checksum algorithms in direct upload
qqmyers Feb 10, 2021
9322570
set default checksumtype, add support via UI
qqmyers Feb 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.api.AbstractApiBean;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.Permission;
Expand Down Expand Up @@ -1788,6 +1789,7 @@ private void handleReplaceFileUpload(FacesEvent event, InputStream inputStream,
if (fileReplacePageHelper.handleNativeFileUpload(inputStream,null,
fileName,
contentType,
null,
null
)){
saveEnabled = true;
Expand Down Expand Up @@ -1848,12 +1850,13 @@ the uploadFinished() method, triggered next, after the upload event
private void handleReplaceFileUpload(String fullStorageLocation,
String fileName,
String contentType,
String checkSum){
String checkSumValue,
ChecksumType checkSumType){

fileReplacePageHelper.resetReplaceFileHelper();
saveEnabled = false;
String storageIdentifier = DataAccess.getStorarageIdFromLocation(fullStorageLocation);
if (fileReplacePageHelper.handleNativeFileUpload(null, storageIdentifier, fileName, contentType, checkSum)){
if (fileReplacePageHelper.handleNativeFileUpload(null, storageIdentifier, fileName, contentType, checkSumValue, checkSumType)){
saveEnabled = true;

/**
Expand Down Expand Up @@ -1985,9 +1988,12 @@ public void handleExternalUpload() {
String fullStorageIdentifier = paramMap.get("fullStorageIdentifier");
String fileName = paramMap.get("fileName");
String contentType = paramMap.get("contentType");
String checksumType = paramMap.get("checksumType");
String checksumTypeString = paramMap.get("checksumType");
String checksumValue = paramMap.get("checksumValue");

ChecksumType checksumType = null;
if (!checksumTypeString.isBlank()) {
checksumType = ChecksumType.fromString(checksumTypeString);
}
int lastColon = fullStorageIdentifier.lastIndexOf(':');
String storageLocation= fullStorageIdentifier.substring(0,lastColon) + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + fullStorageIdentifier.substring(lastColon+1);
if (uploadInProgress.isFalse()) {
Expand All @@ -2009,12 +2015,6 @@ public void handleExternalUpload() {
contentType = FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
}

if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) {
String warningMessage = "Non-MD5 checksums not yet supported in external uploads";
localWarningMessage = warningMessage;
//ToDo - methods like handleReplaceFileUpload and classes like OptionalFileParams will need to track the algorithm in addition to the value to enable this
}

/* ----------------------------
Check file size
- Max size NOT specified in db: default is unlimited
Expand All @@ -2029,7 +2029,7 @@ public void handleExternalUpload() {
// Is this a FileReplaceOperation? If so, then diverge!
// -----------------------------------------------------------
if (this.isFileReplaceOperation()){
this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue);
this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue, checksumType);
this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0));
return;
}
Expand All @@ -2047,7 +2047,7 @@ public void handleExternalUpload() {
//datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");


datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, systemConfig);
datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);
} catch (IOException ex) {
logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName});
}
Expand Down
14 changes: 9 additions & 5 deletions src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package edu.harvard.iq.dataverse.datasetutility;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetServiceBean;
Expand Down Expand Up @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{
// -----------------------------------
private Dataset dataset; // constructor (for add, not replace)
private DataverseRequest dvRequest; // constructor
private InputStream newFileInputStream; // step 20
private String newFileName; // step 20
private String newFileContentType; // step 20
private String newStorageIdentifier; // step 20
private String newCheckSum; // step 20
private InputStream newFileInputStream; // step 30
private String newFileName; // step 30
private String newFileContentType; // step 30
private String newStorageIdentifier; // step 30
private String newCheckSum; // step 30
private ChecksumType newCheckSumType; //step 30

// -- Optional
private DataFile fileToReplace; // step 25
Expand Down Expand Up @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner,
if(optionalFileParams != null) {
if(optionalFileParams.hasCheckSum()) {
newCheckSum = optionalFileParams.getCheckSum();
newCheckSumType = optionalFileParams.getCheckSumType();
}
}

Expand Down Expand Up @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){
this.newFileContentType,
this.newStorageIdentifier,
this.newCheckSum,
this.newCheckSumType,
this.systemConfig);

} catch (IOException ex) {
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package edu.harvard.iq.dataverse.datasetutility;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.FileMetadata;
import java.io.InputStream;
Expand Down Expand Up @@ -93,7 +94,7 @@ public boolean resetReplaceFileHelper(){
* @param checkSum
* @param event
*/
public boolean handleNativeFileUpload(InputStream inputStream, String fullStorageId, String fileName, String fileContentType, String checkSum) {
public boolean handleNativeFileUpload(InputStream inputStream, String fullStorageId, String fileName, String fileContentType, String checkSumValue, ChecksumType checkSumType) {

phase1Success = false;

Expand All @@ -110,14 +111,14 @@ public boolean handleNativeFileUpload(InputStream inputStream, String fullStorag
}

OptionalFileParams ofp = null;
if(checkSum != null) {
try {
ofp = new OptionalFileParams(null);
} catch (DataFileTagException e) {
//Shouldn't happen with null input
e.printStackTrace();
}
ofp.setCheckSum(checkSum);
if(checkSumValue != null) {
try {
ofp = new OptionalFileParams(null);
} catch (DataFileTagException e) {
// Shouldn't happen with null input
e.printStackTrace();
}
ofp.setCheckSum(checkSumValue, checkSumType);
}
// Run 1st phase of replace
//
Expand Down
39 changes: 30 additions & 9 deletions src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.DataFileTag;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.api.Util;
Expand Down Expand Up @@ -68,8 +70,12 @@ public class OptionalFileParams {
public static final String FILE_NAME_ATTR_NAME = "fileName";
private String mimeType;
public static final String MIME_TYPE_ATTR_NAME = "mimeType";
private String checkSum;
public static final String CHECKSUM_ATTR_NAME = "md5Hash";
private String checkSumValue;
private ChecksumType checkSumType;
public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash";
public static final String CHECKSUM_OBJECT_NAME = "checksum";
public static final String CHECKSUM_OBJECT_TYPE = "@type";
public static final String CHECKSUM_OBJECT_VALUE = "@value";


public OptionalFileParams(String jsonData) throws DataFileTagException{
Expand Down Expand Up @@ -217,17 +223,22 @@ public String getMimeType() {
return mimeType;
}

public void setCheckSum(String checkSum) {
this.checkSum = checkSum;
public void setCheckSum(String checkSum, ChecksumType type) {
this.checkSumValue = checkSum;
this.checkSumType = type;
}

public boolean hasCheckSum() {
return ((checkSum!=null)&&(!checkSum.isEmpty()));
return ((checkSumValue!=null)&&(!checkSumValue.isEmpty()));
}

public String getCheckSum() {
return checkSum;
return checkSumValue;
}

public ChecksumType getCheckSumType() {
return checkSumType;
}

/**
* Set tags
Expand Down Expand Up @@ -350,11 +361,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{
}

// -------------------------------
// get checkSum as string
// get md5 checkSum as string
// -------------------------------
if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){

this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString();
this.checkSumType= ChecksumType.MD5;
}
// -------------------------------
if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){
// get checkSum type and value
// -------------------------------
else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){

this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString();
this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString());

this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString();
}

// -------------------------------
Expand Down
24 changes: 18 additions & 6 deletions src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,9 +719,24 @@ public static String generateOriginalExtension(String fileType) {
return "";
}

public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException {
public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream,
String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum,
SystemConfig systemConfig) throws IOException {
ChecksumType checkSumType = DataFile.ChecksumType.MD5;
if (newStorageIdentifier == null) {
checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
}
return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig);
}

public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException {
List<DataFile> datafiles = new ArrayList<>();

//When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default
if(newCheckSumType == null) {
newCheckSumType = systemConfig.getFileFixityChecksumAlgorithm();
}

String warningMessage = null;

// save the file, in the temporary location for now:
Expand Down Expand Up @@ -1106,12 +1121,9 @@ public static List<DataFile> createDataFiles(DatasetVersion version, InputStream
if (tempFile != null) {
newFile = tempFile.toFile();
}
ChecksumType checkSumType = DataFile.ChecksumType.MD5;
if (newStorageIdentifier == null) {
checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
}


DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum);
DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
File f = null;
if (tempFile != null) {
f = tempFile.toFile();
Expand Down