Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support different checksum algorithms in direct upload #7602

Merged
merged 2 commits into from
Feb 12, 2021
Merged

support different checksum algorithms in direct upload #7602

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8b4fb3
support different checksum algorithms in direct upload
qqmyers Feb 10, 2021
9322570
set default checksumtype, add support via UI
qqmyers Feb 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package edu.harvard.iq.dataverse.datasetutility;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetServiceBean;
Expand Down Expand Up @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{
// -----------------------------------
private Dataset dataset; // constructor (for add, not replace)
private DataverseRequest dvRequest; // constructor
private InputStream newFileInputStream; // step 20
private String newFileName; // step 20
private String newFileContentType; // step 20
private String newStorageIdentifier; // step 20
private String newCheckSum; // step 20
private InputStream newFileInputStream; // step 30
private String newFileName; // step 30
private String newFileContentType; // step 30
private String newStorageIdentifier; // step 30
private String newCheckSum; // step 30
private ChecksumType newCheckSumType; //step 30

// -- Optional
private DataFile fileToReplace; // step 25
Expand Down Expand Up @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner,
if(optionalFileParams != null) {
if(optionalFileParams.hasCheckSum()) {
newCheckSum = optionalFileParams.getCheckSum();
newCheckSumType = optionalFileParams.getCheckSumType();
}
}

Expand Down Expand Up @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){
this.newFileContentType,
this.newStorageIdentifier,
this.newCheckSum,
this.newCheckSumType,
this.systemConfig);

} catch (IOException ex) {
Expand Down
36 changes: 28 additions & 8 deletions src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import com.google.gson.JsonSyntaxException;
import com.google.gson.internal.LinkedTreeMap;
import com.google.gson.reflect.TypeToken;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFile.ChecksumType;
import edu.harvard.iq.dataverse.DataFileTag;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.api.Util;
Expand Down Expand Up @@ -68,8 +70,12 @@ public class OptionalFileParams {
public static final String FILE_NAME_ATTR_NAME = "fileName";
private String mimeType;
public static final String MIME_TYPE_ATTR_NAME = "mimeType";
private String checkSum;
public static final String CHECKSUM_ATTR_NAME = "md5Hash";
private String checkSumValue;
private ChecksumType checkSumType;
public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash";
public static final String CHECKSUM_OBJECT_NAME = "checksum";
public static final String CHECKSUM_OBJECT_TYPE = "@type";
public static final String CHECKSUM_OBJECT_VALUE = "@value";


public OptionalFileParams(String jsonData) throws DataFileTagException{
Expand Down Expand Up @@ -218,16 +224,20 @@ public String getMimeType() {
}

public void setCheckSum(String checkSum) {
this.checkSum = checkSum;
this.checkSumValue = checkSum;
}

public boolean hasCheckSum() {
return ((checkSum!=null)&&(!checkSum.isEmpty()));
return ((checkSumValue!=null)&&(!checkSumValue.isEmpty()));
}

public String getCheckSum() {
return checkSum;
return checkSumValue;
}

public ChecksumType getCheckSumType() {
return checkSumType;
}

/**
* Set tags
Expand Down Expand Up @@ -350,11 +360,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{
}

// -------------------------------
// get checkSum as string
// get md5 checkSum as string
// -------------------------------
if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){

this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString();
this.checkSumType= ChecksumType.MD5;
}
// -------------------------------
if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){
// get checkSum type and value
// -------------------------------
else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){

this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString();
this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString());

this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString();
}

// -------------------------------
Expand Down
19 changes: 13 additions & 6 deletions src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,7 +719,17 @@ public static String generateOriginalExtension(String fileType) {
return "";
}

public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException {
public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream,
String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum,
SystemConfig systemConfig) throws IOException {
ChecksumType checkSumType = DataFile.ChecksumType.MD5;
if (newStorageIdentifier == null) {
checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
}
return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig);
}

public static List<DataFile> createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException {
List<DataFile> datafiles = new ArrayList<>();

String warningMessage = null;
Expand Down Expand Up @@ -1106,12 +1116,9 @@ public static List<DataFile> createDataFiles(DatasetVersion version, InputStream
if (tempFile != null) {
newFile = tempFile.toFile();
}
ChecksumType checkSumType = DataFile.ChecksumType.MD5;
if (newStorageIdentifier == null) {
checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
}


DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum);
DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
File f = null;
if (tempFile != null) {
f = tempFile.toFile();
Expand Down