Skip to content

Commit

Permalink
tickets #2650 (dataset page optimizations); #2648 (file from an unpub…
Browse files Browse the repository at this point in the history 
…lished version could be downloaded);
  • Loading branch information
@landreev
landreev committed Oct 22, 2015
1 parent 506b470 commit 22ba520
Show file tree
Hide file tree
Showing 11 changed files with 725 additions and 289 deletions.
4 changes: 4 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/DataFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,10 @@ public boolean isImage() {
// generate thumbnails and previews for them)
return (contentType != null && (contentType.startsWith("image/") || contentType.equalsIgnoreCase("application/pdf")));
}

public void setIngestStatus(char ingestStatus) {
this.ingestStatus = ingestStatus;
}

public boolean isIngestScheduled() {
return (ingestStatus == INGEST_STATUS_SCHEDULED);
Expand Down
287 changes: 281 additions & 6 deletions src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@
package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
import edu.harvard.iq.dataverse.search.SortBy;
import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand Down Expand Up @@ -43,6 +47,8 @@ public class DataFileServiceBean implements java.io.Serializable {
DatasetServiceBean datasetService;
@EJB
PermissionServiceBean permissionService;
@EJB
UserServiceBean userService;

@PersistenceContext(unitName = "VDCNet-ejbPU")
private EntityManager em;
Expand Down Expand Up @@ -125,8 +131,6 @@ public DataFile find(Object pk) {
return (DataFile) em.find(DataFile.class, pk);
}



/*public DataFile findByMD5(String md5Value){
if (md5Value == null){
return null;
Expand All @@ -146,7 +150,7 @@ public List<DataFile> findByDatasetId(Long studyId) {
query.setParameter("studyId", studyId);
return query.getResultList();
}

public List<FileMetadata> findFileMetadataByDatasetVersionId(Long datasetVersionId, int maxResults, String userSuppliedSortField, String userSuppliedSortOrder) {
FileSortFieldAndOrder sortFieldAndOrder = new FileSortFieldAndOrder(userSuppliedSortField, userSuppliedSortOrder);
String sortField = sortFieldAndOrder.getSortField();
Expand Down Expand Up @@ -211,7 +215,7 @@ public Long findCountByDatasetVersionId(Long datasetVersionId){
+ " where fmd.datasetVersion_id = " + datasetVersionId
+ ";").getSingleResult();
}

public FileMetadata findFileMetadataByFileAndVersionId(Long dataFileId, Long datasetVersionId) {
Query query = em.createQuery("select object(o) from FileMetadata as o where o.dataFile.id = :dataFileId and o.datasetVersion.id = :datasetVersionId");
query.setParameter("dataFileId", dataFileId);
Expand All @@ -228,6 +232,266 @@ public FileMetadata findFileMetadataByDatasetVersionIdAndDataFileId(Long dataset
return (FileMetadata) query.getSingleResult();
}

public List<FileMetadata> findFileMetadataByDatasetAndVersionExperimental(Dataset owner, DatasetVersion version) {
List<DataFile> dataFiles = new ArrayList<>();
List<DataTable> dataTables = new ArrayList<>();
List<FileMetadata> retList = new ArrayList<>();

Map<Long, AuthenticatedUser> userMap = new HashMap<>();
Map<Long, Integer> filesMap = new HashMap<>();
Map<Long, Integer> datatableMap = new HashMap<>();


int i = 0;

List<Object[]> dataTableResults = em.createNativeQuery("SELECT t0.ID, t0.DATAFILE_ID, t0.UNF, t0.CASEQUANTITY, t0.VARQUANTITY FROM dataTable t0, dataFile t1, dvObject t2 WHERE ((t0.DATAFILE_ID = t1.ID) AND (t1.ID = t2.ID) AND (t2.OWNER_ID = " + owner.getId() + "))").getResultList();

for (Object[] result : dataTableResults) {
DataTable dataTable = new DataTable();
Long fileId = (Long)result[1];

dataTable.setId(((Integer)result[0]).longValue());

dataTable.setUnf((String)result[2]);

dataTable.setCaseQuantity((Long)result[3]);

dataTable.setVarQuantity((Long)result[4]);

dataTables.add(dataTable);
datatableMap.put(fileId, i++);

}
dataTableResults = null;
i = 0;

List<Object[]> fileResults = em.createNativeQuery("SELECT t0.ID, t0.CREATEDATE, t0.INDEXTIME, t0.MODIFICATIONTIME, t0.PERMISSIONINDEXTIME, t0.PERMISSIONMODIFICATIONTIME, t0.PUBLICATIONDATE, t0.CREATOR_ID, t0.RELEASEUSER_ID, t1.CONTENTTYPE, t1.FILESYSTEMNAME, t1.FILESIZE, t1.INGESTSTATUS, t1.MD5, t1.RESTRICTED FROM DVOBJECT t0, DATAFILE t1 WHERE ((t0.OWNER_ID = " + owner.getId() + ") AND ((t1.ID = t0.ID) AND (t0.DTYPE = 'DataFile')))").getResultList();

for (Object[] result : fileResults) {
Integer file_id = (Integer) result[0];

DataFile dataFile = new DataFile();

dataFile.setId(file_id.longValue());

Timestamp createDate = (Timestamp) result[1];
Timestamp indexTime = (Timestamp) result[2];
Timestamp modificationTime = (Timestamp) result[3];
Timestamp permissionIndexTime = (Timestamp) result[4];
Timestamp permissionModificationTime = (Timestamp) result[5];
Timestamp publicationDate = (Timestamp) result[6];

dataFile.setCreateDate(createDate);
dataFile.setIndexTime(indexTime);
dataFile.setModificationTime(modificationTime);
dataFile.setPermissionIndexTime(permissionIndexTime);
dataFile.setPermissionModificationTime(permissionModificationTime);
dataFile.setPublicationDate(publicationDate);

Long creatorId = (Long) result[7];
if (creatorId != null) {
AuthenticatedUser creator = userMap.get(creatorId);
if (creator == null) {
creator = userService.find(creatorId);
if (creator != null) {
userMap.put(creatorId, creator);
}
}
if (creator != null) {
dataFile.setCreator(creator);
}
}

dataFile.setOwner(owner);

Long releaseUserId = (Long) result[8];
if (releaseUserId != null) {
AuthenticatedUser releaseUser = userMap.get(releaseUserId);
if (releaseUser == null) {
releaseUser = userService.find(releaseUserId);
if (releaseUser != null) {
userMap.put(releaseUserId, releaseUser);
}
}
if (releaseUser != null) {
dataFile.setReleaseUser(releaseUser);
}
}

String contentType = (String) result[9];

if (contentType != null) {
dataFile.setContentType(contentType);
}

String storageIdentifier = (String) result[10];

if (storageIdentifier != null) {
dataFile.setStorageIdentifier(storageIdentifier);
}

Long fileSize = (Long) result[11];

if (fileSize != null) {
dataFile.setFilesize(fileSize);
}

if (result[12] != null) {
String ingestStatusString = (String) result[12];
dataFile.setIngestStatus(ingestStatusString.charAt(0));
logger.info("Ingest Status: "+dataFile.getIngestStatus()+" (file id: "+dataFile.getId()+")");
}

String md5 = (String) result[13];

if (md5 != null) {
dataFile.setmd5(md5);
}

Boolean restricted = (Boolean) result[14];
if (restricted != null) {
dataFile.setRestricted(restricted);
}

// TODO:
// - if ingest status is "bad", look up the ingest report;
// - is it a dedicated thumbnail for the dataset? (do we ever need that info?? - not on the dataset page, I don't think...)

// Is this a tabular file?

if (datatableMap.get(dataFile.getId()) != null) {
dataTables.get(datatableMap.get(dataFile.getId())).setDataFile(dataFile);
dataFile.setDataTable(dataTables.get(datatableMap.get(dataFile.getId())));

}

dataFiles.add(dataFile);
filesMap.put(dataFile.getId(), i++);
}

owner.setFiles(dataFiles);
fileResults = null;

// and now, file metadatas:

List<Object[]> metadataResults = em.createNativeQuery("select id, datafile_id, DESCRIPTION, LABEL, RESTRICTED from FileMetadata where datasetversion_id = "+version.getId()).getResultList();

for (Object[] result : metadataResults) {
Integer filemeta_id = (Integer) result[0];

if (filemeta_id == null) {
continue;
}

Long file_id = (Long) result[1];
if (file_id == null) {
continue;
}

Integer file_list_id = filesMap.get(file_id);
if (file_list_id == null) {
continue;
}
FileMetadata fileMetadata = new FileMetadata();
fileMetadata.setId(filemeta_id.longValue());
fileMetadata.setDatasetVersion(version);
//version.getFileMetadatas().add(fileMetadata);

fileMetadata.setDataFile(owner.getFiles().get(file_list_id));
//owner.getFiles().get(file_list_id).getFileMetadatas().add(fileMetadata);
//

String description = (String) result[2];

if (description != null) {
fileMetadata.setDescription(description);
}

String label = (String) result[3];

if (label != null) {
fileMetadata.setLabel(label);
}

Boolean restricted = (Boolean) result[4];
if (restricted != null) {
fileMetadata.setRestricted(restricted);
}

retList.add(fileMetadata);
}

metadataResults = null;

// TODO:
// combine the 2 queries above - do it all in one step... (or not??)

for (DatasetVersion otherVersion : owner.getVersions()) {
if (otherVersion.getId().equals(version.getId())) {
continue;
}

// otherwise:

otherVersion.setFileMetadatas(retrieveFileMetadataForVersion(owner, otherVersion, filesMap));

}


return retList;
}

private List<FileMetadata> retrieveFileMetadataForVersion(Dataset dataset, DatasetVersion version, Map<Long, Integer> filesMap) {
List<FileMetadata> retList = new ArrayList<>();
List<Object[]> metadataResults = em.createNativeQuery("select id, datafile_id, DESCRIPTION, LABEL, RESTRICTED from FileMetadata where datasetversion_id = "+version.getId()).getResultList();

for (Object[] result : metadataResults) {
Integer filemeta_id = (Integer) result[0];

if (filemeta_id == null) {
continue;
}

Long file_id = (Long) result[1];
if (file_id == null) {
continue;
}

Integer file_list_id = filesMap.get(file_id);
if (file_list_id == null) {
continue;
}
FileMetadata fileMetadata = new FileMetadata();
fileMetadata.setId(filemeta_id.longValue());
fileMetadata.setDatasetVersion(version);

fileMetadata.setDataFile(dataset.getFiles().get(file_list_id));

String description = (String) result[2];

if (description != null) {
fileMetadata.setDescription(description);
}

String label = (String) result[3];

if (label != null) {
fileMetadata.setLabel(label);
}

Boolean restricted = (Boolean) result[4];
if (restricted != null) {
fileMetadata.setRestricted(restricted);
}

retList.add(fileMetadata);
}

metadataResults = null;

return retList;
}

public List<DataFile> findIngestsInProgress() {
if ( em.isOpen() ) {
Query query = em.createQuery("select object(o) from DataFile as o where o.ingestStatus =:scheduledStatusCode or o.ingestStatus =:progressStatusCode order by o.id");
Expand Down Expand Up @@ -363,7 +627,7 @@ public boolean thumbnailSupported (DataFile file) {
* ready to be downloaded. (it will try to generate a thumbnail for supported
* file types, if not yet available)
*/
public boolean isThumbnailAvailable (DataFile file, User user) {
public boolean isThumbnailAvailable (DataFile file) {
if (file == null) {
return false;
}
Expand All @@ -375,13 +639,24 @@ public boolean isThumbnailAvailable (DataFile file, User user) {
return false;
}

/*
Checking the permission here was resulting in extra queries;
it is now the responsibility of the client - such as the DatasetPage -
to make sure the permission check out, before calling this method.
(or *after* calling this method? - checking permissions costs db
queries; checking if the thumbnail is available may cost cpu time, if
it has to be generated on the fly - so you have to figure out which
is more important...
TODO: adding a boolean flag isImageAlreadyGenerated to the DataFile
db table should help with this. -- L.A. 4.2.1
// Also, thumbnails are only shown to users who have permission to see
// the full-size image file. So before we do anything else, let's
// do some authentication and authorization:
if (!permissionService.userOn(user, file).has(Permission.DownloadFile)) {
logger.fine("No permission to download the file.");
return false;
}
}*/



Expand Down
Loading

0 comments on commit 22ba520

Please sign in to comment.