diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java index d1151ccb4db..7a7b0c22a1d 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java @@ -1,6 +1,5 @@ package org.orcid.persistence.dao; -import java.util.Date; import java.util.List; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; @@ -17,5 +16,5 @@ public interface GroupIdRecordDao extends GenericDao boolean duplicateExists(Long putCode, String groupId); - List getIssnRecordsNotModifiedSince(int batchSize, Date start); + List getIssnRecordsSortedById(int batchSize, long initialId); } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java index bd239f72aeb..7befb1feaa5 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java @@ -1,6 +1,5 @@ package org.orcid.persistence.dao.impl; -import java.util.Date; import java.util.List; import javax.persistence.Query; @@ -9,8 +8,8 @@ import org.orcid.persistence.dao.GroupIdRecordDao; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; -public class GroupIdRecordDaoImpl extends GenericDaoImpl implements GroupIdRecordDao { - +public class GroupIdRecordDaoImpl extends GenericDaoImpl implements GroupIdRecordDao { + public GroupIdRecordDaoImpl() { super(GroupIdRecordEntity.class); } @@ -71,13 +70,13 @@ public boolean duplicateExists(Long putCode, String groupId) { } Long result = query.getSingleResult(); return (result != null && result > 0); - } - - @SuppressWarnings("unchecked") + } + @Override - public List getIssnRecordsNotModifiedSince(int pageSize, Date date) { - Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g LEFT OUTER JOIN invalid_issn_group_id_record p ON g.id = p.id where p.id IS NULL AND g.group_id like 'issn:%' and g.last_modified < :date", GroupIdRecordEntity.class); - query.setParameter("date", date); + public List getIssnRecordsSortedById(int batchSize, long initialId) { + Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g LEFT OUTER JOIN invalid_issn_group_id_record p ON g.id = p.id where p.id IS NULL AND g.group_id like 'issn:%' and g.id > :initialId order by g.id", GroupIdRecordEntity.class); + query.setParameter("initialId", initialId); + query.setMaxResults(batchSize); return query.getResultList(); } } diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java index 25527fa6cdc..1424912b39a 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java @@ -1,6 +1,5 @@ package org.orcid.scheduler.loader.source.issn; -import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,7 +28,7 @@ public class IssnLoadSource { private static Pattern issnGroupTypePattern = Pattern.compile("^issn:(\\d{4}-\\d{3}[\\dXx])$"); - @Value("${org.orcid.scheduler.issnLoadSource.batchSize:50}") + @Value("${org.orcid.scheduler.issnLoadSource.batchSize:5000}") private int batchSize; @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:30000}") @@ -69,8 +68,10 @@ public void loadIssn(String issnSource) { } private void updateIssnGroupIdRecords() { - Date start = new Date(); - List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(batchSize, start); + Long nextBatchStartId = 0L; + // Get the first batch of issn's + LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId + " batch size: " + batchSize); + List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId); int batchCount = 0; int total = 0; while (!issnEntities.isEmpty()) { @@ -104,8 +105,13 @@ private void updateIssnGroupIdRecords() { // TODO Auto-generated catch block LOG.warn("Exception while pausing the issn loader", e); } + + if (issnEntity.getId() > nextBatchStartId) { + nextBatchStartId = issnEntity.getId(); + } } - issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsNotModifiedSince(batchSize, start); + LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId); + issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId); } }