Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: optmize cleander job #3737

Merged
merged 1 commit into from
Jan 31, 2023
Merged

feat: optmize cleander job #3737

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 103 additions & 52 deletions jans-fido2/server/src/main/java/io/jans/fido2/service/app/CleanerTimer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,29 @@
package io.jans.fido2.service.app;

import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import org.slf4j.Logger;

import com.google.common.base.Stopwatch;
import com.google.common.collect.Sets;
import com.google.common.collect.Maps;

import io.jans.as.model.config.StaticConfiguration;
import io.jans.fido2.model.conf.AppConfiguration;
import io.jans.fido2.service.persist.AuthenticationPersistenceService;
import io.jans.fido2.service.persist.RegistrationPersistenceService;
import io.jans.orm.PersistenceEntryManager;
import io.jans.orm.model.base.DeletableEntity;
import io.jans.orm.model.BatchOperation;
import io.jans.orm.model.ProcessBatchOperation;
import io.jans.orm.model.SearchScope;
import io.jans.orm.model.base.SimpleBranch;
import io.jans.orm.model.fido2.Fido2AuthenticationEntry;
import io.jans.orm.model.fido2.Fido2RegistrationEntry;
import io.jans.orm.search.filter.Filter;
import io.jans.service.cdi.async.Asynchronous;
import io.jans.service.cdi.event.CleanerEvent;
Expand All @@ -48,7 +56,7 @@ public class CleanerTimer {
private Logger log;

@Inject
private PersistenceEntryManager persistenceEntryManager;
private PersistenceEntryManager entryManager;

@Inject
private StaticConfiguration staticConfiguration;
Expand Down Expand Up @@ -112,68 +120,111 @@ private boolean isStartProcess() {
return timeDiffrence >= cleaningInterval;
}

public void processImpl() {
try {
if (!isStartProcess()) {
log.trace("Starting conditions aren't reached");
return;
}
public void processImpl() {
try {
if (!isStartProcess()) {
log.trace("Starting conditions aren't reached");
return;
}

int chunkSize = appConfiguration.getCleanServiceBatchChunkSize();
if (chunkSize <= 0)
chunkSize = BATCH_SIZE;
int chunkSize = appConfiguration.getCleanServiceBatchChunkSize();
if (chunkSize <= 0)
chunkSize = BATCH_SIZE;

Date now = new Date();
Date now = new Date();

for (String baseDn : createCleanServiceBaseDns()) {
try {
if (persistenceEntryManager.hasExpirationSupport(baseDn)) {
continue;
}
final Set<String> processedBaseDns = new HashSet<>();
for (Map.Entry<String, Class<?>> baseDn : createCleanServiceBaseDns().entrySet()) {
final String processedKey = createProcessedKey(baseDn);
if (entryManager.hasExpirationSupport(baseDn.getKey()) || processedBaseDns.contains(processedKey)) {
continue;
}

log.debug("Start clean up for baseDn: " + baseDn);
final Stopwatch started = Stopwatch.createStarted();
processedBaseDns.add(processedKey);

int removed = cleanup(baseDn, now, chunkSize);
if (log.isDebugEnabled())
log.debug("Start clean up for baseDn: {}, class: {}", baseDn.getValue(), baseDn.getValue());

log.debug("Finished clean up for baseDn: {}, takes: {}ms, removed items: {}", baseDn,
started.elapsed(TimeUnit.MILLISECONDS), removed);
} catch (Exception e) {
log.error("Failed to process clean up for baseDn: " + baseDn, e);
}
}
final Stopwatch started = Stopwatch.createStarted();

registrationPersistenceService.cleanup(now, chunkSize);
authenticationPersistenceService.cleanup(now, chunkSize);
int removed = cleanup(baseDn, now, chunkSize);

this.lastFinishedTime = System.currentTimeMillis();
} catch (Exception e) {
log.error("Failed to process clean up.", e);
}
}
if (log.isDebugEnabled())
log.debug("Finished clean up for baseDn: {}, takes: {}ms, removed items: {}", baseDn, started.elapsed(TimeUnit.MILLISECONDS), removed);
}

public Set<String> createCleanServiceBaseDns() {
final Set<String> cleanServiceBaseDns = Sets.newHashSet();
// Process sub-branches
String baseDn = staticConfiguration.getBaseDn().getPeople();
if (log.isDebugEnabled())
log.debug("Start clean up for baseDn: {}", baseDn);

cleanServiceBaseDns.add(staticConfiguration.getBaseDn().getFido2Attestation());
cleanServiceBaseDns.add(staticConfiguration.getBaseDn().getFido2Assertion());
final Stopwatch started = Stopwatch.createStarted();

log.debug("Built-in base dns: " + cleanServiceBaseDns);
int removed = cleanupBranches(baseDn, now, chunkSize);

return cleanServiceBaseDns;
}
if (log.isDebugEnabled())
log.debug("Finished clean up for baseDn: {}, takes: {}ms, removed items: {}", baseDn, started.elapsed(TimeUnit.MILLISECONDS), removed);

public int cleanup(final String baseDn, final Date now, final int batchSize) {
try {
Filter filter = Filter.createANDFilter(Filter.createEqualityFilter("del", true),
Filter.createLessOrEqualFilter("exp", persistenceEntryManager.encodeTime(baseDn, now)));

return persistenceEntryManager.remove(baseDn, DeletableEntity.class, filter, batchSize);
} catch (Exception e) {
log.error("Failed to perform clean up.", e);
}
this.lastFinishedTime = System.currentTimeMillis();
} catch (Exception e) {
log.error("Failed to process clean up.", e);
}
}

private static String createProcessedKey(Map.Entry<String, Class<?>> baseDn) {
return baseDn.getKey() + "_" + (baseDn.getValue() == null ? "" : baseDn.getValue().getSimpleName());
}

private Map<String, Class<?>> createCleanServiceBaseDns() {

final Map<String, Class<?>> cleanServiceBaseDns = Maps.newHashMap();

cleanServiceBaseDns.put(staticConfiguration.getBaseDn().getFido2Attestation(), Fido2RegistrationEntry.class);
cleanServiceBaseDns.put(staticConfiguration.getBaseDn().getFido2Assertion(), Fido2AuthenticationEntry.class);
cleanServiceBaseDns.put(staticConfiguration.getBaseDn().getPeople(), Fido2RegistrationEntry.class);
cleanServiceBaseDns.put(staticConfiguration.getBaseDn().getPeople(), Fido2AuthenticationEntry.class);

return cleanServiceBaseDns;
}

public int cleanup(final Map.Entry<String, Class<?>> baseDn, final Date now, final int batchSize) {
try {
Filter filter = Filter.createANDFilter(
Filter.createEqualityFilter("del", true),
Filter.createLessOrEqualFilter("exp", entryManager.encodeTime(baseDn.getKey(), now)));

int removedCount = entryManager.remove(baseDn.getKey(), baseDn.getValue(), filter, batchSize);
log.trace("Removed {} entries from {}", removedCount, baseDn.getKey());
return removedCount;
} catch (Exception e) {
log.error("Failed to perform clean up.", e);
}

return 0;
}

public int cleanupBranches(String branchDn, Date now, int batchSize) {
try {
// Cleaning branches entries
if (entryManager.hasExpirationSupport(branchDn) || !entryManager.hasBranchesSupport(branchDn)) {
return 0;
}

return 0;
}
// Cleaning empty branches
Filter filter = Filter.createANDFilter(
Filter.createORFilter(Filter.createEqualityFilter("ou", "fido2_register"),
Filter.createEqualityFilter("ou", "fido2_auth")),
Filter.createORFilter(Filter.createEqualityFilter("numsubordinates", "0"),
Filter.createEqualityFilter("hasSubordinates", "FALSE")));

int removedCount = entryManager.remove(branchDn, SimpleBranch.class, filter, batchSize);
log.trace("Removed {} entries from {}", removedCount, branchDn);
return removedCount;
} catch (Exception e) {
log.error("Failed to perform clean up.", e);
}

return 0;
}

}
81 changes: 0 additions & 81 deletions .../server/src/main/java/io/jans/fido2/service/persist/AuthenticationPersistenceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,85 +189,4 @@ public String getDnForUser(String userInum) {
return String.format("inum=%s,%s", userInum, peopleDn);
}

public void cleanup(Date now, int batchSize) {
// Cleaning expired entries
BatchOperation<Fido2AuthenticationEntry> cleanerAuthenticationBatchService = new ProcessBatchOperation<Fido2AuthenticationEntry>() {
@Override
public void performAction(List<Fido2AuthenticationEntry> entries) {
for (Fido2AuthenticationEntry p : entries) {
log.debug("Removing Fido2 authentication entry: {}, Creation date: {}", p.getChallange(), p.getCreationDate());
try {
persistenceEntryManager.remove(p);
} catch (Exception e) {
log.error("Failed to remove entry", e);
}
}
}
};

String baseDn = getDnForUser(null);
if (persistenceEntryManager.hasExpirationSupport(baseDn)) {
return;
}
persistenceEntryManager.findEntries(baseDn, Fido2AuthenticationEntry.class, getExpiredAuthenticationFilter(baseDn), SearchScope.SUB, new String[] {"jansCodeChallenge", "creationDate"}, cleanerAuthenticationBatchService, 0, 0, batchSize);

String branchDn = getDnForUser(null);
if (persistenceEntryManager.hasBranchesSupport(branchDn)) {
// Cleaning empty branches
BatchOperation<SimpleBranch> cleanerBranchBatchService = new ProcessBatchOperation<SimpleBranch>() {
@Override
public void performAction(List<SimpleBranch> entries) {
for (SimpleBranch p : entries) {
try {
persistenceEntryManager.remove(p);
} catch (Exception e) {
log.error("Failed to remove entry", e);
}
}
}
};
persistenceEntryManager.findEntries(getDnForUser(null), SimpleBranch.class, getEmptyAuthenticationBranchFilter(), SearchScope.SUB, new String[] {"ou"}, cleanerBranchBatchService, 0, 0, batchSize);
}
}

private Filter getExpiredAuthenticationFilter(String baseDn) {
int unfinishedRequestExpiration = appConfiguration.getFido2Configuration().getUnfinishedRequestExpiration();
unfinishedRequestExpiration = unfinishedRequestExpiration == 0 ? 120 : unfinishedRequestExpiration;

int authenticationHistoryExpiration = appConfiguration.getFido2Configuration().getAuthenticationHistoryExpiration();
authenticationHistoryExpiration = authenticationHistoryExpiration == 0 ? 15 * 24 * 3600 : authenticationHistoryExpiration;

Calendar calendar1 = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
calendar1.add(Calendar.SECOND, -unfinishedRequestExpiration);
final Date unfinishedRequestExpirationDate = calendar1.getTime();

Calendar calendar2 = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
calendar2.add(Calendar.SECOND, -authenticationHistoryExpiration);
final Date authenticationHistoryExpirationDate = calendar2.getTime();

// Build unfinished request expiration filter
Filter authenticationStatusFilter1 = Filter.createNOTFilter(Filter.createEqualityFilter("jansStatus", Fido2AuthenticationStatus.authenticated.getValue()));

Filter exirationDateFilter1 = Filter.createLessOrEqualFilter("creationDate",
persistenceEntryManager.encodeTime(baseDn, unfinishedRequestExpirationDate));

Filter unfinishedRequestFilter = Filter.createANDFilter(authenticationStatusFilter1, exirationDateFilter1);

// Build authentication history expiration filter
Filter authenticationStatusFilter2 = Filter.createEqualityFilter("jansStatus", Fido2AuthenticationStatus.authenticated.getValue());

Filter exirationDateFilter2 = Filter.createLessOrEqualFilter("creationDate",
persistenceEntryManager.encodeTime(baseDn, authenticationHistoryExpirationDate));

Filter authenticationHistoryFilter = Filter.createANDFilter(authenticationStatusFilter2, exirationDateFilter2);

return Filter.createORFilter(unfinishedRequestFilter, authenticationHistoryFilter);
}

private Filter getEmptyAuthenticationBranchFilter() {
return Filter.createANDFilter(Filter.createEqualityFilter("ou", "fido2_auth"), Filter.createORFilter(
Filter.createEqualityFilter("numsubordinates", "0"), Filter.createEqualityFilter("hasSubordinates", "FALSE")));
}


}
69 changes: 0 additions & 69 deletions ...o2/server/src/main/java/io/jans/fido2/service/persist/RegistrationPersistenceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
package io.jans.fido2.service.persist;

import java.nio.charset.StandardCharsets;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
Expand All @@ -25,10 +24,6 @@
import io.jans.fido2.model.conf.AppConfiguration;
import io.jans.fido2.service.shared.UserService;
import io.jans.orm.PersistenceEntryManager;
import io.jans.orm.model.BatchOperation;
import io.jans.orm.model.ProcessBatchOperation;
import io.jans.orm.model.SearchScope;
import io.jans.orm.model.base.SimpleBranch;
import io.jans.orm.model.fido2.Fido2RegistrationData;
import io.jans.orm.model.fido2.Fido2RegistrationEntry;
import io.jans.orm.model.fido2.Fido2RegistrationStatus;
Expand Down Expand Up @@ -179,70 +174,6 @@ public String getBasedPeopleDn() {
return staticConfiguration.getBaseDn().getPeople();
}

public void cleanup(Date now, int batchSize) {
// Cleaning expired entries
BatchOperation<Fido2RegistrationEntry> cleanerRegistrationBatchService = new ProcessBatchOperation<Fido2RegistrationEntry>() {
@Override
public void performAction(List<Fido2RegistrationEntry> entries) {
for (Fido2RegistrationEntry p : entries) {
log.debug("Removing Fido2 registration entry: {}, Creation date: {}", p.getChallange(), p.getCreationDate());
try {
persistenceEntryManager.remove(p);
} catch (Exception e) {
log.error("Failed to remove entry", e);
}
}
}
};
String baseDn = getDnForUser(null);
if (persistenceEntryManager.hasExpirationSupport(baseDn)) {
return;
}
persistenceEntryManager.findEntries(baseDn, Fido2RegistrationEntry.class, getExpiredRegistrationFilter(baseDn), SearchScope.SUB, new String[] {"jansCodeChallenge", "creationDate"}, cleanerRegistrationBatchService, 0, 0, batchSize);

String branchDn = getDnForUser(null);
if (persistenceEntryManager.hasBranchesSupport(branchDn)) {
// Cleaning empty branches
BatchOperation<SimpleBranch> cleanerBranchBatchService = new ProcessBatchOperation<SimpleBranch>() {
@Override
public void performAction(List<SimpleBranch> entries) {
for (SimpleBranch p : entries) {
try {
persistenceEntryManager.remove(p);
} catch (Exception e) {
log.error("Failed to remove entry", e);
}
}
}
};
persistenceEntryManager.findEntries(branchDn, SimpleBranch.class, getEmptyRegistrationBranchFilter(), SearchScope.SUB, new String[] {"ou"}, cleanerBranchBatchService, 0, 0, batchSize);
}
}

private Filter getExpiredRegistrationFilter(String baseDn) {
int unfinishedRequestExpiration = appConfiguration.getFido2Configuration().getUnfinishedRequestExpiration();
unfinishedRequestExpiration = unfinishedRequestExpiration == 0 ? 120 : unfinishedRequestExpiration;

Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
calendar.add(Calendar.SECOND, -unfinishedRequestExpiration);
final Date unfinishedRequestExpirationDate = calendar.getTime();

// Build unfinished request expiration filter
Filter registrationStatusFilter = Filter.createNOTFilter(Filter.createEqualityFilter("jansStatus", Fido2RegistrationStatus.registered.getValue()));

Filter exirationDateFilter = Filter.createLessOrEqualFilter("creationDate",
persistenceEntryManager.encodeTime(baseDn, unfinishedRequestExpirationDate));

Filter unfinishedRequestFilter = Filter.createANDFilter(registrationStatusFilter, exirationDateFilter);

return unfinishedRequestFilter;
}

private Filter getEmptyRegistrationBranchFilter() {
return Filter.createANDFilter(Filter.createEqualityFilter("ou", "fido2_register"), Filter.createORFilter(
Filter.createEqualityFilter("numsubordinates", "0"), Filter.createEqualityFilter("hasSubordinates", "FALSE")));
}

public int getChallengeHashCode(String challenge) {
int hash = 0;
byte[] challengeBytes = challenge.getBytes(StandardCharsets.UTF_8);
Expand Down