Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adjust CombinedDeletionPolicy for multiple commits #27456

Merged
merged 1 commit into from
Nov 23, 2017
Merged

Adjust CombinedDeletionPolicy for multiple commits #27456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,18 @@ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
}

private void setLastCommittedTranslogGeneration(List<? extends IndexCommit> commits) throws IOException {
// when opening an existing lucene index, we currently always open the last commit.
// we therefore use the translog gen as the one that will be required for recovery
final IndexCommit indexCommit = commits.get(commits.size() - 1);
assert indexCommit.isDeleted() == false : "last commit is deleted";
long minGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minGen);
// We need to keep translog since the smallest translog generation of un-deleted commits.
// However, there are commits that are not deleted just because they are being snapshotted (rather than being kept by the policy).
// TODO: We need to distinguish those commits and skip them in calculating the minimum required translog generation.
long minRequiredGen = Long.MAX_VALUE;
for (IndexCommit indexCommit : commits) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can do some streaming java8 magic here.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, but indexCommit.getUserData() throws IOException.

if (indexCommit.isDeleted() == false) {
long translogGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
minRequiredGen = Math.min(translogGen, minRequiredGen);
}
}
assert minRequiredGen != Long.MAX_VALUE : "All commits are deleted";
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minRequiredGen);
}

public SnapshotDeletionPolicy getIndexDeletionPolicy() {
Expand Down
13 changes: 8 additions & 5 deletions core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,20 +60,23 @@ public void testSettingMinTranslogGen() throws IOException {
EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
List<IndexCommit> commitList = new ArrayList<>();
long count = randomIntBetween(10, 20);
long lastGen = 0;
long minGen = Long.MAX_VALUE;
for (int i = 0; i < count; i++) {
lastGen += randomIntBetween(10, 20000);
long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
commitList.add(mockIndexCommitWithTranslogGen(lastGen));
}
combinedDeletionPolicy.onInit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
commitList.clear();
minGen = Long.MAX_VALUE;
for (int i = 0; i < count; i++) {
lastGen += randomIntBetween(10, 20000);
long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
commitList.add(mockIndexCommitWithTranslogGen(lastGen));
}
combinedDeletionPolicy.onCommit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
}

IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException {
Expand Down