Skip to content

Commit

Permalink
Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390)
Browse files Browse the repository at this point in the history
The main highlight is the removal of the reclaim_deletes_weight in the TieredMergePolicy.
The es setting index.merge.policy.reclaim_deletes_weight is deprecated in this commit and the value is ignored. The new merge policy setting setDeletesPctAllowed should be added in a follow up.
  • Loading branch information
jimczi authored Jul 27, 2018
1 parent 9198bcf commit 53ff06e
Show file tree
Hide file tree
Showing 61 changed files with 50 additions and 62 deletions.
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 7.0.0-alpha1
lucene = 7.4.0
lucene = 7.5.0-snapshot-608f0277b0

# optional dependencies
spatial4j = 0.7
Expand Down
4 changes: 2 additions & 2 deletions docs/Versions.asciidoc
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
:version: 7.0.0-alpha1
:major-version: 7.x
:lucene_version: 7.4.0
:lucene_version_path: 7_4_0
:lucene_version: 7.5.0
:lucene_version_path: 7_5_0
:branch: master
:jdk: 1.8.0_131
:jdk_major: 8
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bd7d8078a2d0ad11a24f54156cc015630c96858a
2 changes: 1 addition & 1 deletion plugins/analysis-icu/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ forbiddenApis {

dependencies {
compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
compile 'com.ibm.icu:icu4j:61.1'
compile 'com.ibm.icu:icu4j:62.1'
}

dependencyLicenses {
Expand Down
1 change: 0 additions & 1 deletion plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a4d00d5ec5febd252a6182e8b6e87a0a9821f81

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a37816def72a748416c4ae8b0f6817e30efb99f

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ca7437178cdbf7b8bfe0d75c75e3c8eb93925724

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3f5dec44f380d6d58bc1c8aec51964fcb5390b60

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
453bf1d60df0415439095624e0b3e42492ad4716

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
70095a45257bca9f46629b5fb6cedf9eff5e2b07

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7199d6962d268b7877f7b5160e98e4ff21cce5c7

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
12aff508d39d206a1aead5013ecd11882062eb06
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new MMapDirectory(file));
}

@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
}

@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}
1 change: 0 additions & 1 deletion server/licenses/lucene-analyzers-common-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d27958843ca118db2ffd2c242ae3761bd5a47328
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ea220ba8e4accb8b04e280463042ad470e23bc0
1 change: 0 additions & 1 deletion server/licenses/lucene-core-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
471096d6e92338b208aa91f3a85feb2f9cfc4afd
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f0af947c60d24f779c22f774e81ebd7dd91cc932
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fbc83ac5a0139ed7e7faf6c95a2718f46f28c641
1 change: 0 additions & 1 deletion server/licenses/lucene-join-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
30adfe493982b0db059dc243e269eea38d850d46
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
656f304261d9aad05070fb68593beffafe9147e3
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8bf22ad81a7480c255b55bada401eb131bfdb4df
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ece30d5f1e18d96f61644451c858c3d9960558f
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ad3bd0c2ed96556193c7215bef328e689d0b157f
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
07e748d2d80000a7a213f3405b82b6e26b452948
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fd737bd5562f3943618ee7e73a0aaffb6319fdb2
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
public static final int V_7_0_0_alpha1_ID = 7000001;
public static final Version V_7_0_0_alpha1 =
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
public static final Version CURRENT = V_7_0_0_alpha1;

static {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCou

@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
}

public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
Expand Down Expand Up @@ -108,13 +108,4 @@ public void setSegmentsPerTier(double segmentsPerTier) {
public double getSegmentsPerTier() {
return regularMergePolicy.getSegmentsPerTier();
}

public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}

public double getReclaimDeletesWeight() {
return regularMergePolicy.getReclaimDeletesWeight();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,6 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);

scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
mergeSchedulerConfig::setMaxThreadAndMergeCount);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public final class MergePolicyConfig {
Property.Dynamic, Property.IndexScope);
public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
Property.Dynamic, Property.IndexScope);
Property.Dynamic, Property.IndexScope, Property.Deprecated);
public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin


Expand All @@ -176,17 +176,12 @@ public final class MergePolicyConfig {
mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
mergePolicy.setSegmentsPerTier(segmentsPerTier);
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
if (logger.isTraceEnabled()) {
logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
}
}

void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}

void setSegmentsPerTier(Double segmentsPerTier) {
mergePolicy.setSegmentsPerTier(segmentsPerTier);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,14 @@ public void testDefaults() {
assertEquals(
new TieredMergePolicy().getMaxMergedSegmentMB(),
policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}

public void testSetMaxMergedSegmentMB() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergedSegmentMB(10 * 1024);
assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}

public void testSetForceMergeDeletesPctAllowed() {
Expand Down Expand Up @@ -71,10 +69,4 @@ public void testSetSegmentsPerTier() {
policy.setSegmentsPerTier(42);
assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
}

public void testSetReclaimDeletesWeight() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setReclaimDeletesWeight(42);
assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);

assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);

assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
Expand All @@ -110,7 +106,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
}

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
471096d6e92338b208aa91f3a85feb2f9cfc4afd

0 comments on commit 53ff06e

Please sign in to comment.