Skip to content

Commit

Permalink
Merge branch 'master' into allowed-index-store-types
Browse files Browse the repository at this point in the history
* master:
  Fix ordering of bootstrap checks in docs (elastic#32417)
  [TEST] Mute failing InternalEngineTests#testSeqNoAndCheckpoints
  [TEST] Mute failing testConvertLongHexError
  bump lucene version after backport
  Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (elastic#32390)
  [Kerberos] Avoid vagrant update on precommit (elastic#32416)
  • Loading branch information
jasontedor committed Jul 27, 2018
2 parents 4f74cd0 + 3ac57f0 commit 08b825b
Show file tree
Hide file tree
Showing 65 changed files with 106 additions and 117 deletions.
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 7.0.0-alpha1
lucene = 7.4.0
lucene = 7.5.0-snapshot-608f0277b0

# optional dependencies
spatial4j = 0.7
Expand Down
4 changes: 2 additions & 2 deletions docs/Versions.asciidoc
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
:version: 7.0.0-alpha1
:major-version: 7.x
:lucene_version: 7.4.0
:lucene_version_path: 7_4_0
:lucene_version: 7.5.0
:lucene_version_path: 7_5_0
:branch: master
:jdk: 1.8.0_131
:jdk_major: 8
Expand Down
26 changes: 13 additions & 13 deletions docs/reference/setup/bootstrap-checks.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,19 @@ least 4096 threads. This can be done via `/etc/security/limits.conf`
using the `nproc` setting (note that you might have to increase the
limits for the `root` user too).

=== Max file size check

The segment files that are the components of individual shards and the translog
generations that are components of the translog can get large (exceeding
multiple gigabytes). On systems where the max size of files that can be created
by the Elasticsearch process is limited, this can lead to failed
writes. Therefore, the safest option here is that the max file size is unlimited
and that is what the max file size bootstrap check enforces. To pass the max
file check, you must configure your system to allow the Elasticsearch process
the ability to write files of unlimited size. This can be done via
`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
you might have to increase the limits for the `root` user too).

[[max-size-virtual-memory-check]]
=== Maximum size virtual memory check

Expand All @@ -133,19 +146,6 @@ address space. This can be done via `/etc/security/limits.conf` using
the `as` setting to `unlimited` (note that you might have to increase
the limits for the `root` user too).

=== Max file size check

The segment files that are the components of individual shards and the translog
generations that are components of the translog can get large (exceeding
multiple gigabytes). On systems where the max size of files that can be created
by the Elasticsearch process is limited, this can lead to failed
writes. Therefore, the safest option here is that the max file size is unlimited
and that is what the max file size bootstrap check enforces. To pass the max
file check, you must configure your system to allow the Elasticsearch process
the ability to write files of unlimited size. This can be done via
`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
you might have to increase the limits for the `root` user too).

=== Maximum map count check

Continuing from the previous <<max-size-virtual-memory-check,point>>, to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,24 @@

package org.elasticsearch.ingest.common;

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;

import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.sameInstance;

public class ConvertProcessorTests extends ESTestCase {

Expand Down Expand Up @@ -138,6 +138,7 @@ public void testConvertLongLeadingZero() throws Exception {
assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
}

@AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
public void testConvertLongHexError() {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String value = "0x" + randomAlphaOfLengthBetween(1, 10);
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bd7d8078a2d0ad11a24f54156cc015630c96858a
2 changes: 1 addition & 1 deletion plugins/analysis-icu/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ forbiddenApis {

dependencies {
compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
compile 'com.ibm.icu:icu4j:61.1'
compile 'com.ibm.icu:icu4j:62.1'
}

dependencyLicenses {
Expand Down
1 change: 0 additions & 1 deletion plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a4d00d5ec5febd252a6182e8b6e87a0a9821f81

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a37816def72a748416c4ae8b0f6817e30efb99f

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ca7437178cdbf7b8bfe0d75c75e3c8eb93925724

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3f5dec44f380d6d58bc1c8aec51964fcb5390b60

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
453bf1d60df0415439095624e0b3e42492ad4716

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
70095a45257bca9f46629b5fb6cedf9eff5e2b07

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7199d6962d268b7877f7b5160e98e4ff21cce5c7

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
12aff508d39d206a1aead5013ecd11882062eb06
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new MMapDirectory(file));
}

@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
}

@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}
1 change: 0 additions & 1 deletion server/licenses/lucene-analyzers-common-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d27958843ca118db2ffd2c242ae3761bd5a47328
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ea220ba8e4accb8b04e280463042ad470e23bc0
1 change: 0 additions & 1 deletion server/licenses/lucene-core-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
471096d6e92338b208aa91f3a85feb2f9cfc4afd
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f0af947c60d24f779c22f774e81ebd7dd91cc932
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fbc83ac5a0139ed7e7faf6c95a2718f46f28c641
1 change: 0 additions & 1 deletion server/licenses/lucene-join-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
30adfe493982b0db059dc243e269eea38d850d46
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
656f304261d9aad05070fb68593beffafe9147e3
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8bf22ad81a7480c255b55bada401eb131bfdb4df
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ece30d5f1e18d96f61644451c858c3d9960558f
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ad3bd0c2ed96556193c7215bef328e689d0b157f
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
07e748d2d80000a7a213f3405b82b6e26b452948
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fd737bd5562f3943618ee7e73a0aaffb6319fdb2
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-7.4.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5
4 changes: 2 additions & 2 deletions server/src/main/java/org/elasticsearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -179,10 +179,10 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final int V_6_4_0_ID = 6040099;
public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
public static final int V_6_5_0_ID = 6050099;
public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
public static final int V_7_0_0_alpha1_ID = 7000001;
public static final Version V_7_0_0_alpha1 =
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
public static final Version CURRENT = V_7_0_0_alpha1;

static {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCou

@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
}

public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
Expand Down Expand Up @@ -108,13 +108,4 @@ public void setSegmentsPerTier(double segmentsPerTier) {
public double getSegmentsPerTier() {
return regularMergePolicy.getSegmentsPerTier();
}

public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}

public double getReclaimDeletesWeight() {
return regularMergePolicy.getReclaimDeletesWeight();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,6 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);

scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
mergeSchedulerConfig::setMaxThreadAndMergeCount);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public final class MergePolicyConfig {
Property.Dynamic, Property.IndexScope);
public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
Property.Dynamic, Property.IndexScope);
Property.Dynamic, Property.IndexScope, Property.Deprecated);
public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin


Expand All @@ -176,17 +176,12 @@ public final class MergePolicyConfig {
mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
mergePolicy.setSegmentsPerTier(segmentsPerTier);
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
if (logger.isTraceEnabled()) {
logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
}
}

void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}

void setSegmentsPerTier(Double segmentsPerTier) {
mergePolicy.setSegmentsPerTier(segmentsPerTier);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,14 @@ public void testDefaults() {
assertEquals(
new TieredMergePolicy().getMaxMergedSegmentMB(),
policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}

public void testSetMaxMergedSegmentMB() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergedSegmentMB(10 * 1024);
assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}

public void testSetForceMergeDeletesPctAllowed() {
Expand Down Expand Up @@ -71,10 +69,4 @@ public void testSetSegmentsPerTier() {
policy.setSegmentsPerTier(42);
assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
}

public void testSetReclaimDeletesWeight() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setReclaimDeletesWeight(42);
assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);

assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);

assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
Expand All @@ -110,7 +106,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
}

Expand Down
Loading

0 comments on commit 08b825b

Please sign in to comment.